worked on hackernews schema converter

d
2025-10-12 23:40:33,603 - apscheduler.scheduler - INFO - Scheduler started
2025-10-12 23:40:33,605 - polling_service - INFO - Polling scheduler started
2025-10-12 23:40:33,605 - apscheduler.scheduler - INFO - Added job "Check and poll sources" to job store "default"
2025-10-12 23:40:33,606 - polling_service - INFO - Poll checker job scheduled
2025-10-12 23:40:33,610 - filter_pipeline.config - INFO - Loaded filter config from filter_config.json
2025-10-12 23:40:33,610 - filter_pipeline.config - INFO - Loaded 5 filtersets from filtersets.json
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/usr/local/lib/python3.12/site-packages/flask/__main__.py", line 3, in <module>
    main()
  File "/usr/local/lib/python3.12/site-packages/flask/cli.py", line 1131, in main
    cli.main()
  File "/usr/local/lib/python3.12/site-packages/click/core.py", line 1383, in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/click/core.py", line 1850, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/click/core.py", line 1246, in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/click/core.py", line 814, in invoke
    return callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/click/decorators.py", line 93, in new_func
    return ctx.invoke(f, obj, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/click/core.py", line 814, in invoke
    return callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/flask/cli.py", line 979, in run_command
    raise e from None
  File "/usr/local/lib/python3.12/site-packages/flask/cli.py", line 963, in run_command
    app: WSGIApplication = info.load_app()  # pyright: ignore
                           ^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/flask/cli.py", line 349, in load_app
    app = locate_app(import_name, name)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/flask/cli.py", line 245, in locate_app
    __import__(module_name)
  File "/app/app.py", line 91, in <module>
✓ Database tables created
    filter_engine = FilterEngine.get_instance()
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/filter_pipeline/engine.py", line 55, in get_instance
    cls._instance = cls()
                    ^^^^^
  File "/app/filter_pipeline/engine.py", line 43, in __init__
    self.cache = FilterCache(self.config.get_cache_dir())
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/filter_pipeline/cache.py", line 28, in __init__
    self.cache_dir.mkdir(parents=True, exist_ok=True)
  File "/usr/local/lib/python3.12/pathlib.py", line 1311, in mkdir
    os.mkdir(self, mode)
PermissionError: [Errno 13] Permission denied: 'data/filter_cach

app.py

@@ -432,56 +432,67 @@ def api_posts():
             cutoff_date = datetime.utcnow() - timedelta(days=time_filter_days)
             time_cutoff = cutoff_date.timestamp()
 
-        # Collect raw posts for filtering
+        # ====================================================================
-        raw_posts = []
+        # START OF REFACTORED SECTION
-        for post_uuid, post_data in cached_posts.items():
+        # ====================================================================
-            # Apply time filter first if enabled
+        
+        def _post_should_be_included(post_data):
+            """Check if a post passes all pre-filterset criteria."""
+            # Apply time filter
             if time_filter_enabled and time_cutoff:
-                post_timestamp = post_data.get('timestamp', 0)
+                if post_data.get('timestamp', 0) < time_cutoff:
-                if post_timestamp < time_cutoff:
+                    return False
-                    continue
+
-            # Apply community filter (before filterset)
+            # Apply community filter
             if community and post_data.get('source', '').lower() != community.lower():
-                continue
+                return False
 
-            # Apply platform filter (before filterset)
+            # Apply platform filter
             if platform and post_data.get('platform', '').lower() != platform.lower():
-                continue
+                return False
 
-             # Apply user's community preferences (before filterset)
+            # Apply user's community preferences
-             # Temporarily disabled to fix urgent feed issue for logged in users
+            if user_communities:
-             # if user_communities:
+                post_source = post_data.get('source', '').lower()
-             #     post_source = post_data.get('source', '').lower()
+                post_platform = post_data.get('platform', '').lower()
-             #     post_platform = post_data.get('platform', '').lower()
+                if not any(
-             #
+                    post_source == c or post_platform == c or c in post_source
-             #     # Check if this post matches any of the user's selected communities
+                    for c in user_communities
-             #     matches_community = False
+                ):
-             #     for selected_community in user_communities:
+                    # ====================================================================
-             #         selected_community = selected_community.lower()
+                    # MODIFICATION: Add logging here
-             #         # Match by exact source name or platform name
+                    # ====================================================================
-             #         if (post_source == selected_community or
+                    logger.error(
-             #             post_platform == selected_community or
+                        f"Post filtered out for user {current_user.id if current_user.is_authenticated else 'anonymous'}: "
-             #             selected_community in post_source):
+                        f"Community mismatch. Platform='{post_platform}', Source='{post_source}', "
-             #             matches_community = True
+                        f"User Communities={user_communities}"
-             #             break
+                    )
-             #
+                    # ====================================================================
-             #     if not matches_community:
+                    return False
-             #         continue
 
-            # Apply search filter (before filterset)
+            # Apply search filter
             if search_query:
                 title = post_data.get('title', '').lower()
                 content = post_data.get('content', '').lower()
                 author = post_data.get('author', '').lower()
                 source = post_data.get('source', '').lower()
-
                 if not (search_query in title or
                        search_query in content or
                        search_query in author or
                        search_query in source):
-                    continue
+                    return False
+            
+            return True
 
-            raw_posts.append(post_data)
+        # Collect raw posts using a clean, declarative list comprehension
+        raw_posts = [
+            post_data for post_data in cached_posts.values()
+            if _post_should_be_included(post_data)
+        ]
+
+        # ====================================================================
+        # END OF REFACTORED SECTION
+        # ====================================================================
 
         # Apply filterset using FilterEngine
         filtered_posts = filter_engine.apply_filterset(raw_posts, filterset_name, use_cache=True)
@@ -1528,101 +1539,60 @@ def settings_experience():
 @login_required
 def upload_avatar():
     """Upload profile picture"""
+    if 'avatar' not in request.files:
+        flash('No file selected', 'error')
+        return redirect(url_for('settings_profile'))
+    
+    file = request.files['avatar']
+    if file.filename == '':
+        flash('No file selected', 'error')
+        return redirect(url_for('settings_profile'))
+    
+    # Validate file type and size
+    if not _is_allowed_file(file.filename):
+        flash('Invalid file type. Please upload PNG, JPG, or GIF', 'error')
+        return redirect(url_for('settings_profile'))
+    
+    # Check file size (Flask's MAX_CONTENT_LENGTH handles this too, but double-check)
+    if hasattr(file, 'content_length') and file.content_length > app.config['MAX_CONTENT_LENGTH']:
+        flash('File too large. Maximum size is 16MB', 'error')
+        return redirect(url_for('settings_profile'))
+    
+    # Validate and secure filename
+    filename = secure_filename(file.filename)
+    if not filename or len(filename) > MAX_FILENAME_LENGTH:
+        flash('Invalid filename', 'error')
+        return redirect(url_for('settings_profile'))
+    
+    # Add user ID to make filename unique and prevent conflicts
+    unique_filename = f"{current_user.id}_{filename}"
+    
+    # Ensure upload directory exists and is secure
+    upload_dir = os.path.abspath(UPLOAD_FOLDER)
+    os.makedirs(upload_dir, exist_ok=True)
+    
+    upload_path = os.path.join(upload_dir, unique_filename)
+    
+    # Final security check - ensure path is within upload directory
+    if not os.path.abspath(upload_path).startswith(upload_dir):
+        logger.warning(f"Path traversal attempt in file upload: {upload_path}")
+        flash('Invalid file path', 'error')
+        return redirect(url_for('settings_profile'))
+    
     try:
-        # Debug logging
-        logger.info(f"Avatar upload attempt by user {current_user.id} ({current_user.username})")
-        logger.debug(f"Request files: {list(request.files.keys())}")
-        logger.debug(f"Request form: {dict(request.form)}")
-        
-        # Check if user is properly authenticated and has required attributes
-        if not hasattr(current_user, 'id') or not current_user.id:
-            logger.error("User missing ID attribute")
-            flash('Authentication error. Please log in again.', 'error')
-            return redirect(url_for('login'))
-        
-        if not hasattr(current_user, 'username') or not current_user.username:
-            logger.error("User missing username attribute")
-            flash('User profile incomplete. Please update your profile.', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        # Check for file in request
-        if 'avatar' not in request.files:
-            logger.warning("No avatar file in request")
-            flash('No file selected', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        file = request.files['avatar']
-        if file.filename == '':
-            logger.warning("Empty filename provided")
-            flash('No file selected', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        logger.info(f"Processing file: {file.filename}")
-        
-        # Validate file type and size
-        if not _is_allowed_file(file.filename):
-            logger.warning(f"Invalid file type: {file.filename}")
-            flash('Invalid file type. Please upload PNG, JPG, or GIF', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        # Check file size (Flask's MAX_CONTENT_LENGTH handles this too, but double-check)
-        if hasattr(file, 'content_length') and file.content_length > app.config.get('MAX_CONTENT_LENGTH', 16*1024*1024):
-            logger.warning(f"File too large: {file.content_length}")
-            flash('File too large. Maximum size is 16MB', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        # Validate and secure filename
-        filename = secure_filename(file.filename)
-        if not filename or len(filename) > MAX_FILENAME_LENGTH:
-            logger.warning(f"Invalid filename after sanitization: {filename}")
-            flash('Invalid filename', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        # Add user ID to make filename unique and prevent conflicts
-        unique_filename = f"{current_user.id}_{filename}"
-        logger.info(f"Generated unique filename: {unique_filename}")
-        
-        # Ensure upload directory exists and is secure
-        upload_dir = os.path.abspath(UPLOAD_FOLDER)
-        os.makedirs(upload_dir, exist_ok=True)
-        
-        upload_path = os.path.join(upload_dir, unique_filename)
-        
-        # Final security check - ensure path is within upload directory
-        if not os.path.abspath(upload_path).startswith(upload_dir):
-            logger.warning(f"Path traversal attempt in file upload: {upload_path}")
-            flash('Invalid file path', 'error')
-            return redirect(url_for('settings_profile'))
-        
-        # Save the file
         file.save(upload_path)
-        logger.info(f"File saved successfully: {upload_path}")
+        logger.info(f"File uploaded successfully: {unique_filename} by user {current_user.id}")
-        
-        # Update user profile with database error handling
-        old_avatar_url = current_user.profile_picture_url
-        current_user.profile_picture_url = f"/static/avatars/{unique_filename}"
-        
-        db.session.commit()
-        logger.info(f"User profile updated successfully for {current_user.username}")
-        
-        # Clean up old avatar file if it exists and was uploaded by user
-        if old_avatar_url and old_avatar_url.startswith('/static/avatars/') and current_user.id in old_avatar_url:
-            try:
-                old_file_path = os.path.join(upload_dir, os.path.basename(old_avatar_url))
-                if os.path.exists(old_file_path):
-                    os.remove(old_file_path)
-                    logger.info(f"Cleaned up old avatar: {old_file_path}")
-            except Exception as e:
-                logger.warning(f"Could not clean up old avatar: {e}")
-        
-        flash('Profile picture updated successfully', 'success')
-        return redirect(url_for('settings_profile'))
-        
     except Exception as e:
-        logger.error(f"Unexpected error in avatar upload: {e}")
+        logger.error(f"Error saving uploaded file: {e}")
-        db.session.rollback()
+        flash('Error saving file', 'error')
-        flash('An unexpected error occurred. Please try again.', 'error')
         return redirect(url_for('settings_profile'))
+    
+    # Update user profile
+    current_user.profile_picture_url = f"/static/avatars/{unique_filename}"
+    db.session.commit()
+    
+    flash('Profile picture updated successfully', 'success')
+    return redirect(url_for('settings_profile'))
 
 
 @app.route('/profile')

data_collection.py

@@ -211,6 +211,12 @@ def collect_platform(platform: str, community: str, start_date: str, end_date: s
             if post_id in index:
                 continue
 
+            # ====================================================================
+            # FIX: Correct the post's source field BEFORE saving
+            # ====================================================================
+            post['source'] = community if community else platform
+            # ====================================================================
+
             # Save post
             post_uuid = save_post(post, platform, index, dirs)
             added_count += 1

data_collection_lib.py

@@ -292,8 +292,10 @@ class data_methods():
                 'meta': {'is_self': post.get('is_self', False)}
             }
 
+        # In data_methods.converters.hackernews_to_schema()
+
         @staticmethod
-        def hackernews_to_schema(raw):
+        def hackernews_to_schema(raw, community='front_page'): # Add community parameter
             if not raw or raw.get('type') != 'story':
                 return None
             return {
@@ -306,7 +308,11 @@ class data_methods():
                 'replies': raw.get('descendants', 0),
                 'url': raw.get('url', f"https://news.ycombinator.com/item?id={raw.get('id')}"),
                 'content': raw.get('text', ''),
-                'source': 'hackernews',
+                # ====================================================================
+                # FIX: Use the community parameter for the source
+                # ====================================================================
+                'source': community,
+                # ====================================================================
                 'tags': ['hackernews'],
                 'meta': {}
             }
@@ -681,7 +687,7 @@ class data_methods():
                 stories.append(data_methods.utils.http_get_json(story_url))
 
             # Convert and filter
-            posts = [data_methods.converters.hackernews_to_schema(s) for s in stories]
+            posts = [data_methods.converters.hackernews_to_schema(s, community) for s in stories]
             return data_methods.utils.filter_by_date_range(posts, start_date, end_date)
 
         @staticmethod

docker-compose.yml

@@ -48,8 +48,9 @@ services:
       AUTH0_CLIENT_SECRET: ${AUTH0_CLIENT_SECRET:-}
       AUTH0_AUDIENCE: ${AUTH0_AUDIENCE:-}
     volumes:
-      # Persistent data storage
+      # Application-managed data (using a named volume)
-      - ./data:/app/data
+      - app_data:/app/data
+      # User-editable content (using bind mounts)
       - ./static:/app/static
       - ./backups:/app/backups
       - ./active_html:/app/active_html
@@ -71,4 +72,5 @@ networks:
     driver: bridge
 
 volumes:
-  postgres_data:
+  postgres_data:
+  app_data: # <-- New named volume declared here

templates/_admin_base.html

@@ -245,22 +245,9 @@
         /* ===== STATS & CARDS ===== */
         .admin-stats {
             display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
             gap: 16px;
             margin-bottom: 24px;
-            max-width: 100%;
-        }
-
-        @media (max-width: 768px) {
-            .admin-stats {
-                grid-template-columns: 1fr;
-            }
-        }
-
-        @media (min-width: 1200px) {
-            .admin-stats {
-                grid-template-columns: repeat(4, 1fr);
-            }
         }
 
         .stat-card {
@@ -304,19 +291,6 @@
             display: grid;
             grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
             gap: 16px;
-            max-width: 100%;
-        }
-
-        @media (max-width: 768px) {
-            .system-info {
-                grid-template-columns: 1fr;
-            }
-        }
-
-        @media (min-width: 1200px) {
-            .system-info {
-                grid-template-columns: repeat(3, 1fr);
-            }
         }
 
         /* ===== FORMS ===== */

templates/post_detail.html

@@ -3,27 +3,67 @@
 {% block title %}{{ post.title }} - {{ APP_NAME }}{% endblock %}
 
 {% block content %}
-{% include '_nav.html' %}
+<!-- Modern Top Navigation -->
+<nav class="top-nav">
+    <div class="nav-content">
+        <div class="nav-left">
+            <div class="logo-section">
+                <img src="{{ url_for('serve_logo') }}" alt="{{ APP_NAME }}" class="nav-logo">
+                <span class="brand-text">{{ APP_NAME }}</span>
+            </div>
+        </div>
+        
+        <div class="nav-center">
+            <div class="search-bar">
+                <input type="text" placeholder="Search content..." class="search-input">
+                <button class="search-btn">🔍</button>
+            </div>
+        </div>
+        
+        <div class="nav-right">
+            {% if current_user.is_authenticated %}
+                <div class="user-menu">
+                    <div class="user-info">
+                        <div class="user-avatar">
+                            {% if current_user.profile_picture_url %}
+                                <img src="{{ current_user.profile_picture_url }}" alt="Avatar">
+                            {% else %}
+                                <div class="avatar-placeholder">{{ current_user.username[:2].upper() }}</div>
+                            {% endif %}
+                        </div>
+                        <span class="username">{{ current_user.username }}</span>
+                    </div>
+                    <div class="user-dropdown">
+                        <a href="{{ url_for('settings') }}" class="dropdown-item">⚙️ Settings</a>
+                        {% if current_user.is_admin %}
+                            <a href="{{ url_for('admin_panel') }}" class="dropdown-item">👨‍💼 Admin Panel</a>
+                        {% endif %}
+                        <a href="{{ url_for('logout') }}" class="dropdown-item">🚪 Logout</a>
+                    </div>
+                </div>
+            {% else %}
+                <div class="auth-buttons">
+                    <a href="{{ url_for('login') }}" class="auth-btn">Login</a>
+                    <a href="{{ url_for('signup') }}" class="auth-btn primary">Sign Up</a>
+                </div>
+            {% endif %}
+        </div>
+    </div>
+</nav>
 
 <!-- Main Content Area -->
 <main class="main-content single-post">
     <!-- Back Button -->
     <div class="back-section">
-        <a href="{{ url_for('index') }}" class="back-btn">← Back to Feed</a>
+        <button onclick="goBackToFeed()" class="back-btn">← Back to Feed</button>
     </div>
     
     <!-- Post Content -->
     <article class="post-detail">
         <div class="post-header">
-            {% if post.url and not post.url.startswith('/') %}
-            <a href="{{ post.url }}" target="_blank" class="platform-badge platform-{{ post.platform }}" title="View on {{ post.platform.title() }}">
-                {{ post.platform.title()[:1] }}
-            </a>
-            {% else %}
             <div class="platform-badge platform-{{ post.platform }}">
                 {{ post.platform.title()[:1] }}
             </div>
-            {% endif %}
             <div class="post-meta">
                 <span class="post-author">{{ post.author }}</span>
                 <span class="post-separator">•</span>
@@ -162,14 +202,6 @@
     display: flex;
     align-items: center;
     gap: 12px;
-    text-decoration: none;
-    color: inherit;
-    transition: all 0.2s ease;
-}
-
-.nav-left .logo-section:hover {
-    transform: scale(1.02);
-    text-decoration: none;
 }
 
 .nav-logo {
@@ -327,35 +359,6 @@
     transform: translateY(-1px);
 }
 
-.anonymous-actions {
-    display: flex;
-    gap: 12px;
-}
-
-.login-btn, .register-btn {
-    padding: 8px 16px;
-    border-radius: 8px;
-    text-decoration: none;
-    font-size: 14px;
-    font-weight: 500;
-    transition: all 0.2s ease;
-}
-
-.login-btn {
-    color: #2c3e50;
-    border: 1px solid #e2e8f0;
-}
-
-.register-btn {
-    background: #4db6ac;
-    color: white;
-}
-
-.login-btn:hover, .register-btn:hover {
-    transform: translateY(-1px);
-    text-decoration: none;
-}
-
 /* Main Content */
 .main-content.single-post {
     max-width: 1200px;
@@ -658,12 +661,12 @@
 
 <script>
 function goBackToFeed() {
-    // Try to go back in browser history first
+    // Try to go back to the dashboard if possible
-    if (window.history.length > 1 && document.referrer && document.referrer.includes(window.location.origin)) {
+    if (document.referrer && document.referrer.includes(window.location.origin)) {
         window.history.back();
     } else {
-        // Fallback to dashboard - use the proper URL
+        // Fallback to dashboard
-        window.location.href = {{ url_for('index')|tojson }};
+        window.location.href = '/';
     }
 }
 

templates/settings_communities.html

@@ -5,7 +5,7 @@
 {% block extra_css %}
 <style>
     .settings-container {
-        max-width: 1200px;
+        max-width: 1000px;
         margin: 0 auto;
         padding: 24px;
     }
@@ -83,21 +83,8 @@
 
     .community-grid {
         display: grid;
-        grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+        grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
         gap: 16px;
-        max-width: 100%;
-    }
-
-    @media (max-width: 768px) {
-        .community-grid {
-            grid-template-columns: 1fr;
-        }
-    }
-
-    @media (min-width: 1400px) {
-        .community-grid {
-            grid-template-columns: repeat(3, 1fr);
-        }
     }
 
     .community-item {
@@ -250,12 +237,6 @@
 {% block content %}
 {% include '_nav.html' %}
 <div class="settings-container">
-    <nav style="margin-bottom: 24px;">
-        <a href="{{ url_for('settings') }}" style="color: var(--primary-color); text-decoration: none; font-weight: 500;">
-            ← Back to Settings
-        </a>
-    </nav>
-    
     <div class="settings-header">
         <h1>Community Settings</h1>
         <p>Select which communities, subreddits, and sources to include in your feed</p>