feat(watcher): add filesystem watcher with dashboard integration (fixed #28)

app/db.py

@@ -77,9 +77,14 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
     if not _column_exists(conn, "meta", "format"):
         _add_column(conn, "meta", "format", "TEXT")
 
+    # migration: ensure 'added_at' column exists in items
+    if not _column_exists(conn, "items", "added_at"):
+        _add_column(conn, "items", "added_at", "REAL")
+
     conn.execute("CREATE INDEX IF NOT EXISTS idx_items_parent   ON items(parent)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_items_name     ON items(name)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_items_isdir    ON items(is_dir)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_items_added_at ON items(added_at)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_meta_series    ON meta(series)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_meta_title     ON meta(title)")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_meta_year      ON meta(year)")
@@ -108,34 +113,42 @@ def begin_scan(conn: sqlite3.Connection) -> None:
         conn.execute("DELETE FROM fts")
     conn.commit()
 
-def upsert_dir(conn: sqlite3.Connection, rel: str, name: str, parent: str, mtime: float) -> None:
+def upsert_dir(conn: sqlite3.Connection, rel: str, name: str, parent: str, mtime: float, added_at: Optional[float] = None) -> None:
+    import time
+    if added_at is None:
+        added_at = time.time()
     conn.execute(
         """
-        INSERT INTO items(rel, name, parent, is_dir, size, mtime, ext)
-        VALUES (?, ?, ?, 1, NULL, ?, NULL)
+        INSERT INTO items(rel, name, parent, is_dir, size, mtime, ext, added_at)
+        VALUES (?, ?, ?, 1, NULL, ?, NULL, ?)
         ON CONFLICT(rel) DO UPDATE SET
           name=excluded.name,
           parent=excluded.parent,
           is_dir=excluded.is_dir,
-          mtime=excluded.mtime
+          mtime=excluded.mtime,
+          added_at=COALESCE(items.added_at, excluded.added_at)
         """,
-        (rel, name, parent, mtime),
+        (rel, name, parent, mtime, added_at),
     )
 
-def upsert_file(conn: sqlite3.Connection, rel: str, name: str, size: int, mtime: float, parent: str, ext: str) -> None:
+def upsert_file(conn: sqlite3.Connection, rel: str, name: str, size: int, mtime: float, parent: str, ext: str, added_at: Optional[float] = None) -> None:
+    import time
+    if added_at is None:
+        added_at = time.time()
     conn.execute(
         """
-        INSERT INTO items(rel, name, parent, is_dir, size, mtime, ext)
-        VALUES (?, ?, ?, 0, ?, ?, ?)
+        INSERT INTO items(rel, name, parent, is_dir, size, mtime, ext, added_at)
+        VALUES (?, ?, ?, 0, ?, ?, ?, ?)
         ON CONFLICT(rel) DO UPDATE SET
           name=excluded.name,
           parent=excluded.parent,
           is_dir=excluded.is_dir,
           size=excluded.size,
           mtime=excluded.mtime,
-          ext=excluded.ext
+          ext=excluded.ext,
+          added_at=COALESCE(items.added_at, excluded.added_at)
         """,
-        (rel, name, parent, size, mtime, ext),
+        (rel, name, parent, size, mtime, ext, added_at),
     )
 
 def upsert_meta(conn: sqlite3.Connection, rel: str, meta: Dict[str, Any]) -> None:
@@ -498,10 +511,10 @@ def _order_by_for_sort(sort: str) -> str:
         return "COALESCE(m.title, i.name) COLLATE NOCASE ASC"
     if s == "series_number":
         return "COALESCE(m.series, i.name) COLLATE NOCASE ASC, CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER) ASC, i.name ASC"
+    if s == "added" or s == "added_desc":
+        return "COALESCE(i.added_at, i.mtime) DESC"
     if s == "added_asc":
-        return "i.mtime ASC"
-    if s == "added_desc":
-        return "i.mtime DESC"
+        return "COALESCE(i.added_at, i.mtime) ASC"
     return "COALESCE(m.series, i.name) ASC, " \
            "CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER) ASC, i.name ASC"
 

app/main.py

@@ -21,11 +21,12 @@ import sys
 import logging
 from math import ceil
 
-from .config import LIBRARY_DIR, PAGE_SIZE, SERVER_BASE, URL_PREFIX, PRECACHE_THUMBS, THUMB_WORKERS, PRECACHE_ON_START, AUTO_INDEX_ON_START
+from .config import LIBRARY_DIR, PAGE_SIZE, SERVER_BASE, URL_PREFIX, PRECACHE_THUMBS, THUMB_WORKERS, PRECACHE_ON_START, AUTO_INDEX_ON_START, ENABLE_WATCH
 from .opds import now_rfc3339, mime_for
 from .auth import require_basic
 from .thumbs import have_thumb, generate_thumb
 from . import db  # SQLite adapter
+from . import watcher  # Filesystem watcher
 
 # -------------------- Logging --------------------
 LOG_LEVEL = os.getenv("LOG_LEVEL", "ERROR").upper()
@@ -105,6 +106,9 @@ _INDEX_STATUS = {
 }
 _INDEX_LOCK = threading.Lock()
 
+# -------------------- Filesystem watcher --------------------
+_LIBRARY_WATCHER: Optional[watcher.LibraryWatcher] = None
+
 # -------------------- Small helpers --------------------
 def rget(row, key: str, default=None):
     """Safe access for sqlite3.Row (no .get())."""
@@ -342,6 +346,13 @@ def startup():
         app_logger.info(f"Page cache auto-clean enabled: every {PAGE_CACHE_CLEAN_INTERVAL_MIN} min, "
                         f"ttl={PAGE_CACHE_TTL_DAYS}d, cap={PAGE_CACHE_MAX_BYTES} bytes")
 
+    # Start filesystem watcher if enabled
+    if ENABLE_WATCH:
+        global _LIBRARY_WATCHER
+        _LIBRARY_WATCHER = watcher.LibraryWatcher(LIBRARY_DIR)
+        _LIBRARY_WATCHER.start()
+        app_logger.warning(f"✓ Filesystem watcher started monitoring: {LIBRARY_DIR}")
+
 
     conn = db.connect()
     try:
@@ -354,6 +365,15 @@ def startup():
     else:
         _set_status(running=False, phase="idle", total=0, done=0, current="", ended_at=time.time())
 
+@app.on_event("shutdown")
+def shutdown():
+    """Clean up resources on shutdown."""
+    global _LIBRARY_WATCHER
+    if _LIBRARY_WATCHER:
+        _LIBRARY_WATCHER.stop()
+        _LIBRARY_WATCHER = None
+        app_logger.warning("✓ Filesystem watcher stopped")
+
 # -------------------- PSE (Page Streaming) helpers --------------------
 PAGE_CACHE_DIR = Path("/data/pages")
 VALID_PAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff"}
@@ -1078,6 +1098,14 @@ def stats(_=Depends(require_basic)):
 
     return JSONResponse(payload)
 
+@app.get("/watcher/status", response_class=JSONResponse)
+def watcher_status(_=Depends(require_basic)):
+    """Get filesystem watcher status."""
+    global _LIBRARY_WATCHER
+    if _LIBRARY_WATCHER:
+        return JSONResponse(_LIBRARY_WATCHER.get_status())
+    return JSONResponse({"enabled": False, "running": False, "event_count": 0, "last_event_time": 0.0})
+
 # -------------------- Debug --------------------
 @app.get("/debug/children", response_class=JSONResponse)
 def debug_children(path: str = ""):

app/templates/dashboard.html

@@ -35,6 +35,8 @@
           • Errors: <a id="errLink" href="#" class="link-danger text-decoration-none"><span id="errCount">0</span></a>
           <!-- NEW: live page cache status -->
           • Cache: <span id="cacheStatus" class="badge text-bg-light cache-pill">—</span>
+          <!-- NEW: watcher status -->
+          • <span id="watcherStatus" class="badge text-bg-secondary" title="Filesystem watcher"><i class="bi bi-eye"></i> Watch</span>
         </span>
         <button id="thumbsBtn" class="btn btn-sm btn-outline-primary">
           <i class="bi bi-images me-1"></i> Pre-cache Thumbnails
@@ -418,10 +420,53 @@
     }
     document.getElementById("cleanCacheBtn").addEventListener("click", cleanCache);
 
+    // ----- Watcher status -----
+    let lastWatcherCount = 0;
+    async function pollWatcher(){
+      let delay=3000;
+      try{
+        const w = await jget("/watcher/status");
+        const badge = document.getElementById("watcherStatus");
+        if (w && w.enabled && w.running) {
+          // Show recent count (last 5 minutes) if events have occurred
+          const recentCount = w.recent_count || 0;
+          if (recentCount > 0) {
+            badge.innerHTML = `<i class="bi bi-eye-fill"></i> Watch (${recentCount})`;
+            badge.className = "badge text-bg-success";
+            badge.title = `${recentCount} events in last 5 minutes (${w.event_count} total)`;
+
+            // If count changed, briefly highlight and reload stats
+            if (w.event_count !== lastWatcherCount && lastWatcherCount > 0) {
+              badge.className = "badge text-bg-warning";
+              setTimeout(() => { badge.className = "badge text-bg-success"; }, 800);
+              // Reload library stats to reflect changes
+              setTimeout(load, 1000);
+            }
+            lastWatcherCount = w.event_count;
+          } else {
+            badge.innerHTML = '<i class="bi bi-eye"></i> Watch';
+            badge.className = "badge text-bg-success";
+            badge.title = w.event_count > 0 ? `Watching (${w.event_count} events total)` : "Filesystem watcher active";
+          }
+          delay = 3000;
+        } else if (w && w.enabled && !w.running) {
+          badge.innerHTML = '<i class="bi bi-eye-slash"></i> Watch';
+          badge.className = "badge text-bg-warning";
+          badge.title = "Watcher enabled but not running";
+        } else {
+          badge.innerHTML = '<i class="bi bi-eye-slash"></i> Off';
+          badge.className = "badge text-bg-secondary";
+          badge.title = "Filesystem watcher disabled";
+        }
+      }catch{ delay=5000; }
+      setTimeout(pollWatcher, delay);
+    }
+
     // Initial load & polls
     load();
     pollIndex();
     pollThumbs();
+    pollWatcher();
     updateCacheStatus();
     // refresh cache pill periodically
     setInterval(updateCacheStatus, 120000); // every 2 min

app/templates/smartlists.html

@@ -62,6 +62,7 @@
                   <option value="series_number">Series + Number</option>
                   <option value="title">Title</option>
                   <option value="publisher">Publisher</option>
+                  <option value="added">Recently Added</option>
                 </select>
               </div>
               <div class="col-6 col-md-2">

app/watcher.py

@@ -0,0 +1,375 @@
+# app/watcher.py
+"""
+Filesystem watcher for incremental library updates.
+Uses watchdog to monitor LIBRARY_DIR for changes and update the database in real-time.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from pathlib import Path
+from typing import Optional
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileSystemEvent
+
+from app import db
+
+logger = logging.getLogger("comicopds.watcher")
+
+VALID_EXTS = {".cbz"}
+
+
+class LibraryEventHandler(FileSystemEventHandler):
+    """Handles filesystem events for the comic library."""
+
+    def __init__(self, library_root: Path):
+        self.library_root = library_root.resolve()
+        self.event_count = 0
+        self.last_event_time = 0.0
+        self.recent_events = []  # List of (timestamp, event_type) tuples
+
+    def _record_event(self, event_type: str):
+        """Record an event and clean up old ones (older than 5 minutes)."""
+        now = time.time()
+        self.recent_events.append((now, event_type))
+        self.event_count += 1
+        self.last_event_time = now
+
+        # Keep only events from last 5 minutes
+        cutoff = now - 300  # 5 minutes
+        self.recent_events = [(t, e) for t, e in self.recent_events if t > cutoff]
+
+    def get_recent_count(self) -> int:
+        """Get count of events in last 5 minutes."""
+        cutoff = time.time() - 300
+        return len([1 for t, _ in self.recent_events if t > cutoff])
+
+    def _get_connection(self) -> db.sqlite3.Connection:
+        """Get a database connection for the current thread."""
+        return db.connect()
+
+    def start(self):
+        """Initialize handler (no-op, connections created per-thread)."""
+        pass
+
+    def stop(self):
+        """Stop handler (no-op, connections closed per-operation)."""
+        pass
+
+    def _relpath(self, path: Path) -> str:
+        """Get relative path from library root."""
+        try:
+            return path.relative_to(self.library_root).as_posix()
+        except ValueError:
+            return ""
+
+    def _parent_rel(self, rel: str) -> str:
+        """Get parent relative path."""
+        return "" if "/" not in rel else rel.rsplit("/", 1)[0]
+
+    def _read_comicinfo(self, cbz_path: Path) -> dict:
+        """Read ComicInfo.xml from a CBZ file."""
+        from xml.etree import ElementTree as ET
+        import zipfile
+
+        meta = {}
+        try:
+            with zipfile.ZipFile(cbz_path, "r") as zf:
+                xml_name = None
+                for n in zf.namelist():
+                    if n.lower().endswith("comicinfo.xml") and not n.endswith("/"):
+                        xml_name = n
+                        break
+                if not xml_name:
+                    return meta
+                with zf.open(xml_name) as fp:
+                    tree = ET.parse(fp)
+                    root = tree.getroot()
+                    for el in root:
+                        k = el.tag.lower()
+                        v = (el.text or "").strip()
+                        if v:
+                            meta[k] = v
+                    if "title" not in meta and "booktitle" in meta:
+                        meta["title"] = meta.get("booktitle")
+                    for k in ("number", "volume", "year", "month", "day"):
+                        if k in meta:
+                            meta[k] = meta[k].strip()
+        except Exception as e:
+            logger.debug(f"Failed to read ComicInfo from {cbz_path}: {e}")
+        return meta
+
+    def on_created(self, event: FileSystemEvent):
+        """Handle file/directory creation."""
+        if event.is_directory:
+            return
+
+        path = Path(event.src_path)
+
+        # Only handle .cbz files
+        if path.suffix.lower() not in VALID_EXTS:
+            return
+
+        conn = None
+        try:
+            if not path.exists():
+                return
+
+            rel = self._relpath(path)
+            if not rel:
+                return
+
+            st = path.stat()
+            added_at = time.time()
+
+            conn = self._get_connection()
+
+            # Insert file
+            db.upsert_file(
+                conn,
+                rel=rel,
+                name=path.stem,
+                size=st.st_size,
+                mtime=st.st_mtime,
+                parent=self._parent_rel(rel),
+                ext="cbz",
+                added_at=added_at
+            )
+
+            # Read and insert metadata
+            meta = self._read_comicinfo(path)
+            if meta:
+                db.upsert_meta(conn, rel=rel, meta=meta)
+
+            # Update FTS index if enabled
+            if db.has_fts5():
+                text_parts = [
+                    path.stem,
+                    meta.get("title", ""),
+                    meta.get("series", ""),
+                    meta.get("writer", ""),
+                    meta.get("publisher", "")
+                ]
+                text = " ".join([p for p in text_parts if p])
+                conn.execute(
+                    "INSERT OR REPLACE INTO fts(rel, text) VALUES (?, ?)",
+                    (rel, text)
+                )
+
+            conn.commit()
+            self._record_event("create")
+            logger.warning(f"✓ Added: {rel}")
+
+        except Exception as e:
+            logger.error(f"Failed to index created file {event.src_path}: {e}")
+            if conn:
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+        finally:
+            if conn:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+
+    def on_modified(self, event: FileSystemEvent):
+        """Handle file modification."""
+        if event.is_directory:
+            return
+
+        path = Path(event.src_path)
+
+        # Only handle .cbz files
+        if path.suffix.lower() not in VALID_EXTS:
+            return
+
+        conn = None
+        try:
+            if not path.exists():
+                return
+
+            rel = self._relpath(path)
+            if not rel:
+                return
+
+            st = path.stat()
+
+            conn = self._get_connection()
+
+            # Update file (preserves existing added_at)
+            db.upsert_file(
+                conn,
+                rel=rel,
+                name=path.stem,
+                size=st.st_size,
+                mtime=st.st_mtime,
+                parent=self._parent_rel(rel),
+                ext="cbz"
+            )
+
+            # Re-read and update metadata
+            meta = self._read_comicinfo(path)
+            if meta:
+                db.upsert_meta(conn, rel=rel, meta=meta)
+
+            # Update FTS index if enabled
+            if db.has_fts5():
+                text_parts = [
+                    path.stem,
+                    meta.get("title", ""),
+                    meta.get("series", ""),
+                    meta.get("writer", ""),
+                    meta.get("publisher", "")
+                ]
+                text = " ".join([p for p in text_parts if p])
+                conn.execute(
+                    "INSERT OR REPLACE INTO fts(rel, text) VALUES (?, ?)",
+                    (rel, text)
+                )
+
+            conn.commit()
+            self._record_event("modify")
+            logger.warning(f"✓ Updated: {rel}")
+
+        except Exception as e:
+            logger.error(f"Failed to update modified file {event.src_path}: {e}")
+            if conn:
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+        finally:
+            if conn:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+
+    def on_deleted(self, event: FileSystemEvent):
+        """Handle file/directory deletion."""
+        path = Path(event.src_path)
+        rel = self._relpath(path)
+
+        if not rel:
+            return
+
+        conn = None
+        try:
+            conn = self._get_connection()
+
+            if event.is_directory:
+                # Delete directory and all its contents
+                # Use LIKE to match all paths that start with this directory
+                pattern = f"{rel}/%"
+                conn.execute("DELETE FROM items WHERE rel=? OR rel LIKE ?", (rel, pattern))
+                conn.execute("DELETE FROM meta WHERE rel=? OR rel LIKE ?", (rel, pattern))
+                if db.has_fts5():
+                    conn.execute("DELETE FROM fts WHERE rel=? OR rel LIKE ?", (rel, pattern))
+                logger.warning(f"✓ Deleted directory and contents: {rel}")
+            else:
+                # Delete single file
+                conn.execute("DELETE FROM items WHERE rel=?", (rel,))
+                conn.execute("DELETE FROM meta WHERE rel=?", (rel,))
+                if db.has_fts5():
+                    conn.execute("DELETE FROM fts WHERE rel=?", (rel,))
+                logger.warning(f"✓ Deleted: {rel}")
+
+            conn.commit()
+            self._record_event("delete")
+
+        except Exception as e:
+            logger.error(f"Failed to delete {event.src_path}: {e}")
+            if conn:
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+        finally:
+            if conn:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+
+    def on_moved(self, event: FileSystemEvent):
+        """Handle file/directory move/rename."""
+        dest_str = getattr(event, 'dest_path', 'unknown')
+        logger.warning(f"✓ Move: {event.src_path} -> {dest_str}")
+
+        # Always delete from source
+        self.on_deleted(event)
+
+        # Only create if destination is within the watched library
+        if hasattr(event, 'dest_path'):
+            dest_path = Path(event.dest_path)
+            try:
+                # Check if destination is within library root
+                dest_rel = self._relpath(dest_path)
+                if dest_rel:
+                    # Destination is within library, create it
+                    from watchdog.events import FileCreatedEvent, DirCreatedEvent
+                    dest_event = FileCreatedEvent(event.dest_path) if not event.is_directory else DirCreatedEvent(event.dest_path)
+                    self.on_created(dest_event)
+                else:
+                    logger.warning(f"  → Destination outside library, removed from index")
+            except Exception as e:
+                logger.debug(f"Destination not in library: {e}")
+
+
+class LibraryWatcher:
+    """Watches the library directory for changes."""
+
+    def __init__(self, library_root: Path):
+        self.library_root = library_root
+        self.observer: Optional[Observer] = None
+        self.event_handler: Optional[LibraryEventHandler] = None
+
+    def get_status(self) -> dict:
+        """Get watcher status."""
+        if self.event_handler:
+            return {
+                "enabled": True,
+                "running": self.observer is not None and self.observer.is_alive(),
+                "event_count": self.event_handler.event_count,
+                "recent_count": self.event_handler.get_recent_count(),
+                "last_event_time": self.event_handler.last_event_time
+            }
+        return {
+            "enabled": False,
+            "running": False,
+            "event_count": 0,
+            "recent_count": 0,
+            "last_event_time": 0.0
+        }
+
+    def start(self):
+        """Start watching the library directory."""
+        if self.observer is not None:
+            logger.warning("Watcher already started")
+            return
+
+        self.event_handler = LibraryEventHandler(self.library_root)
+        self.event_handler.start()
+
+        self.observer = Observer()
+        self.observer.schedule(self.event_handler, str(self.library_root), recursive=True)
+        self.observer.start()
+
+        logger.info(f"Started watching {self.library_root}")
+
+    def stop(self):
+        """Stop watching the library directory."""
+        if self.observer:
+            self.observer.stop()
+            self.observer.join(timeout=5)
+            self.observer = None
+
+        if self.event_handler:
+            self.event_handler.stop()
+            self.event_handler = None
+
+        logger.info("Stopped watching library")

docs/configuration.md

@@ -11,7 +11,7 @@
 | `DISABLE_AUTH`        | `false`     | Optional | If `true`, disables authentication completely (public access). |
 | `OPDS_BASIC_USER`     | `admin`     | Optional | Username for HTTP Basic Auth. Ignored if `DISABLE_AUTH=true`. |
 | `OPDS_BASIC_PASS`     | `change-me` | Optional | Password for HTTP Basic Auth. Ignored if `DISABLE_AUTH=true`. |
-| `ENABLE_WATCH`        | `false`     | Optional | Watch filesystem for changes and update index incrementally. (`true`/`false`). |
+| `ENABLE_WATCH`        | `false`     | Optional | Watch filesystem for changes and update index incrementally. When enabled, new/modified/deleted comic files are automatically indexed without requiring a full rescan. Useful for large libraries. (`true`/`false`). |
 | `AUTO_INDEX_ON_START` | `false`     | Optional | If `true`, reindexes library on every container start. Recommended `false` for large libraries. |
 | `PRECACHE_THUMBS`     | `false`     | Optional | If `true`, enables thumbnail generation when reindexing or via dashboard. |
 | `PRECACHE_ON_START`   | `false`     | Optional | If `true`, automatically triggers full thumbnail pre-cache at container start. Recommended `false` for large libraries.  |

docs/smartlists.md

@@ -189,6 +189,11 @@ Dynamic smart lists work with all regular smart list features:
 - **Distinct**: De-duplicate by series+volume
 - **Limit**: Cap results per sub-folder
 - **Sort**: Control ordering within each group
+  - `Issued (newest first)` - Sort by publication date (year/month/day)
+  - `Series + Number` - Sort by series name and issue number
+  - `Title` - Sort alphabetically by title
+  - `Publisher` - Group by publisher, then series
+  - `Recently Added` - Sort by when the file was added to your library (requires `ENABLE_WATCH=true` for accurate timestamps)
 
 #### JSON Configuration Example
 

requirements.txt

@@ -2,3 +2,4 @@ fastapi>=0.111.0
 uvicorn[standard]>=0.30.0
 jinja2>=3.1.4
 pillow>=10.4.0
+watchdog>=4.0.0