query optimizations

GEMINI.md

@@ -69,3 +69,14 @@ This document (`GEMINI.md`) contains critical, contextual information about the
 *   **Direct Terminology:** Use technical terms like "Backup Manager", "System Status", "Archive Index". Avoid marketing fluff.
 *   **Layout:** Natural page scrolling only. No sticky headers.
 *   **Navigation:** The FileBrowser must maintain internal back/forward history separate from browser page navigation.
+
+### API & Type Safety
+*   **Explicit Response Models:** All FastAPI endpoints MUST explicitly declare a `response_model`. This is critical for generating accurate OpenAPI specs and strictly typed TypeScript SDKs for the frontend.
+*   **Centralized Schemas:** Define shared Pydantic models in `app.api.schemas` to avoid circular dependencies when importing across different routers.
+
+### Hardware Polling & Stability
+*   **Non-Intrusive Polling:** Hardware status checks (e.g., tape drive identity) must prioritize non-intrusive methods like reading the MAM (Media Auxiliary Memory) Barcode (`sg_read_attr`). Intrusive operations (like `mt rewind`) should only be used as fallbacks and never during periodic status polling when the drive is busy.
+*   **Last Known Good (LKG) Caching:** Implement LKG caching in hardware providers to persist the last successful hardware read. If a status poll fails because a device is temporarily busy with an archival job, return the LKG state instead of empty data to prevent UI flickering.
+
+### Frontend Reactivity
+*   **Svelte 5 State:** When mutating complex data structures like `Map` or `Set` in Svelte 5 `$state`, always explicitly reassign the variable (e.g., `myMap = new Map(myMap)`) after mutation to trigger the reactivity engine.

backend/app/api/inventory.py

@@ -513,15 +513,18 @@ def browse_archive_index(path: str = "ROOT", db_session: Session = Depends(get_d
     else:
         query_path = path if path.endswith("/") else path + "/"
 
-    # Find directories (immediate children)
+    # Find directories and their protection stats (Optimized: Single Pass)
     dir_sql = text("""
-        SELECT DISTINCT
-            SUBSTR(file_path, LENGTH(:prefix) + 1, INSTR(SUBSTR(file_path, LENGTH(:prefix) + 1), '/') - 1) as dir_name
+        SELECT
+            SUBSTR(file_path, LENGTH(:prefix) + 1, INSTR(SUBSTR(file_path, LENGTH(:prefix) + 1), '/') - 1) as dir_name,
+            COUNT(*) as total,
+            SUM(CASE WHEN EXISTS(SELECT 1 FROM file_versions fv WHERE fv.filesystem_state_id = filesystem_state.id) THEN 1 ELSE 0 END) as protected
         FROM filesystem_state
         WHERE file_path LIKE :prefix_wildcard
         AND file_path != :prefix
         AND INSTR(SUBSTR(file_path, LENGTH(:prefix) + 1), '/') > 0
         AND is_ignored = 0
+        GROUP BY dir_name
     """)
     dirs = db_session.execute(
         dir_sql, {"prefix": query_path, "prefix_wildcard": f"{query_path}%"}
@@ -548,23 +551,8 @@ def browse_archive_index(path: str = "ROOT", db_session: Session = Depends(get_d
         if not d[0] or d[0] == "/":
             continue
         full_dir_path = query_path + d[0]
-        # Calculate protection for this directory
-        prot_sql = text("""
-            SELECT
-                COUNT(*) as total,
-                SUM(CASE WHEN has_version = 1 THEN 1 ELSE 0 END) as protected
-            FROM (
-                SELECT EXISTS(SELECT 1 FROM file_versions fv WHERE fv.filesystem_state_id = fs.id) as has_version
-                FROM filesystem_state fs
-                WHERE fs.file_path LIKE :prefix
-                AND fs.is_ignored = 0
-            )
-        """)
-        stats = db_session.execute(
-            prot_sql, {"prefix": f"{full_dir_path}/%"}
-        ).fetchone()
-        total = stats[0] or 0
-        protected = stats[1] or 0
+        total = d[1] or 0
+        protected = d[2] or 0
 
         results.append(
             {

backend/app/api/system.py

@@ -375,48 +375,62 @@ def browse_system_path(
     if not os.path.exists(path):
         raise HTTPException(status_code=404, detail="Path not found")
 
-    # Fetch existing indexed files for this path to check for hashes
-    indexed_files = {
-        f.file_path: f.sha256_hash
-        for f in db_session.query(models.FilesystemState)
-        .filter(models.FilesystemState.file_path.like(f"{path}/%"))
-        .all()
-    }
-
     results = []
     try:
+        entries = []
+        immediate_file_paths = []
         with os.scandir(path) as directory_iterator:
             for entry in directory_iterator:
-                try:
-                    # Explicitly don't follow symlinks during browsing to show raw state
-                    file_stats = entry.stat(follow_symlinks=False)
-                    is_tracked, is_ignored = get_tracking_status(
-                        entry.path, tracking_map, exclusion_spec
-                    )
+                entries.append(entry)
+                if not entry.is_dir(follow_symlinks=False):
+                    immediate_file_paths.append(entry.path)
 
-                    # Only show files that have a hash (archivable)
-                    # Directories are always shown
-                    if not entry.is_dir():
-                        if (
-                            entry.path not in indexed_files
-                            or not indexed_files[entry.path]
-                        ):
-                            continue
-
-                    results.append(
-                        FileItemSchema(
-                            name=entry.name,
-                            path=entry.path,
-                            type="directory" if entry.is_dir() else "file",
-                            size=file_stats.st_size,
-                            mtime=file_stats.st_mtime,
-                            tracked=is_tracked,
-                            ignored=is_ignored,
-                            sha256_hash=indexed_files.get(entry.path),
-                        )
+        # Fetch existing indexed files for ONLY the immediate files in this directory
+        indexed_files = {}
+        if immediate_file_paths:
+            # Chunk the IN clause to avoid SQLite limits (typically 999)
+            for i in range(0, len(immediate_file_paths), 900):
+                chunk = immediate_file_paths[i : i + 900]
+                for file_path, sha256_hash in (
+                    db_session.query(
+                        models.FilesystemState.file_path,
+                        models.FilesystemState.sha256_hash,
                     )
-                except (OSError, FileNotFoundError):
-                    continue
+                    .filter(models.FilesystemState.file_path.in_(chunk))
+                    .all()
+                ):
+                    indexed_files[file_path] = sha256_hash
+
+        for entry in entries:
+            try:
+                # Explicitly don't follow symlinks during browsing to show raw state
+                file_stats = entry.stat(follow_symlinks=False)
+                is_tracked, is_ignored = get_tracking_status(
+                    entry.path, tracking_map, exclusion_spec
+                )
+
+                # Only show files that have a hash (archivable)
+                # Directories are always shown
+                if not entry.is_dir(follow_symlinks=False):
+                    if entry.path not in indexed_files or not indexed_files[entry.path]:
+                        continue
+
+                results.append(
+                    FileItemSchema(
+                        name=entry.name,
+                        path=entry.path,
+                        type="directory"
+                        if entry.is_dir(follow_symlinks=False)
+                        else "file",
+                        size=file_stats.st_size,
+                        mtime=file_stats.st_mtime,
+                        tracked=is_tracked,
+                        ignored=is_ignored,
+                        sha256_hash=indexed_files.get(entry.path),
+                    )
+                )
+            except (OSError, FileNotFoundError):
+                continue
     except PermissionError:
         raise HTTPException(status_code=403, detail="Permission denied")