From 0b302d2961c2b652ac236e3e0e3ca8c065ddf508 Mon Sep 17 00:00:00 2001 From: Adam Lamers Date: Wed, 29 Apr 2026 23:53:53 -0400 Subject: [PATCH] cleanup --- GEMINI.md | 1 + ...fdd68ee8ee3_remove_unused_backup_models.py | 48 +++ backend/app/api/backups.py | 4 +- backend/app/api/inventory.py | 14 +- backend/app/api/restores.py | 26 ++ backend/app/db/models.py | 27 -- backend/app/main.py | 5 +- backend/app/providers/mock.py | 2 +- backend/app/providers/tape.py | 5 - frontend/package-lock.json | 18 + frontend/package.json | 1 + frontend/playwright.config.ts | 6 +- frontend/src/routes/+layout.svelte | 6 +- .../src/routes/discrepancies/+page.svelte | 315 ++++++++++++++++++ frontend/src/routes/inventory/+page.svelte | 16 +- frontend/tests/discrepancies.test.ts | 191 +++++++++++ frontend/tests/full-workflow.test.ts | 30 +- frontend/tests/helpers.ts | 29 ++ 18 files changed, 676 insertions(+), 68 deletions(-) create mode 100644 backend/alembic/versions/ffdd68ee8ee3_remove_unused_backup_models.py create mode 100644 frontend/src/routes/discrepancies/+page.svelte create mode 100644 frontend/tests/discrepancies.test.ts diff --git a/GEMINI.md b/GEMINI.md index e66f85e..047f9ba 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -43,6 +43,7 @@ This document (`GEMINI.md`) contains critical, contextual information about the * **Standardized Telemetry:** Providers must implement `get_live_info(force: bool)` to return unified telemetry (e.g., drive status, capacity). * **Sanitization:** Initializing media performs a full purge of existing TapeHoard data if the `force` flag is set. * **Hardware Failure:** Marking media as "Failed" triggers an automatic atomic purge of all associated `file_versions` to surface those files as "Pending" on the dashboard. +* **Tape Registration is Discovery-Only:** Tape media (`lto_tape`, `mock_lto`) cannot be registered through the manual "Register media" dialog. Tapes are only registered via the hardware discovery section (`/inventory` → "Discovered unregistered drives") where the system auto-captures `device_path`, barcode, and serial number from the connected drive's MAM. The `device_path` is excluded from `LTOProvider.config_schema` because it is a per-drive setting configured globally in `tape_drives`, not a per-media attribute. The archiver resolves the drive at runtime when instantiating the provider. ### Database & Performance * **High Concurrency:** SQLite must always run in **WAL (Write-Ahead Logging)** mode with a 30s busy timeout and larger page cache. diff --git a/backend/alembic/versions/ffdd68ee8ee3_remove_unused_backup_models.py b/backend/alembic/versions/ffdd68ee8ee3_remove_unused_backup_models.py new file mode 100644 index 0000000..9acae5b --- /dev/null +++ b/backend/alembic/versions/ffdd68ee8ee3_remove_unused_backup_models.py @@ -0,0 +1,48 @@ +"""remove_unused_backup_models + +Revision ID: ffdd68ee8ee3 +Revises: c2512c86348b +Create Date: 2026-04-29 22:32:01.544848 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "ffdd68ee8ee3" +down_revision: Union[str, Sequence[str], None] = "c2512c86348b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.drop_table("job_logs") + op.drop_table("backups") + + +def downgrade() -> None: + """Downgrade schema.""" + op.create_table( + "backups", + sa.Column("id", sa.Integer(), primary_key=True, nullable=False), + sa.Column("job_name", sa.String(), nullable=False), + sa.Column("job_type", sa.String(), nullable=False), + sa.Column("start_time", sa.DateTime(), nullable=False), + sa.Column("end_time", sa.DateTime(), nullable=True), + sa.Column("status", sa.String(), nullable=False), + ) + op.create_table( + "job_logs", + sa.Column("id", sa.Integer(), primary_key=True, nullable=False), + sa.Column( + "backup_id", sa.Integer(), sa.ForeignKey("backups.id"), nullable=False + ), + sa.Column("timestamp", sa.DateTime(), nullable=False), + sa.Column("log_level", sa.String(), nullable=False), + sa.Column("message", sa.String(), nullable=False), + ) diff --git a/backend/app/api/backups.py b/backend/app/api/backups.py index bc47921..933e3f1 100644 --- a/backend/app/api/backups.py +++ b/backend/app/api/backups.py @@ -16,7 +16,7 @@ router = APIRouter(prefix="/backups", tags=["Backups"]) # --- Request/Response Schemas --- -class BackupJobSchema(BaseModel): +class JobSchema(BaseModel): model_config = ConfigDict(from_attributes=True) id: int @@ -113,7 +113,7 @@ def trigger_backup_job( } -@router.get("/", response_model=List[BackupJobSchema]) +@router.get("/", response_model=List[JobSchema]) def list_archival_history(db_session: Session = Depends(get_db)): """Retrieves a history of archival jobs, sorted by most recent.""" # Note: Using the generic Job model for consistency across the UI diff --git a/backend/app/api/inventory.py b/backend/app/api/inventory.py index 7dfba1a..6e3802d 100644 --- a/backend/app/api/inventory.py +++ b/backend/app/api/inventory.py @@ -786,15 +786,8 @@ def search_archive_index( if len(q) < 2: return [] - path_filter = "" - query_params = {"query": q} - - if path and path != "ROOT": - path_filter = " AND fs.file_path LIKE :path_prefix" - query_params["path_prefix"] = f"{path}%" - search_sql = text( - f""" + """ SELECT fs.id, fs.file_path, fs.size, fs.mtime, EXISTS(SELECT 1 FROM file_versions fv WHERE fv.filesystem_state_id = fs.id) as has_version, @@ -806,12 +799,15 @@ def search_archive_index( FROM filesystem_fts fts JOIN filesystem_state fs ON fs.id = fts.rowid WHERE filesystem_fts MATCH :query - {path_filter} + AND fs.file_path LIKE :path_prefix ORDER BY rank LIMIT 100 """ ) + path_prefix = f"{path}%" if path and path != "ROOT" else "%" + query_params = {"query": q, "path_prefix": path_prefix} + rows = db_session.execute(search_sql, query_params).fetchall() return [ { diff --git a/backend/app/api/restores.py b/backend/app/api/restores.py index 369b112..93ac85c 100644 --- a/backend/app/api/restores.py +++ b/backend/app/api/restores.py @@ -1,3 +1,4 @@ +import os from typing import List, Optional from fastapi import APIRouter, HTTPException, Depends, BackgroundTasks from pydantic import BaseModel, ConfigDict @@ -218,6 +219,31 @@ def trigger_recovery_job( """Initiates the background physical recovery process to the specified destination.""" destination_root = request_data.destination_path + if not os.path.isabs(destination_root): + raise HTTPException( + status_code=400, + detail="Destination path must be an absolute path.", + ) + + if ".." in destination_root: + raise HTTPException( + status_code=400, + detail="Path traversal sequences are not allowed in destination path.", + ) + + normalized = os.path.normpath(destination_root) + if normalized != destination_root: + raise HTTPException( + status_code=400, + detail=f"Destination path must be normalized: '{destination_root}' -> '{normalized}'", + ) + + if not os.path.isdir(destination_root): + raise HTTPException( + status_code=400, + detail="Destination path does not exist or is not a directory.", + ) + # Pre-validation of queue queue_count = db_session.query(models.RestoreCart).count() if queue_count == 0: diff --git a/backend/app/db/models.py b/backend/app/db/models.py index d668682..eb1ca24 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -61,19 +61,6 @@ class StorageMedia(Base): versions: Mapped[List["FileVersion"]] = relationship(back_populates="media") -class BackupJob(Base): - __tablename__ = "backups" - - id: Mapped[int] = mapped_column(primary_key=True) - job_name: Mapped[str] = mapped_column(String) - job_type: Mapped[str] = mapped_column(String) # initial, incremental - start_time: Mapped[datetime] = mapped_column(DateTime) - end_time: Mapped[Optional[datetime]] = mapped_column(DateTime) - status: Mapped[str] = mapped_column(String) # running, success, failed, aborted - - logs: Mapped[List["JobLog"]] = relationship(back_populates="backup") - - class FileVersion(Base): __tablename__ = "file_versions" @@ -150,17 +137,3 @@ class SystemSetting(Base): default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), ) - - -class JobLog(Base): - __tablename__ = "job_logs" - - id: Mapped[int] = mapped_column(primary_key=True) - backup_id: Mapped[int] = mapped_column(ForeignKey("backups.id")) - timestamp: Mapped[datetime] = mapped_column( - DateTime, default=lambda: datetime.now(timezone.utc) - ) - log_level: Mapped[str] = mapped_column(String) # INFO, ERROR, WARN - message: Mapped[str] = mapped_column(String) - - backup: Mapped["BackupJob"] = relationship(back_populates="logs") diff --git a/backend/app/main.py b/backend/app/main.py index 1c55641..54ea539 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -29,10 +29,11 @@ app = FastAPI( ) # Configure Cross-Origin Resource Sharing (CORS) -# In production, this should be restricted to your known frontend domains +# Use TAPEHOARD_CORS_ORIGINS env var (comma-separated) in production +cors_origins = os.getenv("TAPEHOARD_CORS_ORIGINS", "*").split(",") app.add_middleware( CORSMiddleware, - allow_origins=["*"], + allow_origins=[o.strip() for o in cors_origins if o.strip()], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], diff --git a/backend/app/providers/mock.py b/backend/app/providers/mock.py index d7c071a..a0f58f9 100644 --- a/backend/app/providers/mock.py +++ b/backend/app/providers/mock.py @@ -35,7 +35,7 @@ class MockLTOProvider(AbstractStorageProvider): "device_path": { "type": "string", "title": "Mock Directory Path", - "description": "Path to a directory representing the tape drive (optional — auto-created if omitted)", + "description": "Path to a directory representing the tape drive (used internally for testing).", "required": False, } } diff --git a/backend/app/providers/tape.py b/backend/app/providers/tape.py index db94513..dd05d79 100644 --- a/backend/app/providers/tape.py +++ b/backend/app/providers/tape.py @@ -18,11 +18,6 @@ class LTOProvider(AbstractStorageProvider): "supports_hardware_encryption": True, } config_schema = { - "device_path": { - "type": "string", - "title": "Device Path", - "description": "e.g., /dev/nst0", - }, "compression": { "type": "boolean", "title": "Hardware Compression", diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 730c640..bcee27a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -18,6 +18,7 @@ "@sveltejs/vite-plugin-svelte": "^7.0.0", "@tailwindcss/postcss": "^4.2.4", "@tailwindcss/vite": "^4.2.4", + "@types/node": "^25.6.0", "autoprefixer": "^10.5.0", "bits-ui": "^1.0.0-next.98", "clsx": "^2.1.1", @@ -1042,6 +1043,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "25.6.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", + "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.19.0" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -2670,6 +2681,13 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "7.19.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", + "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "dev": true, + "license": "MIT" + }, "node_modules/update-browserslist-db": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index 190667a..464fb72 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,6 +21,7 @@ "@sveltejs/vite-plugin-svelte": "^7.0.0", "@tailwindcss/postcss": "^4.2.4", "@tailwindcss/vite": "^4.2.4", + "@types/node": "^25.6.0", "autoprefixer": "^10.5.0", "bits-ui": "^1.0.0-next.98", "clsx": "^2.1.1", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 64a61f1..6d8d13c 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -19,7 +19,7 @@ export default defineConfig({ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: 'http://localhost:5173', + baseURL: 'http://localhost:5174', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -43,8 +43,8 @@ export default defineConfig({ timeout: 120 * 1000, }, { - command: 'VITE_API_URL=http://localhost:8001 npm run dev', - url: 'http://localhost:5173', + command: 'VITE_API_URL=http://localhost:8001 npm run dev -- --port 5174', + url: 'http://localhost:5174', reuseExistingServer: false, timeout: 120 * 1000, }, diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index 7f903d9..18a55ce 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -14,7 +14,8 @@ Activity, ChevronLeft, ChevronRight, - PieChart + PieChart, + AlertTriangle } from 'lucide-svelte'; import { cn } from '$lib/utils'; import { Toaster } from 'svelte-sonner'; @@ -32,6 +33,7 @@ { name: 'Insights', href: '/insights', icon: PieChart }, { name: 'Archive Index', href: '/index-browser', icon: Library }, { name: 'Filesystem', href: '/filesystem', icon: FolderTree }, + { name: 'Discrepancies', href: '/discrepancies', icon: AlertTriangle }, { name: 'Jobs', href: '/jobs', icon: Activity }, { name: 'Media Inventory', href: '/inventory', icon: CassetteTape }, { name: 'Data Recovery', href: '/restores', icon: History } @@ -61,6 +63,7 @@ else if (key === 'g') window.location.href = '/insights'; else if (key === 'i') window.location.href = '/index-browser'; else if (key === 't') window.location.href = '/filesystem'; + else if (key === 'x') window.location.href = '/discrepancies'; else if (key === 'a') window.location.href = '/jobs'; else if (key === 'm') window.location.href = '/inventory'; else if (key === 'r') window.location.href = '/restores'; @@ -192,6 +195,7 @@
Insights G
Archive Index I
Filesystem T
+
Discrepancies X
Jobs A
diff --git a/frontend/src/routes/discrepancies/+page.svelte b/frontend/src/routes/discrepancies/+page.svelte new file mode 100644 index 0000000..e9b2276 --- /dev/null +++ b/frontend/src/routes/discrepancies/+page.svelte @@ -0,0 +1,315 @@ + + + + Discrepancies - TapeHoard + + +
+ + {#snippet actions()} + + {/snippet} + + + {#if loading && discrepancies.length === 0} +
+
+
+
+ {:else if discrepancies.length === 0} + +
+ +
+

All clear

+

+ No discrepancies detected. All tracked files are present on disk. +

+ + {:else} +
+ +
+ +
+
+ +
+
+ Confirmed deleted +

{deletedItems.length}

+

Files marked as removed from disk

+
+
+ + + +
+
+ +
+
+ Missing from disk +

{missingItems.length}

+

Tracked files not found during scan

+
+
+ +
+ + + +
+ + + + + + + + + + + + + {#each discrepancies as item (item.id)} + + + + + + + + + {/each} + +
StatusFile pathSizeLast seenBacked upActions
+ {#if item.is_deleted} + Deleted + {:else} + Missing + {/if} + +
+ + {item.path.split('/').pop()} + + + {item.path} + +
+ 		+ {formatSize(item.size)} + + + {#if item.last_seen_timestamp} + {formatLocalDate(item.last_seen_timestamp)} + {:else} + — + {/if} + + + {#if item.has_versions} +
+ + Yes +
+ {:else} +
+ + No +
+ {/if} + 		+
+ {#if item.is_deleted} + + + {:else} + + + {/if} +
+
+
+ +
+ {/if} +
diff --git a/frontend/src/routes/inventory/+page.svelte b/frontend/src/routes/inventory/+page.svelte index 617e3e3..4b6e125 100644 --- a/frontend/src/routes/inventory/+page.svelte +++ b/frontend/src/routes/inventory/+page.svelte @@ -539,8 +539,13 @@ {/if} {#if asset.hardware_info.tape}
- MFR: {asset.hardware_info.tape.manufacturer} - {asset.hardware_info.tape.generation_label || asset.hardware_info.tape.generation} + {#if asset.hardware_info.tape.barcode} + BC: {asset.hardware_info.tape.barcode} + {/if} + {#if asset.hardware_info.tape.serial} + SN: {asset.hardware_info.tape.serial} + {/if} + {asset.hardware_info.tape.generation_label || asset.hardware_info.tape.generation}
{/if}
@@ -560,6 +565,11 @@ } } else if (asset.type === 'tape') { dynamicConfig.device_path = asset.device_path; + dynamicConfig.serial = asset.hardware_info?.tape?.serial || ''; + dynamicConfig.barcode = asset.hardware_info?.tape?.barcode || ''; + if (asset.hardware_info?.tape?.max_capacity_mib) { + newMedia.capacity_gb = Math.floor(asset.hardware_info.tape.max_capacity_mib / 1024); + } } showRegisterDialog = true; }}>Add media @@ -856,7 +866,7 @@
- {#each providersList as provider} + {#each providersList.filter(p => !['lto_tape', 'mock_lto'].includes(p.provider_id)) as provider}