diff --git a/AGENTS.md b/AGENTS.md index 62d2ae5..1fc31b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,10 +92,11 @@ All FastAPI route handlers must declare explicit `operation_id` to control the g Generated from the backend OpenAPI spec using `@hey-api/openapi-ts`: ```bash -cd backend && uv run python -c "import json; from app.main import app; json.dump(app.openapi(), open('openapi.json', 'w'))" -cd ../frontend && npx @hey-api/openapi-ts -i ../backend/openapi.json -o src/lib/api +just generate-client ``` +This runs the full pipeline: exports the OpenAPI spec from the running FastAPI app and regenerates the TypeScript SDK in `frontend/src/lib/api/`. Use this **after any backend change** that adds, renames, or modifies endpoints or schemas. + The generated SDK exports clean camelCase functions (e.g., `getDashboardStats`, `listJobs`, `triggerScan`). **Rule:** After renaming any backend handler or changing an `operation_id`, regenerate the SDK and update all frontend imports. The old verbose names will cause TypeScript errors. @@ -151,10 +152,19 @@ On macOS, `localhost` resolves to `::1` (IPv6) by default, but uvicorn may bind 7. Add backend tests in `backend/tests/test_api_system.py` (or a new test file if it's a new domain). 8. Run `just lint` before finishing. -### Regenerating the OpenAPI Spec +### Regenerating the OpenAPI Spec / TypeScript SDK + +Use the convenience command: + +```bash +just generate-client +``` + +Or run the steps manually: ```bash cd backend && uv run python -c "import json; from app.main import app; json.dump(app.openapi(), open('openapi.json', 'w'), indent=2)" +cd ../frontend && npx @hey-api/openapi-ts -i ../backend/openapi.json -o src/lib/api ``` ### Verifying No Auto-Generated operationIds diff --git a/backend/alembic/versions/806e933ac89b_add_is_ignored_by_policy_to_filesystem_.py b/backend/alembic/versions/806e933ac89b_add_is_ignored_by_policy_to_filesystem_.py new file mode 100644 index 0000000..bbf93ee --- /dev/null +++ b/backend/alembic/versions/806e933ac89b_add_is_ignored_by_policy_to_filesystem_.py @@ -0,0 +1,56 @@ +"""add is_ignored_by_policy to filesystem_state + +Revision ID: 806e933ac89b +Revises: 349e61f9e856 +Create Date: 2026-05-04 19:34:38.280865 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "806e933ac89b" +down_revision: Union[str, Sequence[str], None] = "349e61f9e856" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # Check if column already exists (e.g. from manual migration) + conn = op.get_bind() + inspector = sa.inspect(conn) + columns = [c["name"] for c in inspector.get_columns("filesystem_state")] + + if "is_ignored_by_policy" not in columns: + op.add_column( + "filesystem_state", + sa.Column( + "is_ignored_by_policy", sa.Boolean(), nullable=False, server_default="0" + ), + ) + + # Create index for efficient querying + indexes = [idx["name"] for idx in inspector.get_indexes("filesystem_state")] + if "ix_filesystem_state_is_ignored_by_policy" not in indexes: + op.create_index( + "ix_filesystem_state_is_ignored_by_policy", + "filesystem_state", + ["is_ignored_by_policy"], + unique=False, + ) + + # Backfill: set is_ignored_by_policy = is_ignored for existing records + op.execute("UPDATE filesystem_state SET is_ignored_by_policy = is_ignored") + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index( + "ix_filesystem_state_is_ignored_by_policy", table_name="filesystem_state" + ) + op.drop_column("filesystem_state", "is_ignored_by_policy") diff --git a/backend/app/api/common.py b/backend/app/api/common.py index 08f077a..b751fec 100644 --- a/backend/app/api/common.py +++ b/backend/app/api/common.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional import pathspec from pydantic import BaseModel, ConfigDict +from sqlalchemy import text from sqlalchemy.orm import Session from app.db import models @@ -82,6 +83,41 @@ def get_ignored_status( return False +def get_ignored_by_policy( + absolute_path: str, + exclusion_spec: Optional[pathspec.PathSpec], +) -> bool: + """Determines if a path is excluded by global policy only (ignores manual tracking rules).""" + if exclusion_spec and exclusion_spec.match_file(absolute_path): + return True + return False + + +def recompute_exclusion_policy(db_session: Session) -> None: + """Recomputes is_ignored_by_policy and effective is_ignored for all indexed files.""" + exclusion_spec = get_exclusion_spec(db_session) + tracking_rules = db_session.query(models.TrackedSource).all() + tracking_map = {rule.path: rule.action for rule in tracking_rules} + + # Update is_ignored_by_policy in batches + all_files = db_session.query( + models.FilesystemState.id, models.FilesystemState.file_path + ).all() + + for file_id, file_path in all_files: + is_ignored_by_policy = get_ignored_by_policy(file_path, exclusion_spec) + is_ignored = get_ignored_status(file_path, tracking_map, exclusion_spec) + + db_session.execute( + text( + "UPDATE filesystem_state SET is_ignored_by_policy = :policy, is_ignored = :ignored WHERE id = :id" + ), + {"policy": is_ignored_by_policy, "ignored": is_ignored, "id": file_id}, + ) + + db_session.commit() + + def _validate_path_within_roots(path: str, roots: List[str]) -> bool: """Validates that a path does not contain traversal sequences and is within configured roots.""" if ".." in path: diff --git a/backend/app/api/system/filesystem.py b/backend/app/api/system/filesystem.py index a483107..a7c906f 100644 --- a/backend/app/api/system/filesystem.py +++ b/backend/app/api/system/filesystem.py @@ -76,7 +76,9 @@ def browse_system_path( entry_path = entry.path is_dir = entry.is_dir() is_ignored = get_ignored_status( - entry_path, tracking_map, exclusion_spec + entry_path + "/" if is_dir else entry_path, + tracking_map, + exclusion_spec, ) if is_dir: live_results.append( @@ -129,7 +131,7 @@ def browse_system_path( if child_path not in seen: seen.add(child_path) dir_ignored = get_ignored_status( - child_path, tracking_map, exclusion_spec + child_path + "/", tracking_map, exclusion_spec ) results.append( FileItemSchema( diff --git a/backend/app/api/system/settings.py b/backend/app/api/system/settings.py index 4229fb3..f6a5605 100644 --- a/backend/app/api/system/settings.py +++ b/backend/app/api/system/settings.py @@ -8,7 +8,7 @@ from fastapi.responses import StreamingResponse from pydantic import BaseModel from sqlalchemy.orm import Session -from app.api.common import FileItemSchema, SettingSchema +from app.api.common import FileItemSchema, SettingSchema, recompute_exclusion_policy from app.db import models from app.db.database import get_db @@ -57,6 +57,10 @@ def update_settings(setting_data: SettingSchema, db_session: Session = Depends(g scheduler_manager.reload() + # Recompute exclusion policy when global exclusions change + if setting_data.key == "global_exclusions": + recompute_exclusion_policy(db_session) + return {"message": "Setting committed."} diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 58ae5a7..928f728 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -21,7 +21,10 @@ class FilesystemState(Base): ) is_ignored: Mapped[bool] = mapped_column( Boolean, default=False - ) # True if matches exclusion + ) # Effective ignored state (manual OR policy, with manual override) + is_ignored_by_policy: Mapped[bool] = mapped_column( + Boolean, default=False + ) # True if excluded by global policy (excludes manual tracking rules) is_deleted: Mapped[bool] = mapped_column( Boolean, default=False ) # True if confirmed missing from disk diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts index 5997627..f10cf5a 100644 --- a/frontend/src/lib/api/index.ts +++ b/frontend/src/lib/api/index.ts @@ -1,4 +1,4 @@ // This file is auto-generated by @hey-api/openapi-ts export { addDirectoryToRestoreQueue, addFileToRestoreQueue, archiveBrowse, archiveMetadata, archiveSearch, archiveTree, batchAddToRestoreQueue, batchConfirmDiscrepancies, batchDeleteDiscrepancies, batchDismissDiscrepancies, batchTrack, browseDiscrepancies, browseRestoreQueue, cancelJob, checkHealth, clearRestoreQueue, confirmDiscrepancy, createMedia, deleteDiscrepancy, deleteMedia, detectMedia, discoverHardware, dismissDiscrepancy, downloadExclusionReport, exportDatabase, filesystemBrowse, filesystemSearch, filesystemTree, getAnalytics, getDashboardStats, getDiscrepancyTree, getJob, getJobCount, getJobLogs, getJobStats, getRestoreManifest, getRestoreQueue, getRestoreQueueTree, getScanStatus, getSettings, getTreemap, ignoreHardware, importDatabase, initializeMedia, listBackups, listDirectories, listDiscrepancies, listJobs, listMedia, listProviders, type Options, removeFromRestoreQueue, reorderMedia, resetTestEnvironment, retryJob, streamJobs, testExclusions, testNotification, triggerAutoBackup, triggerBackup, triggerIndexing, triggerRestore, triggerScan, undoDismissDiscrepancy, updateMedia, updateSettings } from './sdk.gen'; -export type { AddDirectoryToRestoreQueueData, AddDirectoryToRestoreQueueError, AddDirectoryToRestoreQueueErrors, AddDirectoryToRestoreQueueResponses, AddFileToRestoreQueueData, AddFileToRestoreQueueError, AddFileToRestoreQueueErrors, AddFileToRestoreQueueResponses, AppApiBackupsJobSchema, AppApiCommonJobSchema, ArchiveBrowseData, ArchiveBrowseError, ArchiveBrowseErrors, ArchiveBrowseResponses, ArchiveMetadataData, ArchiveMetadataError, ArchiveMetadataErrors, ArchiveMetadataResponse, ArchiveMetadataResponses, ArchiveSearchData, ArchiveSearchError, ArchiveSearchErrors, ArchiveSearchResponses, ArchiveTreeData, ArchiveTreeError, ArchiveTreeErrors, ArchiveTreeResponse, ArchiveTreeResponses, BatchAddToRestoreQueueData, BatchAddToRestoreQueueError, BatchAddToRestoreQueueErrors, BatchAddToRestoreQueueResponses, BatchCartRequest, BatchConfirmDiscrepanciesData, BatchConfirmDiscrepanciesError, BatchConfirmDiscrepanciesErrors, BatchConfirmDiscrepanciesResponses, BatchDeleteDiscrepanciesData, BatchDeleteDiscrepanciesError, BatchDeleteDiscrepanciesErrors, BatchDeleteDiscrepanciesResponses, BatchDiscrepancyAction, BatchDismissDiscrepanciesData, BatchDismissDiscrepanciesError, BatchDismissDiscrepanciesErrors, BatchDismissDiscrepanciesResponses, BatchTrackData, BatchTrackError, BatchTrackErrors, BatchTrackRequest, BatchTrackResponses, BrowseDiscrepanciesData, BrowseDiscrepanciesError, BrowseDiscrepanciesErrors, BrowseDiscrepanciesResponse, BrowseDiscrepanciesResponses, BrowseResponseSchema, BrowseRestoreQueueData, BrowseRestoreQueueError, BrowseRestoreQueueErrors, BrowseRestoreQueueResponse, BrowseRestoreQueueResponses, CancelJobData, CancelJobError, CancelJobErrors, CancelJobResponses, CartFileItemSchema, CartItemSchema, CartTreeNodeSchema, CheckHealthData, CheckHealthResponses, ClearRestoreQueueData, ClearRestoreQueueResponses, ClientOptions, ConfirmDiscrepancyData, ConfirmDiscrepancyError, ConfirmDiscrepancyErrors, ConfirmDiscrepancyResponses, CreateMediaData, CreateMediaError, CreateMediaErrors, CreateMediaResponse, CreateMediaResponses, DashboardStatsSchema, DeleteDiscrepancyData, DeleteDiscrepancyError, DeleteDiscrepancyErrors, DeleteDiscrepancyResponses, DeleteMediaData, DeleteMediaError, DeleteMediaErrors, DeleteMediaResponses, DetectMediaData, DetectMediaResponses, DirectoryCartRequest, DiscoverHardwareData, DiscoverHardwareResponses, DiscrepancySchema, DismissDiscrepancyData, DismissDiscrepancyError, DismissDiscrepancyErrors, DismissDiscrepancyResponses, DownloadExclusionReportData, DownloadExclusionReportError, DownloadExclusionReportErrors, DownloadExclusionReportResponses, ExportDatabaseData, ExportDatabaseResponses, FileItemSchema, FilesystemBrowseData, FilesystemBrowseError, FilesystemBrowseErrors, FilesystemBrowseResponse, FilesystemBrowseResponses, FilesystemSearchData, FilesystemSearchError, FilesystemSearchErrors, FilesystemSearchResponse, FilesystemSearchResponses, FilesystemTreeData, FilesystemTreeError, FilesystemTreeErrors, FilesystemTreeResponse, FilesystemTreeResponses, GetAnalyticsData, GetAnalyticsResponses, GetDashboardStatsData, GetDashboardStatsResponse, GetDashboardStatsResponses, GetDiscrepancyTreeData, GetDiscrepancyTreeError, GetDiscrepancyTreeErrors, GetDiscrepancyTreeResponse, GetDiscrepancyTreeResponses, GetJobCountData, GetJobCountResponses, GetJobData, GetJobError, GetJobErrors, GetJobLogsData, GetJobLogsError, GetJobLogsErrors, GetJobLogsResponse, GetJobLogsResponses, GetJobResponse, GetJobResponses, GetJobStatsData, GetJobStatsResponses, GetRestoreManifestData, GetRestoreManifestResponse, GetRestoreManifestResponses, GetRestoreQueueData, GetRestoreQueueResponse, GetRestoreQueueResponses, GetRestoreQueueTreeData, GetRestoreQueueTreeError, GetRestoreQueueTreeErrors, GetRestoreQueueTreeResponse, GetRestoreQueueTreeResponses, GetScanStatusData, GetScanStatusResponse, GetScanStatusResponses, GetSettingsData, GetSettingsResponse, GetSettingsResponses, GetTreemapData, GetTreemapResponses, HttpValidationError, IgnoreHardwareData, IgnoreHardwareError, IgnoreHardwareErrors, IgnoreHardwareRequest, IgnoreHardwareResponses, ImportDatabaseData, ImportDatabaseError, ImportDatabaseErrors, ImportDatabaseResponses, InitializeMediaData, InitializeMediaError, InitializeMediaErrors, InitializeMediaResponses, ItemMetadataSchema, JobLogSchema, ListBackupsData, ListBackupsResponse, ListBackupsResponses, ListDirectoriesData, ListDirectoriesError, ListDirectoriesErrors, ListDirectoriesResponses, ListDiscrepanciesData, ListDiscrepanciesResponse, ListDiscrepanciesResponses, ListJobsData, ListJobsError, ListJobsErrors, ListJobsResponse, ListJobsResponses, ListMediaData, ListMediaError, ListMediaErrors, ListMediaResponse, ListMediaResponses, ListProvidersData, ListProvidersResponse, ListProvidersResponses, ManifestMediaSchema, MediaCreateSchema, MediaSchema, MediaUpdateSchema, RemoveFromRestoreQueueData, RemoveFromRestoreQueueError, RemoveFromRestoreQueueErrors, RemoveFromRestoreQueueResponses, ReorderMediaData, ReorderMediaError, ReorderMediaErrors, ReorderMediaRequest, ReorderMediaResponses, ResetTestEnvironmentData, ResetTestEnvironmentResponses, RestoreManifestSchema, RestoreTriggerRequest, RetryJobData, RetryJobError, RetryJobErrors, RetryJobResponses, ScanStatusSchema, SettingSchema, StorageProviderSchema, StreamJobsData, StreamJobsResponses, TestExclusionsData, TestExclusionsError, TestExclusionsErrors, TestExclusionsRequest, TestExclusionsResponses, TestNotificationData, TestNotificationError, TestNotificationErrors, TestNotificationRequest, TestNotificationResponses, TreeNodeSchema, TriggerAutoBackupData, TriggerAutoBackupResponses, TriggerBackupData, TriggerBackupError, TriggerBackupErrors, TriggerBackupResponses, TriggerIndexingData, TriggerIndexingResponses, TriggerRestoreData, TriggerRestoreError, TriggerRestoreErrors, TriggerRestoreResponses, TriggerScanData, TriggerScanResponses, UndoDismissDiscrepancyData, UndoDismissDiscrepancyError, UndoDismissDiscrepancyErrors, UndoDismissDiscrepancyResponses, UpdateMediaData, UpdateMediaError, UpdateMediaErrors, UpdateMediaResponse, UpdateMediaResponses, UpdateSettingsData, UpdateSettingsError, UpdateSettingsErrors, UpdateSettingsResponses, ValidationError } from './types.gen'; +export type { AddDirectoryToRestoreQueueData, AddDirectoryToRestoreQueueError, AddDirectoryToRestoreQueueErrors, AddDirectoryToRestoreQueueResponses, AddFileToRestoreQueueData, AddFileToRestoreQueueError, AddFileToRestoreQueueErrors, AddFileToRestoreQueueResponses, AppApiBackupsJobSchema, AppApiCommonJobSchema, ArchiveBrowseData, ArchiveBrowseError, ArchiveBrowseErrors, ArchiveBrowseResponses, ArchiveMetadataData, ArchiveMetadataError, ArchiveMetadataErrors, ArchiveMetadataResponse, ArchiveMetadataResponses, ArchiveSearchData, ArchiveSearchError, ArchiveSearchErrors, ArchiveSearchResponses, ArchiveTreeData, ArchiveTreeError, ArchiveTreeErrors, ArchiveTreeResponse, ArchiveTreeResponses, BatchAddToRestoreQueueData, BatchAddToRestoreQueueError, BatchAddToRestoreQueueErrors, BatchAddToRestoreQueueResponses, BatchCartRequest, BatchConfirmDiscrepanciesData, BatchConfirmDiscrepanciesError, BatchConfirmDiscrepanciesErrors, BatchConfirmDiscrepanciesResponses, BatchDeleteDiscrepanciesData, BatchDeleteDiscrepanciesError, BatchDeleteDiscrepanciesErrors, BatchDeleteDiscrepanciesResponses, BatchDiscrepancyAction, BatchDismissDiscrepanciesData, BatchDismissDiscrepanciesError, BatchDismissDiscrepanciesErrors, BatchDismissDiscrepanciesResponses, BatchTrackData, BatchTrackError, BatchTrackErrors, BatchTrackRequest, BatchTrackResponses, BrowseDiscrepanciesData, BrowseDiscrepanciesError, BrowseDiscrepanciesErrors, BrowseDiscrepanciesResponse, BrowseDiscrepanciesResponses, BrowseResponseSchema, BrowseRestoreQueueData, BrowseRestoreQueueError, BrowseRestoreQueueErrors, BrowseRestoreQueueResponse, BrowseRestoreQueueResponses, CancelJobData, CancelJobError, CancelJobErrors, CancelJobResponses, CartFileItemSchema, CartItemSchema, CartTreeNodeSchema, CheckHealthData, CheckHealthResponses, ClearRestoreQueueData, ClearRestoreQueueResponses, ClientOptions, ConfirmDiscrepancyData, ConfirmDiscrepancyError, ConfirmDiscrepancyErrors, ConfirmDiscrepancyResponses, CreateMediaData, CreateMediaError, CreateMediaErrors, CreateMediaResponse, CreateMediaResponses, DashboardStatsSchema, DeleteDiscrepancyData, DeleteDiscrepancyError, DeleteDiscrepancyErrors, DeleteDiscrepancyResponses, DeleteMediaData, DeleteMediaError, DeleteMediaErrors, DeleteMediaResponses, DetectMediaData, DetectMediaResponses, DirectoryCartRequest, DiscoverHardwareData, DiscoverHardwareResponses, DiscrepancySchema, DismissDiscrepancyData, DismissDiscrepancyError, DismissDiscrepancyErrors, DismissDiscrepancyResponses, DownloadExclusionReportData, DownloadExclusionReportError, DownloadExclusionReportErrors, DownloadExclusionReportResponses, ExportDatabaseData, ExportDatabaseResponses, FileItemSchema, FilesystemBrowseData, FilesystemBrowseError, FilesystemBrowseErrors, FilesystemBrowseResponse, FilesystemBrowseResponses, FilesystemSearchData, FilesystemSearchError, FilesystemSearchErrors, FilesystemSearchResponse, FilesystemSearchResponses, FilesystemTreeData, FilesystemTreeError, FilesystemTreeErrors, FilesystemTreeResponse, FilesystemTreeResponses, GetAnalyticsData, GetAnalyticsResponses, GetDashboardStatsData, GetDashboardStatsResponse, GetDashboardStatsResponses, GetDiscrepancyTreeData, GetDiscrepancyTreeError, GetDiscrepancyTreeErrors, GetDiscrepancyTreeResponse, GetDiscrepancyTreeResponses, GetJobCountData, GetJobCountResponses, GetJobData, GetJobError, GetJobErrors, GetJobLogsData, GetJobLogsError, GetJobLogsErrors, GetJobLogsResponse, GetJobLogsResponses, GetJobResponse, GetJobResponses, GetJobStatsData, GetJobStatsResponses, GetRestoreManifestData, GetRestoreManifestResponse, GetRestoreManifestResponses, GetRestoreQueueData, GetRestoreQueueResponse, GetRestoreQueueResponses, GetRestoreQueueTreeData, GetRestoreQueueTreeError, GetRestoreQueueTreeErrors, GetRestoreQueueTreeResponse, GetRestoreQueueTreeResponses, GetScanStatusData, GetScanStatusResponse, GetScanStatusResponses, GetSettingsData, GetSettingsResponse, GetSettingsResponses, GetTreemapData, GetTreemapResponses, HttpValidationError, IgnoreHardwareData, IgnoreHardwareError, IgnoreHardwareErrors, IgnoreHardwareRequest, IgnoreHardwareResponses, ImportDatabaseData, ImportDatabaseError, ImportDatabaseErrors, ImportDatabaseResponses, InitializeMediaData, InitializeMediaError, InitializeMediaErrors, InitializeMediaResponses, ItemMetadataSchema, JobLogSchema, ListBackupsData, ListBackupsResponse, ListBackupsResponses, ListDirectoriesData, ListDirectoriesError, ListDirectoriesErrors, ListDirectoriesResponses, ListDiscrepanciesData, ListDiscrepanciesResponse, ListDiscrepanciesResponses, ListJobsData, ListJobsError, ListJobsErrors, ListJobsResponse, ListJobsResponses, ListMediaData, ListMediaError, ListMediaErrors, ListMediaResponse, ListMediaResponses, ListProvidersData, ListProvidersResponse, ListProvidersResponses, ManifestMediaSchema, MediaCreateSchema, MediaSchema, MediaUpdateSchema, RemoveFromRestoreQueueData, RemoveFromRestoreQueueError, RemoveFromRestoreQueueErrors, RemoveFromRestoreQueueResponses, ReorderMediaData, ReorderMediaError, ReorderMediaErrors, ReorderMediaRequest, ReorderMediaResponses, ResetTestEnvironmentData, ResetTestEnvironmentResponses, RestoreManifestSchema, RestoreTriggerRequest, RetryJobData, RetryJobError, RetryJobErrors, RetryJobResponses, ScanStatusSchema, SettingSchema, StorageProviderSchema, StreamJobsData, StreamJobsResponses, TestExclusionsData, TestExclusionsError, TestExclusionsErrors, TestExclusionsRequest, TestExclusionsResponse, TestExclusionsResponse2, TestExclusionsResponses, TestNotificationData, TestNotificationError, TestNotificationErrors, TestNotificationRequest, TestNotificationResponses, TreeNodeSchema, TriggerAutoBackupData, TriggerAutoBackupResponses, TriggerBackupData, TriggerBackupError, TriggerBackupErrors, TriggerBackupResponses, TriggerIndexingData, TriggerIndexingResponses, TriggerRestoreData, TriggerRestoreError, TriggerRestoreErrors, TriggerRestoreResponses, TriggerScanData, TriggerScanResponses, UndoDismissDiscrepancyData, UndoDismissDiscrepancyError, UndoDismissDiscrepancyErrors, UndoDismissDiscrepancyResponses, UpdateMediaData, UpdateMediaError, UpdateMediaErrors, UpdateMediaResponse, UpdateMediaResponses, UpdateSettingsData, UpdateSettingsError, UpdateSettingsErrors, UpdateSettingsResponses, ValidationError } from './types.gen'; diff --git a/frontend/src/lib/api/types.gen.ts b/frontend/src/lib/api/types.gen.ts index f0a5e67..acd9d88 100644 --- a/frontend/src/lib/api/types.gen.ts +++ b/frontend/src/lib/api/types.gen.ts @@ -676,6 +676,32 @@ export type TestExclusionsRequest = { limit?: number; }; +/** + * TestExclusionsResponse + */ +export type TestExclusionsResponse = { + /** + * Total Files + */ + total_files: number; + /** + * Total Size + */ + total_size: number; + /** + * Matched Count + */ + matched_count: number; + /** + * Matched Size + */ + matched_size: number; + /** + * Sample + */ + sample: Array; +}; + /** * TestNotificationRequest */ @@ -1185,7 +1211,7 @@ export type GetSettingsResponses = { * Successful Response */ 200: { - [key: string]: unknown; + [key: string]: string; }; }; @@ -1234,9 +1260,11 @@ export type TestExclusionsResponses = { /** * Successful Response */ - 200: unknown; + 200: TestExclusionsResponse; }; +export type TestExclusionsResponse2 = TestExclusionsResponses[keyof TestExclusionsResponses]; + export type DownloadExclusionReportData = { body: TestExclusionsRequest; path?: never; diff --git a/frontend/tests/exclusion-policy.test.ts b/frontend/tests/exclusion-policy.test.ts new file mode 100644 index 0000000..6a54582 --- /dev/null +++ b/frontend/tests/exclusion-policy.test.ts @@ -0,0 +1,259 @@ +import { test, expect } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; +import { API_URL, SOURCE_ROOT, setupRequestContext, triggerScanAndWait } from './helpers'; + +test.describe('Exclusion Policy', () => { + test.beforeEach(async () => { + fs.mkdirSync(SOURCE_ROOT, { recursive: true }); + fs.mkdirSync(path.join(SOURCE_ROOT, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(SOURCE_ROOT, 'temp'), { recursive: true }); + fs.writeFileSync(path.join(SOURCE_ROOT, 'readme.txt'), 'hello'); + fs.writeFileSync(path.join(SOURCE_ROOT, 'notes.txt'), 'world'); + fs.writeFileSync(path.join(SOURCE_ROOT, 'data.tmp'), 'temp1'); + fs.writeFileSync(path.join(SOURCE_ROOT, 'cache.tmp'), 'temp2'); + fs.writeFileSync(path.join(SOURCE_ROOT, 'docs', 'guide.txt'), 'guide'); + fs.writeFileSync(path.join(SOURCE_ROOT, 'docs', 'draft.tmp'), 'draft'); + fs.writeFileSync(path.join(SOURCE_ROOT, 'temp', 'scratch.tmp'), 'scratch'); + }); + + test('global exclusions mark matching files as ignored', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'global_exclusions', value: '*.tmp\n' } + }); + + await triggerScanAndWait(requestContext); + + const browseResp = await requestContext.get( + `${API_URL}/system/browse?path=${SOURCE_ROOT}` + ); + const browseData = await browseResp.json(); + const files = (browseData as any).files; + + const tmpFiles = (files as Array).filter((f: any) => f.name.endsWith('.tmp')); + const txtFiles = (files as Array).filter((f: any) => f.name.endsWith('.txt')); + + expect(tmpFiles.length).toBeGreaterThan(0); + tmpFiles.forEach((f: any) => { + expect(f.ignored, `expected ${f.name} to be ignored`).toBe(true); + }); + + txtFiles.forEach((f: any) => { + expect(f.ignored, `expected ${f.name} to NOT be ignored`).toBe(false); + }); + + await requestContext.dispose(); + }); + + test('manual include overrides global exclusion', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'global_exclusions', value: '*.tmp\n' } + }); + + // Include one specific .tmp file despite the global exclusion + await requestContext.post(`${API_URL}/system/track/batch`, { + data: { + tracks: [path.join(SOURCE_ROOT, 'data.tmp')], + untracks: [] + } + }); + + await triggerScanAndWait(requestContext); + + const browseResp = await requestContext.get( + `${API_URL}/system/browse?path=${SOURCE_ROOT}` + ); + const browseData = await browseResp.json(); + const files = (browseData as any).files; + + const dataTmp = (files as Array).find((f: any) => f.name === 'data.tmp'); + const cacheTmp = (files as Array).find((f: any) => f.name === 'cache.tmp'); + + expect(dataTmp).toBeDefined(); + expect(dataTmp.ignored).toBe(false); + + expect(cacheTmp).toBeDefined(); + expect(cacheTmp.ignored).toBe(true); + + await requestContext.dispose(); + }); + + test('updating global exclusions recomputes existing indexed files', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + // No exclusions initially + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'global_exclusions', value: '' } + }); + + await triggerScanAndWait(requestContext); + + // Verify nothing is ignored before exclusions are set + const browseBefore = await requestContext.get( + `${API_URL}/system/browse?path=${SOURCE_ROOT}` + ); + const beforeData = await browseBefore.json(); + const beforeFiles = (beforeData as any).files as Array; + beforeFiles.forEach((f: any) => { + expect(f.ignored, `expected ${f.name} to NOT be ignored before policy`).toBe(false); + }); + + // Now apply global exclusions — should recompute without requiring a new scan + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'global_exclusions', value: '*.tmp\n' } + }); + + const browseAfter = await requestContext.get( + `${API_URL}/system/browse?path=${SOURCE_ROOT}` + ); + const afterData = await browseAfter.json(); + const afterFiles = (afterData as any).files as Array; + + const tmpAfter = afterFiles.filter((f: any) => f.name.endsWith('.tmp')); + const txtAfter = afterFiles.filter((f: any) => f.name.endsWith('.txt')); + + tmpAfter.forEach((f: any) => { + expect(f.ignored, `expected ${f.name} to be ignored after policy update`).toBe(true); + }); + txtAfter.forEach((f: any) => { + expect(f.ignored, `expected ${f.name} to NOT be ignored after policy update`).toBe(false); + }); + + await requestContext.dispose(); + }); + + test('exclusion preview returns correct counts and sample', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + + await triggerScanAndWait(requestContext); + + const previewResp = await requestContext.post(`${API_URL}/system/settings/test-exclusions`, { + data: { patterns: '*.tmp', limit: 10 } + }); + expect(previewResp.ok()).toBe(true); + const preview = await previewResp.json(); + + expect(preview.total_files).toBeGreaterThan(0); + expect(preview.matched_count).toBeGreaterThan(0); + expect(preview.matched_size).toBeGreaterThanOrEqual(0); + expect(Array.isArray(preview.sample)).toBe(true); + expect(preview.sample.length).toBeGreaterThan(0); + expect(preview.sample.length).toBeLessThanOrEqual(10); + + preview.sample.forEach((s: any) => { + expect(s.name.endsWith('.tmp')).toBe(true); + expect(s.path).toBeDefined(); + expect(s.size).toBeDefined(); + }); + + await requestContext.dispose(); + }); + + test('exclusion preview with no patterns returns empty result', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + + await triggerScanAndWait(requestContext); + + const previewResp = await requestContext.post(`${API_URL}/system/settings/test-exclusions`, { + data: { patterns: '', limit: 10 } + }); + expect(previewResp.ok()).toBe(true); + const preview = await previewResp.json(); + + expect(preview.total_files).toBe(0); + expect(preview.matched_count).toBe(0); + expect(preview.matched_size).toBe(0); + expect(preview.sample).toEqual([]); + + await requestContext.dispose(); + }); + + test('exclusion CSV download contains matched files', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + + await triggerScanAndWait(requestContext); + + const downloadResp = await requestContext.post( + `${API_URL}/system/settings/test-exclusions/download`, + { data: { patterns: '*.tmp' } } + ); + expect(downloadResp.ok()).toBe(true); + + const contentType = downloadResp.headers()['content-type']; + expect(contentType).toContain('text/csv'); + + const body = await downloadResp.text(); + expect(body).toContain('path,size,mtime,sha256_hash'); + expect(body).toContain('.tmp'); + + const lines = body.trim().split('\n'); + expect(lines.length).toBeGreaterThan(1); // header + at least one row + + await requestContext.dispose(); + }); + + test('directory-level global exclusion ignores nested files', async ({ page }) => { + const requestContext = await setupRequestContext(); + + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'source_roots', value: JSON.stringify([SOURCE_ROOT]) } + }); + await requestContext.post(`${API_URL}/system/settings`, { + data: { key: 'global_exclusions', value: 'temp/\n' } + }); + + await triggerScanAndWait(requestContext); + + const browseRoot = await requestContext.get( + `${API_URL}/system/browse?path=${SOURCE_ROOT}` + ); + const rootData = await browseRoot.json(); + const rootFiles = (rootData as any).files as Array; + + const tempDir = rootFiles.find((f: any) => f.name === 'temp'); + expect(tempDir).toBeDefined(); + expect(tempDir.ignored).toBe(true); + + // Files inside temp should also be ignored + const browseTemp = await requestContext.get( + `${API_URL}/system/browse?path=${path.join(SOURCE_ROOT, 'temp')}` + ); + const tempData = await browseTemp.json(); + const tempFiles = (tempData as any).files as Array; + tempFiles.forEach((f: any) => { + expect(f.ignored, `expected ${f.name} inside temp/ to be ignored`).toBe(true); + }); + + // Files outside temp should NOT be ignored + const readme = rootFiles.find((f: any) => f.name === 'readme.txt'); + expect(readme).toBeDefined(); + expect(readme.ignored).toBe(false); + + await requestContext.dispose(); + }); +}); diff --git a/justfile b/justfile index 30bb0ee..7a3d647 100644 --- a/justfile +++ b/justfile @@ -64,12 +64,25 @@ db-migrate message: # --- Code Generation --- +# Export the OpenAPI spec JSON without regenerating the TypeScript client +export-openapi: + @echo "Exporting OpenAPI spec..." + @cd backend && uv run python scripts/generate_openapi.py /tmp/tapehoard_openapi.json + # Generate the TypeScript API client from the FastAPI OpenAPI spec generate-client: db-upgrade @echo "Generating TypeScript API client..." @cd backend && uv run python scripts/generate_openapi.py /tmp/tapehoard_openapi.json @cd frontend && npx @hey-api/openapi-ts -i /tmp/tapehoard_openapi.json -o src/lib/api -c @hey-api/client-fetch +# Full regeneration workflow after schema changes: migrate, upgrade, generate client, lint +regenerate message: db-upgrade + @echo "Running full regeneration workflow..." + cd backend && uv run alembic revision --autogenerate -m "{{message}}" + cd backend && uv run alembic upgrade head + @just generate-client + @just lint + # --- Docker --- # Build the production Docker image @@ -97,3 +110,10 @@ playwright: playwright-ui: @echo "Starting playweight UI..." cd frontend && npx playwright test --ui + +# Clean test artifacts and kill stale test servers +clean-test: + @echo "Cleaning test artifacts..." + pkill -f "start_test_server" 2>/dev/null || true + rm -f backend/e2e_test.db backend/e2e_test.db-* + rm -rf frontend/test-results/