From d7eb1ece04902de598bf0813bc2ef7e46d3e248f Mon Sep 17 00:00:00 2001 From: Phoebe Goldman Date: Tue, 30 Jun 2026 14:30:04 -0400 Subject: [PATCH] moduledef extensions for manual migrations --- crates/lib/src/db/raw_def/v10.rs | 56 +++++++++++++++++++++ crates/schema/src/def.rs | 44 +++++++++++++++-- crates/schema/src/def/validate/v10.rs | 71 +++++++++++++++++++++++++-- crates/schema/src/def/validate/v9.rs | 1 + crates/schema/src/error.rs | 6 +++ 5 files changed, 170 insertions(+), 8 deletions(-) diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 1ab04d7a6b..9982031c77 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -8,6 +8,7 @@ use crate::db::raw_def::v9::{Lifecycle, RawIndexAlgorithm, TableAccess, TableType}; use core::fmt; use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_sats::hash::Hash; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::typespace::TypespaceBuilder; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SpacetimeType, Typespace}; @@ -98,6 +99,9 @@ pub enum RawModuleDefV10Section { /// Primary key metadata for views. ViewPrimaryKeys(Vec), + + /// Guest-defined logic for manual migrations. + ManualMigrationFunctions(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -557,6 +561,19 @@ pub struct RawViewPrimaryKeyDefV10 { pub columns: Vec, } +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawManualMigrationFunctionDefV10 { + pub function_source_name: RawIdentifier, + /// The Blake3 hash of the raw module def of the module version from which this function migrates. + /// + /// Stored here as a [`Hash`] rather than a [`blake3::Hash`] as the former type implements [`SpacetimeType`] and [`Ord`]. + // TODO(manual-migrations): determine precise semantics for computing this. + // Is it always the hash of the BSATN-ified `RawModuleDefV10`, even if the previous module exports a `RawModuleDefV9` or earlier? + pub previous_raw_module_def_bsatn_blake3_hash: Hash, +} + impl RawModuleDefV10 { /// Get the types section, if present. pub fn types(&self) -> Option<&Vec> { @@ -678,6 +695,13 @@ impl RawModuleDefV10 { _ => None, }) } + + pub fn manual_migration_functions(&self) -> Option<&Vec> { + self.sections.iter().find_map(|s| match s { + RawModuleDefV10Section::ManualMigrationFunctions(migrations) => Some(migrations), + _ => None, + }) + } } /// A builder for a [`RawModuleDefV10`]. @@ -935,6 +959,26 @@ impl RawModuleDefV10Builder { } } + /// Get mutable access to the manual migration functions section, creating it if missing. + fn manual_migration_functions_mut(&mut self) -> &mut Vec { + let idx = self + .module + .sections + .iter() + .position(|s| matches!(s, RawModuleDefV10Section::ManualMigrationFunctions(_))) + .unwrap_or_else(|| { + self.module + .sections + .push(RawModuleDefV10Section::ManualMigrationFunctions(Vec::new())); + self.module.sections.len() - 1 + }); + + match &mut self.module.sections[idx] { + RawModuleDefV10Section::ManualMigrationFunctions(migrations) => migrations, + _ => unreachable!("Just ensured ManualMigrationFunctions section exists"), + } + } + /// Create a table builder. /// /// Does not validate that the product_type_ref is valid; this is left to the module validation code. @@ -1217,6 +1261,18 @@ impl RawModuleDefV10Builder { self.explicit_names_mut().merge(names); } + pub fn add_manual_migration_function( + &mut self, + source_name: impl Into, + previous_raw_module_def_bsatn_blake3_hash: Hash, + ) { + self.manual_migration_functions_mut() + .push(RawManualMigrationFunctionDefV10 { + function_source_name: source_name.into(), + previous_raw_module_def_bsatn_blake3_hash, + }); + } + /// Set the case conversion policy for this module. /// /// By default, SpacetimeDB applies `SnakeCase` conversion to table names, diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 9ad07dce72..4c9a01f5b4 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -33,9 +33,9 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap}; use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawConstraintDefV10, RawHttpHandlerDefV10, RawHttpRouteDefV10, RawIndexDefV10, - RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, RawProcedureDefV10, RawReducerDefV10, - RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, - RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, + RawLifeCycleReducerDefV10, RawManualMigrationFunctionDefV10, RawModuleDefV10, RawModuleDefV10Section, + RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, + RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -164,6 +164,13 @@ pub struct ModuleDef { /// was authored under. #[allow(unused)] raw_module_def_version: RawModuleDefVersion, + + /// Manual migration functions defined by the module, + /// keyed on the Blake3 hash of the raw module def of the previous module version. + /// + /// Uses [`IndexMap`] to preserve order so that `__call_manual_migration_function__` + /// receives stable integer IDs. + manual_migration_functions: IndexMap, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -507,6 +514,7 @@ impl From for RawModuleDefV9 { http_handlers: _, http_routes: _, raw_module_def_version: _, + manual_migration_functions: _, } = val; // Extract column defaults from tables before consuming tables @@ -565,6 +573,7 @@ impl From for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, + manual_migration_functions: _, } = val; let mut sections = Vec::new(); @@ -1911,6 +1920,25 @@ impl From for RawMiscModuleExportV9 { } } +#[derive(Debug, Clone, Eq, PartialEq)] +#[non_exhaustive] +pub struct ManualMigrationFunctionDef { + pub function_source_name: RawIdentifier, + /// The Blake3 hash of the raw module def of the module version from which this function migrates. + // TODO(manual-migrations): determine precise semantics for computing this. + // Is it always the hash of the BSATN-ified `RawModuleDefV10`, even if the previous module exports a `RawModuleDefV9` or earlier? + pub previous_raw_module_def_bsatn_blake3_hash: spacetimedb_lib::Hash, +} + +impl From for RawManualMigrationFunctionDefV10 { + fn from(def: ManualMigrationFunctionDef) -> Self { + RawManualMigrationFunctionDefV10 { + function_source_name: def.function_source_name, + previous_raw_module_def_bsatn_blake3_hash: def.previous_raw_module_def_bsatn_blake3_hash, + } + } +} + impl ModuleDefLookup for TableDef { type Key<'a> = &'a Identifier; @@ -2090,6 +2118,16 @@ impl ModuleDefLookup for ViewDef { } } +impl ModuleDefLookup for ManualMigrationFunctionDef { + type Key<'a> = &'a spacetimedb_lib::Hash; + fn key(&self) -> Self::Key<'_> { + &self.previous_raw_module_def_bsatn_blake3_hash + } + fn lookup<'def>(module_def: &'def ModuleDef, key: Self::Key<'_>) -> Option<&'def Self> { + module_def.manual_migration_functions.get(key) + } +} + fn to_raw(data: HashMap) -> Vec where Def: ModuleDefLookup + Into, diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index aa398743f9..1a03149ae7 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -248,6 +248,10 @@ pub fn validate(def: RawModuleDefV10) -> Result { validate_http_routes_are_unique(&routes)?; Ok((handlers, routes)) }); + + let migration_functions = + validate_migration_function_hashes_are_unique(def.manual_migration_functions().unwrap_or(&Vec::new())); + // Combine all validation results let tables_types_reducers_procedures_views = ( tables, @@ -258,10 +262,21 @@ pub fn validate(def: RawModuleDefV10) -> Result { schedules, lifecycle_validations, http_handlers_and_routes, + migration_functions, ) .combine_errors() .and_then( - |(mut tables, types, reducers, procedures, views, schedules, lifecycles, http_handlers_and_routes)| { + |( + mut tables, + types, + reducers, + procedures, + views, + schedules, + lifecycles, + http_handlers_and_routes, + migration_functions, + )| { let (mut reducers, mut procedures, mut views) = check_function_names_are_unique(reducers, procedures, views)?; // Attach lifecycles to their respective reducers @@ -275,7 +290,15 @@ pub fn validate(def: RawModuleDefV10) -> Result { attach_view_primary_keys(&mut views, view_primary_keys)?; assign_query_view_primary_keys(&tables, &mut views); - Ok((tables, types, reducers, procedures, views, http_handlers_and_routes)) + Ok(( + tables, + types, + reducers, + procedures, + views, + http_handlers_and_routes, + migration_functions, + )) }, ); let CoreValidator { @@ -292,11 +315,20 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let (tables, types, reducers, procedures, views, http_handlers, http_routes) = + let (tables, types, reducers, procedures, views, http_handlers, http_routes, migration_functions) = tables_types_reducers_procedures_views .map( - |(tables, types, reducers, procedures, views, (http_handlers, http_routes))| { - (tables, types, reducers, procedures, views, http_handlers, http_routes) + |(tables, types, reducers, procedures, views, (http_handlers, http_routes), migration_functions)| { + ( + tables, + types, + reducers, + procedures, + views, + http_handlers, + http_routes, + migration_functions, + ) }, ) .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; @@ -318,6 +350,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_handlers, http_routes, raw_module_def_version: RawModuleDefVersion::V10, + manual_migration_functions: migration_functions, }) } @@ -412,6 +445,34 @@ fn check_http_handler_names_are_unique( ErrorStream::add_extra_errors(Ok(handlers_map), errors) } +fn validate_migration_function_hashes_are_unique( + migrations: &[RawManualMigrationFunctionDefV10], +) -> Result> { + let mut errors = vec![]; + let mut migrations_map: IndexMap = + IndexMap::with_capacity(migrations.len()); + + for migration in migrations { + if let Some(other_migration) = migrations_map.get(&migration.previous_raw_module_def_bsatn_blake3_hash) { + errors.push(ValidationError::DuplicateManualMigrationPreviousHash { + previous_raw_module_def_bsatn_blake3_hash: migration.previous_raw_module_def_bsatn_blake3_hash, + function_source_name_a: other_migration.function_source_name.clone(), + function_source_name_b: migration.function_source_name.clone(), + }); + } else { + migrations_map.insert( + migration.previous_raw_module_def_bsatn_blake3_hash, + ManualMigrationFunctionDef { + function_source_name: migration.function_source_name.clone(), + previous_raw_module_def_bsatn_blake3_hash: migration.previous_raw_module_def_bsatn_blake3_hash, + }, + ); + } + } + + ErrorStream::add_extra_errors(Ok(migrations_map), errors) +} + struct ModuleValidatorV10<'a> { core: CoreValidator<'a>, } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 618f8e3c9c..388197ab90 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -168,6 +168,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_handlers: IndexMap::new(), http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, + manual_migration_functions: IndexMap::new(), }) } diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 0bab5445d9..95ac6c26ab 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -176,6 +176,12 @@ pub enum ValidationError { ok_type: PrettyAlgebraicType, err_type: PrettyAlgebraicType, }, + #[error("multiple manual migration functions defined to update from module def hash {previous_raw_module_def_bsatn_blake3_hash}, with identifiers {function_source_name_a} and {function_source_name_b}")] + DuplicateManualMigrationPreviousHash { + previous_raw_module_def_bsatn_blake3_hash: spacetimedb_lib::Hash, + function_source_name_a: RawIdentifier, + function_source_name_b: RawIdentifier, + }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`.