moduledef extensions for manual migrations

crates/lib/src/db/raw_def/v10.rs

@@ -8,6 +8,7 @@
 use crate::db::raw_def::v9::{Lifecycle, RawIndexAlgorithm, TableAccess, TableType};
 use core::fmt;
 use spacetimedb_primitives::{ColId, ColList};
+use spacetimedb_sats::hash::Hash;
 use spacetimedb_sats::raw_identifier::RawIdentifier;
 use spacetimedb_sats::typespace::TypespaceBuilder;
 use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SpacetimeType, Typespace};
@@ -98,6 +99,9 @@ pub enum RawModuleDefV10Section {
 
     /// Primary key metadata for views.
     ViewPrimaryKeys(Vec<RawViewPrimaryKeyDefV10>),
+
+    /// Guest-defined logic for manual migrations.
+    ManualMigrationFunctions(Vec<RawManualMigrationFunctionDefV10>),
 }
 
 #[derive(Debug, Clone, SpacetimeType)]
@@ -557,6 +561,19 @@ pub struct RawViewPrimaryKeyDefV10 {
     pub columns: Vec<RawIdentifier>,
 }
 
+#[derive(Debug, Clone, SpacetimeType)]
+#[sats(crate = crate)]
+#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))]
+pub struct RawManualMigrationFunctionDefV10 {
+    pub function_source_name: RawIdentifier,
+    /// The Blake3 hash of the raw module def of the module version from which this function migrates.
+    ///
+    /// Stored here as a [`Hash`] rather than a [`blake3::Hash`] as the former type implements [`SpacetimeType`] and [`Ord`].
+    // TODO(manual-migrations): determine precise semantics for computing this.
+    // Is it always the hash of the BSATN-ified `RawModuleDefV10`, even if the previous module exports a `RawModuleDefV9` or earlier?
+    pub previous_raw_module_def_bsatn_blake3_hash: Hash,
+}
+
 impl RawModuleDefV10 {
     /// Get the types section, if present.
     pub fn types(&self) -> Option<&Vec<RawTypeDefV10>> {
@@ -678,6 +695,13 @@ impl RawModuleDefV10 {
             _ => None,
         })
     }
+
+    pub fn manual_migration_functions(&self) -> Option<&Vec<RawManualMigrationFunctionDefV10>> {
+        self.sections.iter().find_map(|s| match s {
+            RawModuleDefV10Section::ManualMigrationFunctions(migrations) => Some(migrations),
+            _ => None,
+        })
+    }
 }
 
 /// A builder for a [`RawModuleDefV10`].
@@ -935,6 +959,26 @@ impl RawModuleDefV10Builder {
         }
     }
 
+    /// Get mutable access to the manual migration functions section, creating it if missing.
+    fn manual_migration_functions_mut(&mut self) -> &mut Vec<RawManualMigrationFunctionDefV10> {
+        let idx = self
+            .module
+            .sections
+            .iter()
+            .position(|s| matches!(s, RawModuleDefV10Section::ManualMigrationFunctions(_)))
+            .unwrap_or_else(|| {
+                self.module
+                    .sections
+                    .push(RawModuleDefV10Section::ManualMigrationFunctions(Vec::new()));
+                self.module.sections.len() - 1
+            });
+
+        match &mut self.module.sections[idx] {
+            RawModuleDefV10Section::ManualMigrationFunctions(migrations) => migrations,
+            _ => unreachable!("Just ensured ManualMigrationFunctions section exists"),
+        }
+    }
+
     /// Create a table builder.
     ///
     /// Does not validate that the product_type_ref is valid; this is left to the module validation code.
@@ -1217,6 +1261,18 @@ impl RawModuleDefV10Builder {
         self.explicit_names_mut().merge(names);
     }
 
+    pub fn add_manual_migration_function(
+        &mut self,
+        source_name: impl Into<RawIdentifier>,
+        previous_raw_module_def_bsatn_blake3_hash: Hash,
+    ) {
+        self.manual_migration_functions_mut()
+            .push(RawManualMigrationFunctionDefV10 {
+                function_source_name: source_name.into(),
+                previous_raw_module_def_bsatn_blake3_hash,
+            });
+    }
+
     /// Set the case conversion policy for this module.
     ///
     /// By default, SpacetimeDB applies `SnakeCase` conversion to table names,

crates/schema/src/def.rs

@@ -33,9 +33,9 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap};
 use spacetimedb_lib::db::raw_def;
 use spacetimedb_lib::db::raw_def::v10::{
     ExplicitNames, MethodOrAny, RawConstraintDefV10, RawHttpHandlerDefV10, RawHttpRouteDefV10, RawIndexDefV10,
-    RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, RawProcedureDefV10, RawReducerDefV10,
-    RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10,
-    RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10,
+    RawLifeCycleReducerDefV10, RawManualMigrationFunctionDefV10, RawModuleDefV10, RawModuleDefV10Section,
+    RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10,
+    RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10,
 };
 use spacetimedb_lib::db::raw_def::v9::{
     Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9,
@@ -164,6 +164,13 @@ pub struct ModuleDef {
     /// was authored under.
     #[allow(unused)]
     raw_module_def_version: RawModuleDefVersion,
+
+    /// Manual migration functions defined by the module,
+    /// keyed on the Blake3 hash of the raw module def of the previous module version.
+    ///
+    /// Uses [`IndexMap`] to preserve order so that `__call_manual_migration_function__`
+    /// receives stable integer IDs.
+    manual_migration_functions: IndexMap<spacetimedb_lib::Hash, ManualMigrationFunctionDef>,
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -507,6 +514,7 @@ impl From<ModuleDef> for RawModuleDefV9 {
             http_handlers: _,
             http_routes: _,
             raw_module_def_version: _,
+            manual_migration_functions: _,
         } = val;
 
         // Extract column defaults from tables before consuming tables
@@ -565,6 +573,7 @@ impl From<ModuleDef> for RawModuleDefV10 {
             http_handlers,
             http_routes,
             raw_module_def_version: _,
+            manual_migration_functions: _,
         } = val;
 
         let mut sections = Vec::new();
@@ -1911,6 +1920,25 @@ impl From<ProcedureDef> for RawMiscModuleExportV9 {
     }
 }
 
+#[derive(Debug, Clone, Eq, PartialEq)]
+#[non_exhaustive]
+pub struct ManualMigrationFunctionDef {
+    pub function_source_name: RawIdentifier,
+    /// The Blake3 hash of the raw module def of the module version from which this function migrates.
+    // TODO(manual-migrations): determine precise semantics for computing this.
+    // Is it always the hash of the BSATN-ified `RawModuleDefV10`, even if the previous module exports a `RawModuleDefV9` or earlier?
+    pub previous_raw_module_def_bsatn_blake3_hash: spacetimedb_lib::Hash,
+}
+
+impl From<ManualMigrationFunctionDef> for RawManualMigrationFunctionDefV10 {
+    fn from(def: ManualMigrationFunctionDef) -> Self {
+        RawManualMigrationFunctionDefV10 {
+            function_source_name: def.function_source_name,
+            previous_raw_module_def_bsatn_blake3_hash: def.previous_raw_module_def_bsatn_blake3_hash,
+        }
+    }
+}
+
 impl ModuleDefLookup for TableDef {
     type Key<'a> = &'a Identifier;
 
@@ -2090,6 +2118,16 @@ impl ModuleDefLookup for ViewDef {
     }
 }
 
+impl ModuleDefLookup for ManualMigrationFunctionDef {
+    type Key<'a> = &'a spacetimedb_lib::Hash;
+    fn key(&self) -> Self::Key<'_> {
+        &self.previous_raw_module_def_bsatn_blake3_hash
+    }
+    fn lookup<'def>(module_def: &'def ModuleDef, key: Self::Key<'_>) -> Option<&'def Self> {
+        module_def.manual_migration_functions.get(key)
+    }
+}
+
 fn to_raw<Def, RawDef, Name>(data: HashMap<Name, Def>) -> Vec<RawDef>
 where
     Def: ModuleDefLookup + Into<RawDef>,

crates/schema/src/def/validate/v10.rs

@@ -248,6 +248,10 @@ pub fn validate(def: RawModuleDefV10) -> Result<ModuleDef> {
         validate_http_routes_are_unique(&routes)?;
         Ok((handlers, routes))
     });
+
+    let migration_functions =
+        validate_migration_function_hashes_are_unique(def.manual_migration_functions().unwrap_or(&Vec::new()));
+
     // Combine all validation results
     let tables_types_reducers_procedures_views = (
         tables,
@@ -258,10 +262,21 @@ pub fn validate(def: RawModuleDefV10) -> Result<ModuleDef> {
         schedules,
         lifecycle_validations,
         http_handlers_and_routes,
+        migration_functions,
     )
         .combine_errors()
         .and_then(
-            |(mut tables, types, reducers, procedures, views, schedules, lifecycles, http_handlers_and_routes)| {
+            |(
+                mut tables,
+                types,
+                reducers,
+                procedures,
+                views,
+                schedules,
+                lifecycles,
+                http_handlers_and_routes,
+                migration_functions,
+            )| {
                 let (mut reducers, mut procedures, mut views) =
                     check_function_names_are_unique(reducers, procedures, views)?;
                 // Attach lifecycles to their respective reducers
@@ -275,7 +290,15 @@ pub fn validate(def: RawModuleDefV10) -> Result<ModuleDef> {
                 attach_view_primary_keys(&mut views, view_primary_keys)?;
                 assign_query_view_primary_keys(&tables, &mut views);
 
-                Ok((tables, types, reducers, procedures, views, http_handlers_and_routes))
+                Ok((
+                    tables,
+                    types,
+                    reducers,
+                    procedures,
+                    views,
+                    http_handlers_and_routes,
+                    migration_functions,
+                ))
             },
         );
     let CoreValidator {
@@ -292,11 +315,20 @@ pub fn validate(def: RawModuleDefV10) -> Result<ModuleDef> {
         .map(|rls| (rls.sql.clone(), rls.to_owned()))
         .collect();
 
-    let (tables, types, reducers, procedures, views, http_handlers, http_routes) =
+    let (tables, types, reducers, procedures, views, http_handlers, http_routes, migration_functions) =
         tables_types_reducers_procedures_views
             .map(
-                |(tables, types, reducers, procedures, views, (http_handlers, http_routes))| {
-                    (tables, types, reducers, procedures, views, http_handlers, http_routes)
+                |(tables, types, reducers, procedures, views, (http_handlers, http_routes), migration_functions)| {
+                    (
+                        tables,
+                        types,
+                        reducers,
+                        procedures,
+                        views,
+                        http_handlers,
+                        http_routes,
+                        migration_functions,
+                    )
                 },
             )
             .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?;
@@ -318,6 +350,7 @@ pub fn validate(def: RawModuleDefV10) -> Result<ModuleDef> {
         http_handlers,
         http_routes,
         raw_module_def_version: RawModuleDefVersion::V10,
+        manual_migration_functions: migration_functions,
     })
 }
 
@@ -412,6 +445,34 @@ fn check_http_handler_names_are_unique(
     ErrorStream::add_extra_errors(Ok(handlers_map), errors)
 }
 
+fn validate_migration_function_hashes_are_unique(
+    migrations: &[RawManualMigrationFunctionDefV10],
+) -> Result<IndexMap<spacetimedb_lib::Hash, ManualMigrationFunctionDef>> {
+    let mut errors = vec![];
+    let mut migrations_map: IndexMap<spacetimedb_lib::Hash, ManualMigrationFunctionDef> =
+        IndexMap::with_capacity(migrations.len());
+
+    for migration in migrations {
+        if let Some(other_migration) = migrations_map.get(&migration.previous_raw_module_def_bsatn_blake3_hash) {
+            errors.push(ValidationError::DuplicateManualMigrationPreviousHash {
+                previous_raw_module_def_bsatn_blake3_hash: migration.previous_raw_module_def_bsatn_blake3_hash,
+                function_source_name_a: other_migration.function_source_name.clone(),
+                function_source_name_b: migration.function_source_name.clone(),
+            });
+        } else {
+            migrations_map.insert(
+                migration.previous_raw_module_def_bsatn_blake3_hash,
+                ManualMigrationFunctionDef {
+                    function_source_name: migration.function_source_name.clone(),
+                    previous_raw_module_def_bsatn_blake3_hash: migration.previous_raw_module_def_bsatn_blake3_hash,
+                },
+            );
+        }
+    }
+
+    ErrorStream::add_extra_errors(Ok(migrations_map), errors)
+}
+
 struct ModuleValidatorV10<'a> {
     core: CoreValidator<'a>,
 }

crates/schema/src/def/validate/v9.rs

@@ -168,6 +168,7 @@ pub fn validate(def: RawModuleDefV9) -> Result<ModuleDef> {
         http_handlers: IndexMap::new(),
         http_routes: Vec::new(),
         raw_module_def_version: RawModuleDefVersion::V9OrEarlier,
+        manual_migration_functions: IndexMap::new(),
     })
 }
 

crates/schema/src/error.rs

@@ -176,6 +176,12 @@ pub enum ValidationError {
         ok_type: PrettyAlgebraicType,
         err_type: PrettyAlgebraicType,
     },
+    #[error("multiple manual migration functions defined to update from module def hash {previous_raw_module_def_bsatn_blake3_hash}, with identifiers {function_source_name_a} and {function_source_name_b}")]
+    DuplicateManualMigrationPreviousHash {
+        previous_raw_module_def_bsatn_blake3_hash: spacetimedb_lib::Hash,
+        function_source_name_a: RawIdentifier,
+        function_source_name_b: RawIdentifier,
+    },
 }
 
 /// A wrapper around an `AlgebraicType` that implements `fmt::Display`.