Add a metric for the number of module instances (#4674)

# Description of Changes

It looks like we could be creating a lot of instances, so it would be
nice to have a metric to track it.


# Expected complexity level and risk

1

crates/core/src/host/module_host.rs

@@ -752,6 +752,7 @@ impl CallProcedureParams {
 struct ModuleInstanceManager<M: GenericModule> {
     instances: Mutex<VecDeque<M::Instance>>,
     module: M,
+    module_instances_metric: ModuleInstancesMetric,
     create_instance_time_metric: CreateInstanceTimeMetric,
 }
 
@@ -763,6 +764,15 @@ struct CreateInstanceTimeMetric {
     database_identity: Identity,
 }
 
+/// Handle on the `spacetime_module_instances` label for a particular database
+/// which calls `remove_label_values` to clean up on drop.
+struct ModuleInstancesMetric {
+    metric: IntGauge,
+    host_type: HostType,
+    database_identity: Identity,
+    count: std::sync::Mutex<i64>,
+}
+
 impl Drop for CreateInstanceTimeMetric {
     fn drop(&mut self) {
         let _ = WORKER_METRICS
@@ -771,6 +781,31 @@ impl Drop for CreateInstanceTimeMetric {
     }
 }
 
+impl Drop for ModuleInstancesMetric {
+    fn drop(&mut self) {
+        let _ = WORKER_METRICS
+            .module_instances
+            .remove_label_values(&self.database_identity, &self.host_type);
+    }
+}
+
+impl ModuleInstancesMetric {
+    fn inc(&self) {
+        let mut count = self.count.lock().unwrap();
+        *count += 1;
+        self.metric.set(*count);
+    }
+
+    fn dec(&self) {
+        let mut count = self.count.lock().unwrap();
+        if *count == 0 {
+            return;
+        }
+        *count -= 1;
+        self.metric.set(*count);
+    }
+}
+
 impl CreateInstanceTimeMetric {
     fn observe(&self, duration: std::time::Duration) {
         self.metric.observe(duration.as_secs_f64());
@@ -780,6 +815,15 @@ impl CreateInstanceTimeMetric {
 impl<M: GenericModule> ModuleInstanceManager<M> {
     fn new(module: M, init_inst: M::Instance, database_identity: Identity) -> Self {
         let host_type = module.host_type();
+        let module_instances_metric = ModuleInstancesMetric {
+            metric: WORKER_METRICS
+                .module_instances
+                .with_label_values(&database_identity, &host_type),
+            host_type,
+            database_identity,
+            count: std::sync::Mutex::new(1),
+        };
+
         let create_instance_time_metric = CreateInstanceTimeMetric {
             metric: WORKER_METRICS
                 .module_create_instance_time_seconds
@@ -795,6 +839,7 @@ impl<M: GenericModule> ModuleInstanceManager<M> {
         Self {
             instances: Mutex::new(instances),
             module,
+            module_instances_metric,
             create_instance_time_metric,
         }
     }
@@ -815,6 +860,7 @@ impl<M: GenericModule> ModuleInstanceManager<M> {
             let res = self.module.create_instance().await;
             let elapsed_time = start_time.elapsed();
             self.create_instance_time_metric.observe(elapsed_time);
+            self.module_instances_metric.inc();
             res
         }
     }
@@ -824,6 +870,7 @@ impl<M: GenericModule> ModuleInstanceManager<M> {
             // Don't return trapped instances;
             // they may have left internal data structures in the guest `Instance`
             // (WASM linear memory, V8 global scope) in a bad state.
+            self.module_instances_metric.dec();
             return;
         }
 

crates/core/src/worker_metrics/mod.rs

@@ -254,6 +254,11 @@ metrics_group!(
         #[buckets(0, 1, 2, 5, 10, 25, 50, 75, 100, 200, 300, 400, 500, 1000)]
         pub instance_queue_length_histogram: HistogramVec,
 
+        #[name = spacetime_module_instances]
+        #[help = "Current number of live module instances (WASM or V8) for this database"]
+        #[labels(database_identity: Identity, module_type: HostType)]
+        pub module_instances: IntGaugeVec,
+
         #[name = spacetime_reducer_wait_time_sec]
         #[help = "The amount of time (in seconds) a reducer spends in the queue waiting to run"]
         #[labels(db: Identity, reducer: str)]