Add spacetime_worker_sender_errors_total metric

Currently we have a metric for reducer panics called
`spacetime_worker_wasm_instance_errors_total`. This commit adds a
metric for tracking errors returned from the module, like for example an
Err result in Rust, or throwing a SenderError in TypeScript

Author: drogus

crates/core/src/host/wasm_common/module_host_actor.rs

@@ -4,6 +4,7 @@ use spacetimedb_datastore::locking_tx_datastore::FuncCallType;
 use spacetimedb_datastore::locking_tx_datastore::ViewCall;
 use spacetimedb_lib::db::raw_def::v9::Lifecycle;
 use spacetimedb_lib::de::DeserializeSeed as _;
+use spacetimedb_lib::Hash;
 use spacetimedb_primitives::ProcedureId;
 use spacetimedb_primitives::ViewDatabaseId;
 use spacetimedb_primitives::ViewId;
@@ -601,7 +602,13 @@ impl InstanceCommon {
                 )
             }
             Err(ExecutionError::User(err)) => {
-                log_reducer_error(inst.replica_ctx(), timestamp, reducer_name, &err);
+                log_reducer_error(
+                    inst.replica_ctx(),
+                    timestamp,
+                    reducer_name,
+                    &err,
+                    &self.info.module_hash,
+                );
                 EventStatus::Failed(err.into())
             }
             // We haven't actually committed yet - `commit_and_broadcast_event` will commit
@@ -619,7 +626,13 @@ impl InstanceCommon {
                     Ok(()) => EventStatus::Committed(DatabaseUpdate::default()),
                     Err(err) => {
                         let err = err.to_string();
-                        log_reducer_error(inst.replica_ctx(), timestamp, reducer_name, &err);
+                        log_reducer_error(
+                            inst.replica_ctx(),
+                            timestamp,
+                            reducer_name,
+                            &err,
+                            &self.info.module_hash,
+                        );
                         EventStatus::Failed(err)
                     }
                 }
@@ -902,9 +915,20 @@ fn maybe_log_long_running_function(reducer_name: &str, total_duration: Duration)
 }
 
 /// Logs an error `message` for `reducer` at `timestamp` into `replica_ctx`.
-fn log_reducer_error(replica_ctx: &ReplicaContext, timestamp: Timestamp, reducer: &str, message: &str) {
+fn log_reducer_error(
+    replica_ctx: &ReplicaContext,
+    timestamp: Timestamp,
+    reducer: &str,
+    message: &str,
+    module_hash: &Hash,
+) {
     use database_logger::Record;
 
+    WORKER_METRICS
+        .sender_errors
+        .with_label_values(&replica_ctx.database_identity, module_hash, reducer)
+        .inc();
+
     log::info!("reducer returned error: {message}");
 
     let record = Record {

crates/core/src/worker_metrics/mod.rs

@@ -246,6 +246,11 @@ metrics_group!(
         #[labels(caller_identity: Identity, module_hash: Hash, caller_connection_id: ConnectionId, reducer_symbol: str)]
         pub wasm_instance_errors: IntCounterVec,
 
+        #[name = spacetime_worker_sender_errors_total]
+        #[help = "The number of sender errors returned from reducers."]
+        #[labels(database_identity: Identity, module_hash: Hash, reducer_symbol: str)]
+        pub sender_errors: IntCounterVec,
+
         #[name = spacetime_worker_wasm_memory_bytes]
         #[help = "The number of bytes of linear memory allocated by the database's WASM module instance"]
         #[labels(database_identity: Identity)]