feat: route compute-tray power control through machine state controller

crates/api-core/src/handlers/component_manager.rs

@@ -43,8 +43,8 @@ use model::component_manager::{
     PowerShelfComponent,
 };
 use model::firmware::FirmwareComponentType;
-use model::machine::Machine;
 use model::machine::machine_search_config::MachineSearchConfig;
+use model::machine::{Machine, MachineMaintenanceOperation};
 use model::power_shelf::PowerShelfMaintenanceOperation;
 use model::rack::{FirmwareUpgradeJob, MaintenanceActivity};
 use model::switch::SwitchMaintenanceOperation;
@@ -307,6 +307,18 @@ fn map_switch_maintenance_operation(action: PowerAction) -> SwitchMaintenanceOpe
     }
 }
 
+fn map_machine_maintenance_operation(action: PowerAction) -> MachineMaintenanceOperation {
+    match action {
+        PowerAction::On => MachineMaintenanceOperation::PowerOn,
+        PowerAction::GracefulShutdown | PowerAction::ForceOff => {
+            MachineMaintenanceOperation::PowerOff
+        }
+        PowerAction::GracefulRestart | PowerAction::ForceRestart | PowerAction::AcPowercycle => {
+            MachineMaintenanceOperation::Reset
+        }
+    }
+}
+
 fn map_power_shelf_maintenance_operation(
     action: PowerAction,
 ) -> Result<PowerShelfMaintenanceOperation, &'static str> {
@@ -362,6 +374,47 @@ fn switch_maintenance_request_result_to_component_result(
     }
 }
 
+async fn queue_machine_power_control_via_state_controller(
+    api: &Api,
+    cm: &ComponentManager,
+    machine_ids: &[carbide_uuid::machine::MachineId],
+    action: PowerAction,
+) -> Result<Vec<rpc::ComponentResult>, Status> {
+    let operation = map_machine_maintenance_operation(action);
+    queue_machine_maintenance_via_state_controller(api, cm, machine_ids, operation).await
+}
+
+async fn queue_machine_maintenance_via_state_controller(
+    api: &Api,
+    cm: &ComponentManager,
+    machine_ids: &[carbide_uuid::machine::MachineId],
+    operation: MachineMaintenanceOperation,
+) -> Result<Vec<rpc::ComponentResult>, Status> {
+    let results = cm
+        .request_machine_maintenance_via_state_controller(
+            &api.database_connection,
+            machine_ids,
+            operation,
+            "component-manager",
+        )
+        .await
+        .map_err(component_manager_error_to_status)?;
+
+    Ok(results
+        .iter()
+        .map(machine_maintenance_request_result_to_component_result)
+        .collect())
+}
+
+fn machine_maintenance_request_result_to_component_result(
+    result: &component_manager::component_manager::MachineMaintenanceRequestResult,
+) -> rpc::ComponentResult {
+    match &result.error {
+        Some(error) => error_result(&result.machine_id.to_string(), error.clone()),
+        None => success_result(&result.machine_id.to_string()),
+    }
+}
+
 async fn queue_power_shelf_power_control_via_state_controller(
     api: &Api,
     power_shelf_ids: &[PowerShelfId],
@@ -1580,10 +1633,15 @@ pub(crate) async fn component_power_control(
         }
         rpc::component_power_control_request::Target::MachineIds(list) => {
             if cm.compute_tray_use_state_controller && !bypass_state_controller {
-                // TODO: implement state controller path for compute tray power control
-                return Err(Status::unimplemented(
-                    "compute tray power control through the state controller is not yet supported",
-                ));
+                let results = queue_machine_power_control_via_state_controller(
+                    api,
+                    cm,
+                    &list.machine_ids,
+                    action,
+                )
+                .await?;
+                let ips = Vec::new();
+                (results, ips)
             } else {
                 let resolved = resolve_compute_tray_endpoints(api, &list.machine_ids).await?;
 
@@ -3305,6 +3363,30 @@ mod tests {
         );
     }
 
+    #[test]
+    fn map_machine_maintenance_operation_variants() {
+        use model::machine::MachineMaintenanceOperation;
+
+        use super::map_machine_maintenance_operation;
+
+        assert_eq!(
+            map_machine_maintenance_operation(PowerAction::On),
+            MachineMaintenanceOperation::PowerOn,
+        );
+        assert_eq!(
+            map_machine_maintenance_operation(PowerAction::ForceOff),
+            MachineMaintenanceOperation::PowerOff,
+        );
+        assert_eq!(
+            map_machine_maintenance_operation(PowerAction::GracefulShutdown),
+            MachineMaintenanceOperation::PowerOff,
+        );
+        assert_eq!(
+            map_machine_maintenance_operation(PowerAction::ForceRestart),
+            MachineMaintenanceOperation::Reset,
+        );
+    }
+
     #[test]
     fn map_power_shelf_maintenance_operation_variants() {
         use model::power_shelf::PowerShelfMaintenanceOperation;

crates/api-core/src/setup.rs

@@ -1362,6 +1362,8 @@ async fn initialize_and_start_controllers<'a>(
                 redfish_client_pool: shared_redfish_pool.clone(),
                 ipmi_tool: ipmi_tool.clone(),
                 site_config: carbide_config.machine_state_handler_site_config().into(),
+                component_manager: component_manager.clone().map(Arc::new),
+                credential_manager: credential_manager.clone(),
                 per_object_metrics_registry: per_object_metrics_registry.clone(),
             }
             .into(),

crates/api-core/src/tests/common/api_fixtures/mod.rs

@@ -336,6 +336,8 @@ impl TestEnv {
             redfish_client_pool: self.redfish_sim.clone(),
             ipmi_tool: self.ipmi_tool.clone(),
             site_config: self.config.machine_state_handler_site_config().into(),
+            component_manager: self.test_component_manager.clone(),
+            credential_manager: self.test_credential_manager.clone(),
             per_object_metrics_registry: self.per_object_metrics_registry(),
         }
     }
@@ -420,6 +422,7 @@ impl TestEnv {
                 ManagedHostState::HostInit { machine_state: mc }
             }
             ManagedHostState::Ready => state.clone(),
+            ManagedHostState::Maintenance { .. } => state.clone(),
             ManagedHostState::Assigned { .. } => state.clone(),
             ManagedHostState::WaitingForCleanup { .. } => state.clone(),
             ManagedHostState::Created => state.clone(),
@@ -1542,6 +1545,8 @@ pub async fn create_test_env_with_overrides(
                 redfish_client_pool: redfish_sim.clone(),
                 ipmi_tool: ipmi_tool.clone(),
                 site_config: config.machine_state_handler_site_config().into(),
+                component_manager: test_component_manager.clone(),
+                credential_manager: credential_manager.clone(),
                 per_object_metrics_registry: per_object_metrics_registry.clone(),
             }
             .into(),

crates/api-db/migrations/20260717120000_machine_maintenance_requested.sql

@@ -0,0 +1,7 @@
+-- Add machine_maintenance_requested column to machines table.
+-- machine_maintenance_requested: when set by an external entity, the state controller
+-- transitions the host from Ready (or Failed) into Maintenance to execute the requested
+-- operation (PowerOn / PowerOff / Reset). Mirrors switches.switch_maintenance_requested.
+
+ALTER TABLE machines
+    ADD COLUMN machine_maintenance_requested JSONB;

crates/api-db/src/machine.rs

@@ -50,8 +50,8 @@ use model::machine::upgrade_policy::AgentUpgradePolicy;
 use model::machine::{
     Dpf, DpuInfo, DpuInfoStatusObservation, DpuOsOperationalState, DpuRepresentorStatus,
     FailureDetails, HostProfile, Machine, MachineInterfaceSnapshot, MachineLastRebootRequested,
-    MachineLastRebootRequestedMode, MachineValidationContext, ManagedHostState, ReprovisionRequest,
-    UpgradeDecision,
+    MachineLastRebootRequestedMode, MachineMaintenanceOperation, MachineValidationContext,
+    ManagedHostState, ReprovisionRequest, UpgradeDecision,
 };
 use model::machine_interface_address::MachineInterfaceAssociation;
 use model::metadata::Metadata;
@@ -2448,6 +2448,41 @@ pub async fn set_machine_validation_request(
     Ok(())
 }
 
+pub async fn set_machine_maintenance_requested(
+    txn: &mut PgConnection,
+    machine_id: MachineId,
+    initiator: &str,
+    operation: MachineMaintenanceOperation,
+) -> DatabaseResult<()> {
+    let req = model::machine::MachineMaintenanceRequest {
+        requested_at: Utc::now(),
+        initiator: initiator.to_string(),
+        operation,
+    };
+    let query = "UPDATE machines SET machine_maintenance_requested = $1 WHERE id = $2 RETURNING id";
+    sqlx::query_as::<_, MachineId>(query)
+        .bind(sqlx::types::Json(req))
+        .bind(machine_id)
+        .fetch_one(txn)
+        .await
+        .map_err(|e| DatabaseError::new("set_machine_maintenance_requested", e))?;
+    Ok(())
+}
+
+pub async fn clear_machine_maintenance_requested(
+    txn: &mut PgConnection,
+    machine_id: MachineId,
+) -> DatabaseResult<()> {
+    let query =
+        "UPDATE machines SET machine_maintenance_requested = NULL WHERE id = $1 RETURNING id";
+    sqlx::query_as::<_, MachineId>(query)
+        .bind(machine_id)
+        .fetch_one(txn)
+        .await
+        .map_err(|e| DatabaseError::new("clear_machine_maintenance_requested", e))?;
+    Ok(())
+}
+
 pub async fn update_dpu_asns(
     db_pool: &Pool<Postgres>,
     common_pools: &CommonPools,

crates/api-model/src/machine/json.rs

@@ -36,7 +36,8 @@ use crate::machine::spx::MachineSpxStatusObservation;
 use crate::machine::topology::MachineTopology;
 use crate::machine::{
     Dpf, FailureDetails, HostProfile, HostReprovisionRequest, Machine, MachineInterfaceSnapshot,
-    MachineLastRebootRequested, ManagedHostState, ReprovisionRequest, UpgradeDecision,
+    MachineLastRebootRequested, MachineMaintenanceRequest, ManagedHostState, ReprovisionRequest,
+    UpgradeDecision,
 };
 use crate::metadata::Metadata;
 use crate::power_manager::PowerOptions;
@@ -73,6 +74,7 @@ pub struct MachineSnapshotPgJson {
     pub failure_details: FailureDetails,
     pub reprovisioning_requested: Option<ReprovisionRequest>,
     pub host_reprovisioning_requested: Option<HostReprovisionRequest>,
+    pub machine_maintenance_requested: Option<MachineMaintenanceRequest>,
     pub manual_firmware_upgrade_completed: Option<DateTime<Utc>>,
     pub bios_password_set_time: Option<DateTime<Utc>>,
     pub last_machine_validation_time: Option<DateTime<Utc>>,
@@ -189,6 +191,7 @@ impl TryFrom<MachineSnapshotPgJson> for Machine {
             failure_details: value.failure_details,
             reprovision_requested: value.reprovisioning_requested,
             host_reprovision_requested: value.host_reprovisioning_requested,
+            machine_maintenance_requested: value.machine_maintenance_requested,
             manual_firmware_upgrade_completed: value.manual_firmware_upgrade_completed,
             dpu_agent_upgrade_requested: value.dpu_agent_upgrade_requested,
             health_reports: value.health_reports.unwrap_or_default(),

crates/api-model/src/machine/mod.rs

@@ -817,6 +817,10 @@ pub struct Machine {
     /// Last time when host reprovision requested
     pub host_reprovision_requested: Option<HostReprovisionRequest>,
 
+    /// When set by an external entity, the state controller transitions the host into
+    /// [`ManagedHostState::Maintenance`] to execute the requested operation.
+    pub machine_maintenance_requested: Option<MachineMaintenanceRequest>,
+
     /// Does the forge-dpu-agent on this DPU need upgrading?
     pub dpu_agent_upgrade_requested: Option<UpgradeDecision>,
 
@@ -1206,6 +1210,12 @@ pub enum ManagedHostState {
     },
     /// Host is Ready for instance creation.
     Ready,
+
+    /// Host is executing an operator-requested maintenance operation.
+    Maintenance {
+        operation: MachineMaintenanceOperation,
+    },
+
     /// Host is assigned to an Instance.
     Assigned {
         instance_state: InstanceState,
@@ -1357,6 +1367,11 @@ impl std::fmt::Display for ValidationState {
 pub const MAX_FIRMWARE_UPGRADE_RETRIES: u32 = 5;
 
 impl ManagedHostState {
+    /// Builds the controller state for a requested maintenance operation.
+    pub fn maintenance_for_operation(operation: MachineMaintenanceOperation) -> Self {
+        Self::Maintenance { operation }
+    }
+
     pub fn as_reprovision_state(&self, dpu_id: &MachineId) -> Option<&ReprovisionState> {
         match self {
             ManagedHostState::DPUReprovision { dpu_states } => dpu_states.states.get(dpu_id),
@@ -2175,6 +2190,25 @@ pub struct ReprovisionRequest {
     pub restart_reprovision_requested_at: DateTime<Utc>,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(tag = "operation", rename_all = "lowercase")]
+#[allow(clippy::enum_variant_names)]
+pub enum MachineMaintenanceOperation {
+    /// Power on the host.
+    PowerOn,
+    /// Power off the host.
+    PowerOff,
+    /// Reset the host (restart / AC power cycle).
+    Reset,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MachineMaintenanceRequest {
+    pub requested_at: DateTime<Utc>,
+    pub initiator: String,
+    pub operation: MachineMaintenanceOperation,
+}
+
 /// Struct to store information if host reprovision is requested.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct HostReprovisionRequest {
@@ -2321,6 +2355,9 @@ impl Display for ManagedHostState {
                 write!(f, "HostInitializing/{machine_state}")
             }
             ManagedHostState::Ready => write!(f, "Ready"),
+            ManagedHostState::Maintenance { operation } => {
+                write!(f, "Maintenance({operation:?})")
+            }
             ManagedHostState::Assigned { instance_state, .. } => match instance_state {
                 InstanceState::DPUReprovision { dpu_states } => {
                     let dpu_lowest_state = dpu_states
@@ -2413,6 +2450,9 @@ impl ManagedHostState {
                 format!("HostInitializing/{machine_state}")
             }
             ManagedHostState::Ready => "Ready".to_string(),
+            ManagedHostState::Maintenance { operation } => {
+                format!("Maintenance({operation:?})")
+            }
             ManagedHostState::Assigned { instance_state } => match instance_state {
                 InstanceState::DPUReprovision { dpu_states } => {
                     format!(
@@ -2612,6 +2652,9 @@ pub fn state_sla(
             _ => StateSla::with_sla(slas::HOST_INIT, time_in_state),
         },
         ManagedHostState::Ready => StateSla::no_sla(),
+        ManagedHostState::Maintenance { .. } => {
+            StateSla::with_sla(slas::MAINTENANCE, time_in_state)
+        }
         ManagedHostState::Assigned { instance_state } => match instance_state {
             InstanceState::Ready => StateSla::no_sla(),
             InstanceState::BootingWithDiscoveryImage { retry } => {

crates/api-model/src/machine/slas.rs

@@ -57,6 +57,8 @@ pub const ASSIGNED: Duration = Duration::from_secs(30 * 60);
 pub const ASSIGNED_HOST_PLATFORM_CONFIGURATION: Duration = Duration::from_secs(90 * 60);
 pub const VALIDATION: Duration = Duration::from_secs(30 * 60);
 
+pub const MAINTENANCE: Duration = Duration::from_secs(5 * 60);
+
 /// Configuration for machine state SLA durations.
 #[derive(Clone, Debug, PartialEq)]
 pub struct MachineSlaConfig {

crates/api-model/src/test_support/machine_snapshot.rs

@@ -359,6 +359,7 @@ pub fn machine_snapshot_pg_json(machine_id: MachineId) -> MachineSnapshotPgJson
     };
 
     MachineSnapshotPgJson {
+        machine_maintenance_requested: None,
         id: machine_id,
         rack_id: Some("rack-bench-01".parse().expect("valid rack id")),
         created: fixture_time(0),