[reconfigurator] Prechecks and post-update actions for RoT updater (#8157)

Related: #7989

Author: karencfv

Cargo.lock

@@ -12,6 +12,7 @@ base64.workspace = true
 chrono.workspace = true
 daft.workspace = true
 gateway-messages.workspace = true
+gateway-types.workspace = true
 progenitor.workspace = true
 rand.workspace = true
 reqwest = { workspace = true, features = ["rustls-tls", "stream"] }

clients/gateway-client/Cargo.toml

@@ -64,7 +64,6 @@ progenitor::generate_api!(
         ImageVersion = { derives = [PartialEq, Eq, PartialOrd, Ord] },
         RotImageDetails = { derives = [PartialEq, Eq, PartialOrd, Ord] },
         RotImageError = { derives = [ PartialEq, Eq, PartialOrd, Ord] },
-        RotSlot = { derives = [PartialEq, Eq, PartialOrd, Ord] },
         RotState = { derives = [PartialEq, Eq, PartialOrd, Ord] },
         SpComponentCaboose = { derives = [PartialEq, Eq] },
         SpIdentifier = { derives = [Copy, PartialEq, Hash, Eq] },
@@ -75,6 +74,7 @@ progenitor::generate_api!(
         SpUpdateStatus = { derives = [PartialEq, Hash, Eq] },
         UpdatePreparationProgress = { derives = [PartialEq, Hash, Eq] },
     },
+    replace = { RotSlot = gateway_types::rot::RotSlot },
 );
 
 // Override the impl of Ord for SpIdentifier because the default one orders the

clients/gateway-client/src/lib.rs

@@ -28,6 +28,7 @@ futures.workspace = true
 gateway-client.workspace = true
 gateway-messages.workspace = true
 gateway-test-utils.workspace = true
+gateway-types.workspace = true
 http.workspace = true
 humantime.workspace = true
 internal-dns-resolver.workspace = true

dev-tools/omdb/Cargo.toml

@@ -12,7 +12,6 @@ use clap::Args;
 use clap::Subcommand;
 use futures::StreamExt;
 use gateway_client::types::PowerState;
-use gateway_client::types::RotSlot;
 use gateway_client::types::RotState;
 use gateway_client::types::SpComponentCaboose;
 use gateway_client::types::SpComponentInfo;
@@ -22,6 +21,7 @@ use gateway_client::types::SpIgnitionInfo;
 use gateway_client::types::SpIgnitionSystemType;
 use gateway_client::types::SpState;
 use gateway_client::types::SpType;
+use gateway_types::rot::RotSlot;
 use internal_dns_types::names::ServiceName;
 use tabled::Tabled;
 

dev-tools/omdb/src/bin/omdb/mgs.rs

@@ -323,23 +323,22 @@ fn cmd_config(
                 )?;
             }
             PendingMgsUpdateDetails::Rot {
-                expected_slot_a_version,
-                expected_slot_b_version,
                 expected_active_slot,
+                expected_inactive_version,
                 expected_persistent_boot_preference,
                 expected_pending_persistent_boot_preference,
                 expected_transient_boot_preference,
             } => {
                 writeln!(
                     &mut s,
-                    "        preconditions: expected_slot_a_version {:?}
-                                            expected_slot_b_version {:?}
-                                            expected active_slot {:?}
+                    "        preconditions: expected active slot {:?}
+                                            expected active version {:?}
+                                            expected inactive version {:?}
                                             expected persistent_boot_preference {:?}
                                             expected pending_persistent_boot_preference {:?}
                                             expected transient_boot_preference {:?}",
-                    expected_slot_a_version, expected_slot_b_version,
-                    expected_active_slot, expected_persistent_boot_preference,
+                    expected_active_slot.slot(), expected_active_slot.version(),
+                    expected_inactive_version, expected_persistent_boot_preference,
                     expected_pending_persistent_boot_preference,
                     expected_transient_boot_preference,
                 )?;

dev-tools/reconfigurator-sp-updater/src/main.rs

@@ -8,6 +8,7 @@ license = "MPL-2.0"
 workspace = true
 
 [dependencies]
+daft.workspace = true
 gateway-messages.workspace = true
 # Avoid depending on gateway-sp-comms! It is a pretty heavy dependency and
 # would only be used for From impls anyway. We put those impls in

gateway-types/Cargo.toml

@@ -2,6 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+use daft::Diffable;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
@@ -165,7 +166,9 @@ impl From<gateway_messages::RotBootInfo> for RotState {
 
 #[derive(
     Debug,
+    Diffable,
     Clone,
+    Copy,
     PartialEq,
     Eq,
     PartialOrd,

gateway-types/src/rot.rs

@@ -17,6 +17,7 @@ clickhouse-admin-types.workspace = true
 futures.workspace = true
 gateway-client.workspace = true
 gateway-messages.workspace = true
+gateway-types.workspace = true
 id-map.workspace = true
 nexus-sled-agent-shared.workspace = true
 nexus-types.workspace = true

nexus/inventory/Cargo.toml

@@ -591,11 +591,11 @@ mod test {
     use base64::Engine;
     use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
     use gateway_client::types::PowerState;
-    use gateway_client::types::RotSlot;
     use gateway_client::types::RotState;
     use gateway_client::types::SpComponentCaboose;
     use gateway_client::types::SpState;
     use gateway_client::types::SpType;
+    use gateway_types::rot::RotSlot;
     use nexus_sled_agent_shared::inventory::SledRole;
     use nexus_types::inventory::BaseboardId;
     use nexus_types::inventory::Caboose;

nexus/inventory/src/builder.rs

@@ -8,11 +8,11 @@ use crate::CollectionBuilder;
 use clickhouse_admin_types::ClickhouseKeeperClusterMembership;
 use clickhouse_admin_types::KeeperId;
 use gateway_client::types::PowerState;
-use gateway_client::types::RotSlot;
 use gateway_client::types::RotState;
 use gateway_client::types::SpComponentCaboose;
 use gateway_client::types::SpState;
 use gateway_client::types::SpType;
+use gateway_types::rot::RotSlot;
 use nexus_sled_agent_shared::inventory::Baseboard;
 use nexus_sled_agent_shared::inventory::Inventory;
 use nexus_sled_agent_shared::inventory::InventoryDataset;

nexus/inventory/src/examples.rs

@@ -10,6 +10,7 @@ workspace = true
 chrono.workspace = true
 futures.workspace = true
 gateway-client.workspace = true
+gateway-types.workspace = true
 id-map.workspace = true
 internal-dns-resolver.workspace = true
 internal-dns-types.workspace = true

nexus/mgs-updates/Cargo.toml

@@ -10,6 +10,7 @@ use super::UpdateProgress;
 use futures::future::BoxFuture;
 use gateway_client::types::SpType;
 use gateway_client::types::SpUpdateStatus;
+use gateway_types::rot::RotSlot;
 use nexus_types::deployment::ExpectedVersion;
 use nexus_types::deployment::PendingMgsUpdate;
 use slog::Logger;
@@ -278,9 +279,17 @@ pub enum PrecheckStatus {
 
 #[derive(Debug, Error)]
 pub enum PrecheckError {
+    #[error(
+        "pending_persistent_boot_preference and/or transient_boot_preference is set"
+    )]
+    EphemeralRotBootPreferenceSet,
+
     #[error("communicating with MGS")]
     GatewayClientError(#[from] GatewayClientError),
 
+    #[error("communicating with RoT: {message:?}")]
+    RotCommunicationFailed { message: String },
+
     #[error(
         "in {sp_type} slot {slot_id}, expected to find \
          part {expected_part:?} serial {expected_serial:?}, but found \
@@ -295,6 +304,9 @@ pub enum PrecheckError {
         found_serial: String,
     },
 
+    #[error("expected to find active slot {expected:?}, but found {found:?}")]
+    WrongActiveSlot { expected: RotSlot, found: RotSlot },
+
     #[error(
         "expected to find active version {:?}, but found {found:?}",
         .expected.as_str(),

nexus/mgs-updates/src/common_sp_update.rs

@@ -89,7 +89,10 @@ impl SpComponentUpdate {
                     target_sp_type: request.sp_type,
                     target_sp_slot: request.slot_id,
                     // Like the SP, we request an update to the inactive slot
-                    firmware_slot: expected_active_slot.toggled().to_u16(),
+                    firmware_slot: expected_active_slot
+                        .slot()
+                        .toggled()
+                        .to_u16(),
                     update_id,
                 }
             }
@@ -581,11 +584,14 @@ async fn wait_for_update_done(
             // Check if we're done.
             Ok(PrecheckStatus::UpdateComplete) => return Ok(()),
 
-            // An incorrect version in the "inactive" slot is normal during the
-            // upgrade.  We have no reason to think this won't converge so we
-            // proceed with waiting.
+            // An incorrect version in the "inactive" slot, incorrect active slot,
+            // or non-empty pending_persistent_boot_preference/transient_boot_preference
+            // are normal during the upgrade. We have no reason to think these won't
+            // converge so we proceed with waiting.
             Err(PrecheckError::GatewayClientError(_))
             | Err(PrecheckError::WrongInactiveVersion { .. })
+            | Err(PrecheckError::WrongActiveSlot { .. })
+            | Err(PrecheckError::EphemeralRotBootPreferenceSet)
             | Ok(PrecheckStatus::ReadyForUpdate) => {
                 if before.elapsed() >= timeout {
                     return Err(UpdateWaitError::Timeout(timeout));
@@ -596,7 +602,8 @@ async fn wait_for_update_done(
             }
 
             Err(error @ PrecheckError::WrongDevice { .. })
-            | Err(error @ PrecheckError::WrongActiveVersion { .. }) => {
+            | Err(error @ PrecheckError::WrongActiveVersion { .. })
+            | Err(error @ PrecheckError::RotCommunicationFailed { .. }) => {
                 // Stop trying to make this update happen.  It's not going to
                 // happen.
                 return Err(UpdateWaitError::Indeterminate(error));