cherry-pick: Set unhealthy nodes static nodes to down with reset node address

Set unhealthy nodes static nodes to down with reset node address in order to fix ice static nodes after a bootstrap failure being treated as bootstrap failure nodes.

Here's the bootstrap failure case that static node in replacement will ice issue will fail with:
   def is_bootstrap_failure(self):
        """Check if a slurm node has boostrap failure."""
        if self.is_static_nodes_in_replacement and not self.is_backing_instance_valid(log_warn_if_unhealthy=False):
            # Node is currently in replacement and no backing instance
            logger.warning(
                "Node bootstrap error: Node %s is currently in replacement and no backing instance, node state %s:",
                self,
                self.state_string,
            )

Behaviors before the change:
When detect unhealthy static node, static nodes will be set to down when it is unhealthy. In the same iteration, a run_instance call will be performed to launch a new instance for the node, node address will be changed to the new one if the run_instance call is succesfully.
If the run_instance call failed, node address will be remained, node will be treat as bootstrap failure node.

After this change, When detect unhealthy static node, static nodes will be set to down with node address reset. If run_instance call is successfully, node will be set to new address. If run_instance call failed, node address will be node_name. Node will not be treated as bootstrap failure.

Signed-off-by: chenwany <[email protected]>

Author: chenwany

src/slurm_plugin/clustermgtd.py

@@ -27,7 +27,6 @@
     get_partition_info,
     reset_nodes,
     resume_powering_down_nodes,
-    set_nodes_down,
     set_nodes_drain,
     set_nodes_power_down,
     update_all_partitions,
@@ -719,7 +718,7 @@ def _handle_unhealthy_static_nodes(self, unhealthy_static_nodes):
         # Set nodes into down state so jobs can be requeued immediately
         try:
             log.info("Setting unhealthy static nodes to DOWN")
-            set_nodes_down(node_list, reason="Static node maintenance: unhealthy node is being replaced")
+            reset_nodes(node_list, state="down", reason="Static node maintenance: unhealthy node is being replaced")
         except Exception as e:
             log.error("Encountered exception when setting unhealthy static nodes into down state: %s", e)
 

tests/slurm_plugin/test_clustermgtd.py

@@ -816,11 +816,13 @@ def test_handle_unhealthy_static_nodes(
     # Mock add_instances_for_nodes but still try to execute original code
     original_add_instances = cluster_manager._instance_manager.add_instances_for_nodes
     cluster_manager._instance_manager.add_instances_for_nodes = mocker.MagicMock(side_effect=original_add_instances)
-    update_mock = mocker.patch("slurm_plugin.clustermgtd.set_nodes_down", return_value=None, auto_spec=True)
+    reset_mock = mocker.patch("slurm_plugin.clustermgtd.reset_nodes", return_value=None, auto_spec=True)
     # Run test
     cluster_manager._handle_unhealthy_static_nodes(unhealthy_static_nodes)
     # Assert calls
-    update_mock.assert_called_with(add_node_list, reason="Static node maintenance: unhealthy node is being replaced")
+    reset_mock.assert_called_with(
+        add_node_list, reason="Static node maintenance: unhealthy node is being replaced", state="down"
+    )
     if delete_instance_list:
         cluster_manager._instance_manager.delete_instances.assert_called_with(
             delete_instance_list, terminate_batch_size=1
@@ -2221,11 +2223,11 @@ def test_handle_failed_health_check_nodes_in_replacement(
     expected_nodes_in_replacement,
     mocker,
 ):
-    for node, is_static_nodes_in_replacement, is_failing_health_check in zip(
+    for node, is_node_in_replacement, is_node_failing_health_check in zip(
         active_nodes, is_static_nodes_in_replacement, is_failing_health_check
     ):
-        node.is_static_nodes_in_replacement = is_static_nodes_in_replacement
-        node.is_failing_health_check = is_failing_health_check
+        node.is_static_nodes_in_replacement = is_node_in_replacement
+        node.is_failing_health_check = is_node_failing_health_check
 
     cluster_manager = ClusterManager(mocker.MagicMock())
     cluster_manager._static_nodes_in_replacement = current_nodes_in_replacement