Do not count missing records as a failure of the cluster readiness check (#3067)

* Do not consider missing records as a cluster readiness check failure

(cherry picked from commit 75c5867)

* Update CHANGELOG

(cherry picked from commit 94943dd)

* Add note that missing records don't cause failure

(cherry picked from commit 16ad89f)

Author: hgreebe

CHANGELOG.md

@@ -31,6 +31,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
   - Open MPI: openmpi40-aws-4.1.7-2 and openmpi50-aws-5.0.8-11
 
 **BUG FIXES**
+- Prevent cluster readiness check failures due to instances launched while the check is in progress.
 - Fix race condition where compute nodes could deploy the wrong cluster config version after an update failure.
 
 3.14.0

cookbooks/aws-parallelcluster-slurm/files/default/head_node_checks/check_cluster_ready.py

@@ -112,13 +112,21 @@ def check_deployed_config_version(cluster_name: str, table_name: str, expected_c
 
         missing, incomplete, wrong = _check_cluster_config_items(instance_ids, items, expected_config_version)
 
-        if missing or incomplete or wrong:
+        if incomplete or wrong:
             raise CheckFailedError(
-                f"Check failed due to the following erroneous records:\n"
+                f"Check failed due to the following erroneous records "
+                f"(missing records are not counted for the failure):\n"
                 f"  * missing records ({len(missing)}): {missing}\n"
                 f"  * incomplete records ({len(incomplete)}): {incomplete}\n"
                 f"  * wrong records ({len(wrong)}): {wrong}"
             )
+        if missing:
+            logger.warning(
+                "Ignoring the following missing records due them being recently bootstrapped:\n"
+                "  *  missing records (%s): %s",
+                len(missing),
+                missing,
+            )
         logger.info("Verified cluster configuration for cluster node(s) %s", instance_ids)
 
 

test/unit/head_node_checks/test_check_cluster_ready.py

@@ -83,10 +83,7 @@ def _mocked_request_batch_get_items(table_name: str, compute_nodes: [str], ddb_r
             ["i-cmp123456789"],
             ["i-lgn123456789"],
             {},
-            "Check failed due to the following erroneous records:\n"
-            "  * missing records (2): ['i-cmp123456789', 'i-lgn123456789']\n"
-            "  * incomplete records (0): []\n"
-            "  * wrong records (0): []",
+            None,
             id="Check with missing DDB records",
         ),
         pytest.param(
@@ -96,7 +93,7 @@ def _mocked_request_batch_get_items(table_name: str, compute_nodes: [str], ddb_r
                 "i-cmp123456789": {"UNEXPECTED_KEY_A": {"S": "UNEXPECTED_KEY_VALUE_A"}},
                 "i-lgn123456789": {"UNEXPECTED_KEY_B": {"S": "UNEXPECTED_KEY_VALUE_B"}},
             },
-            "Check failed due to the following erroneous records:\n"
+            "Check failed due to the following erroneous records (missing records are not counted for the failure):\n"
             "  * missing records (0): []\n"
             "  * incomplete records (2): ['i-cmp123456789', 'i-lgn123456789']\n"
             "  * wrong records (0): []",
@@ -109,7 +106,7 @@ def _mocked_request_batch_get_items(table_name: str, compute_nodes: [str], ddb_r
                 "i-cmp123456789": {"cluster_config_version": {"S": "WRONG_CLUSTER_CONFIG_VERSION_A"}},
                 "i-lgn123456789": {"cluster_config_version": {"S": "WRONG_CLUSTER_CONFIG_VERSION_B"}},
             },
-            "Check failed due to the following erroneous records:\n"
+            "Check failed due to the following erroneous records (missing records are not counted for the failure):\n"
             "  * missing records (0): []\n"
             "  * incomplete records (0): []\n"
             "  * wrong records (2): [('i-cmp123456789', 'WRONG_CLUSTER_CONFIG_VERSION_A'), "
@@ -127,7 +124,7 @@ def _mocked_request_batch_get_items(table_name: str, compute_nodes: [str], ddb_r
                 "i-cmp1234567893": {"cluster_config_version": {"S": "WRONG_CLUSTER_CONFIG_VERSION_A"}},
                 "i-lgn1234567893": {"cluster_config_version": {"S": "WRONG_CLUSTER_CONFIG_VERSION_B"}},
             },
-            "Check failed due to the following erroneous records:\n"
+            "Check failed due to the following erroneous records (missing records are not counted for the failure):\n"
             "  * missing records (2): ['i-cmp1234567894', 'i-lgn1234567894']\n"
             "  * incomplete records (2): ['i-cmp1234567892', 'i-lgn1234567892']\n"
             "  * wrong records (2): [('i-cmp1234567893', 'WRONG_CLUSTER_CONFIG_VERSION_A'), "