Test fixes for ModelPackage and move check for tags once resource completed (#71)

Checking for tags after the check for the existence of the resource arn may cause flaky tests where the tags aren't created/applied yet. Tests have failed with tags returning as empty even though after checking that resource later on console tags do exist.

This pr moves this check for after the resource has reached created/completed status which should provide sufficient time for tags to be created/applied.

Author: ryansteakley

test/canary/scripts/install_controller_helm.sh

@@ -14,7 +14,6 @@ function install_helm_chart() {
     yq w -i helm/values.yaml "aws.region" $region
     yq w -i helm/values.yaml "aws.account_id" $account_id
 
-    kubectl create namespace $namespace
     kubectl apply -f helm/crds
-    helm install -n $namespace ack-$service-controller --skip-crds helm
+    helm install -n $namespace --create-namespace ack-$service-controller --skip-crds helm
 }

test/e2e/common/config.py

@@ -28,3 +28,6 @@
 ENDPOINT_STATUS_INSERVICE = "InService"
 ENDPOINT_STATUS_CREATING = "Creating"
 ENDPOINT_STATUS_UPDATING = "Updating"
+
+DELETE_WAIT_PERIOD = 4
+DELETE_WAIT_LENGTH = 30

test/e2e/tests/test_endpoint.py

@@ -240,9 +240,6 @@ def create_endpoint_test(self, sagemaker_client, xgboost_endpoint):
         endpoint_arn = endpoint_desc["EndpointArn"]
         assert self._get_resource_endpoint_arn(resource) == endpoint_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(endpoint_arn, resource_tags)
-
         # endpoint transitions Creating -> InService state
         assert_endpoint_status_in_sync(
             endpoint_name, reference, cfg.ENDPOINT_STATUS_CREATING
@@ -254,6 +251,9 @@ def create_endpoint_test(self, sagemaker_client, xgboost_endpoint):
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(endpoint_arn, resource_tags)
+
     def update_endpoint_failed_test(
         self, sagemaker_client, single_variant_config, faulty_config, xgboost_endpoint
     ):

test/e2e/tests/test_hpo.py

@@ -162,9 +162,6 @@ def test_completed(self, xgboost_hpojob):
         hpo_arn = hpo_sm_desc["HyperParameterTuningJobArn"]
         assert k8s.get_resource_arn(resource) == hpo_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(hpo_arn, resource_tags)
-
         assert hpo_sm_desc["HyperParameterTuningJobStatus"] == cfg.JOB_STATUS_INPROGRESS
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
 
@@ -173,6 +170,9 @@ def test_completed(self, xgboost_hpojob):
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(hpo_arn, resource_tags)
+
         # Check that you can delete a completed resource from k8s
         _, deleted = k8s.delete_custom_resource(reference, 3, 10)
         assert deleted is True

test/e2e/tests/test_model_package.py

@@ -33,6 +33,9 @@
 
 RESOURCE_PLURAL = "modelpackages"
 
+DELETE_WAIT_PERIOD = 20
+DELETE_WAIT_LENGTH = 30
+
 # Disabled since versioned model package is not supported
 # Issue with multiple creates.
 
@@ -111,7 +114,9 @@ def xgboost_unversioned_model_package():
     yield (reference, resource)
 
     if k8s.get_resource_exists(reference):
-        _, deleted = k8s.delete_custom_resource(reference, 3, 10)
+        _, deleted = k8s.delete_custom_resource(
+            reference, DELETE_WAIT_PERIOD, DELETE_WAIT_LENGTH
+        )
         assert deleted
 
 
@@ -202,10 +207,6 @@ def test_unversioned_model_package_completed(
 
         assert k8s.get_resource_arn(resource) == model_package_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(model_package_arn, resource_tags)
-
-        assert model_package_desc["ModelPackageStatus"] == cfg.JOB_STATUS_INPROGRESS
         self._assert_model_package_status_in_sync(
             model_package_name, reference, cfg.JOB_STATUS_INPROGRESS
         )
@@ -216,8 +217,13 @@ def test_unversioned_model_package_completed(
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(model_package_arn, resource_tags)
+
         # Check that you can delete a completed resource from k8s
-        _, deleted = k8s.delete_custom_resource(reference, 3, 10)
+        _, deleted = k8s.delete_custom_resource(
+            reference, DELETE_WAIT_PERIOD, DELETE_WAIT_LENGTH
+        )
         assert deleted is True
         assert get_sagemaker_model_package(model_package_name) is None
 
@@ -275,4 +281,4 @@ def test_unversioned_model_package_completed(
     #     # Check that you can delete a completed resource from k8s
     #     _, deleted = k8s.delete_custom_resource(reference, 3, 10)
     #     assert deleted is True
-    #     assert get_sagemaker_model_package(model_package_name) is None
+    #     assert get_sagemaker_model_package(model_package_name) is None

test/e2e/tests/test_model_package_group.py

@@ -35,7 +35,7 @@
 
 @pytest.fixture(scope="module")
 def xgboost_model_package_group():
-    resource_name = random_suffix_name("xgboost-model-package-group", 32)
+    resource_name = random_suffix_name("xgboost-model-package-group", 50)
 
     replacements = REPLACEMENT_VALUES.copy()
     replacements["MODEL_PACKAGE_GROUP_NAME"] = resource_name
@@ -52,7 +52,9 @@ def xgboost_model_package_group():
 
     # Delete the k8s resource if not already deleted by tests
     if k8s.get_resource_exists(reference):
-        _, deleted = k8s.delete_custom_resource(reference, 3, 10)
+        _, deleted = k8s.delete_custom_resource(
+            reference, cfg.DELETE_WAIT_PERIOD, cfg.DELETE_WAIT_LENGTH
+        )
         assert deleted
 
 
@@ -139,16 +141,19 @@ def test_create_model_package_group(self, xgboost_model_package_group):
         )
         model_package_group_arn = model_package_group_sm_desc["ModelPackageGroupArn"]
         assert k8s.get_resource_arn(resource) == model_package_group_arn
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(model_package_group_arn, resource_tags)
 
         self._assert_model_package_group_status_in_sync(
             model_package_group_name, reference, cfg.JOB_STATUS_COMPLETED
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(model_package_group_arn, resource_tags)
+
         # Check that you can delete a completed resource from k8s
-        _, deleted = k8s.delete_custom_resource(reference, 3, 10)
+        _, deleted = k8s.delete_custom_resource(
+            reference, cfg.DELETE_WAIT_PERIOD, cfg.DELETE_WAIT_LENGTH
+        )
         assert deleted is True
 
         assert get_sagemaker_model_package_group(model_package_group_name) is None

test/e2e/tests/test_monitoring_schedule.py

@@ -163,9 +163,6 @@ def test_smoke(
         monitoring_schedule_arn = monitoring_schedule_desc["MonitoringScheduleArn"]
         assert k8s.get_resource_arn(resource) == monitoring_schedule_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(monitoring_schedule_arn, resource_tags)
-
         # scheule transitions Pending -> Scheduled state
         # Pending status is shortlived only for 30 seconds because baselining job has already been run
         # remove the checks for Pending status if the test is flaky because of this
@@ -185,6 +182,9 @@ def test_smoke(
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(monitoring_schedule_arn, resource_tags)
+        
         # Update the resource
         new_cron_expression = "cron(0 * * * ? *)"
         spec["spec"]["monitoringScheduleConfig"]["scheduleConfig"][

test/e2e/tests/test_processingjob.py

@@ -161,9 +161,6 @@ def test_completed(self, kmeans_processing_job):
         processing_job_arn = processing_job_desc["ProcessingJobArn"]
         assert k8s.get_resource_arn(resource) == processing_job_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(processing_job_arn, resource_tags)
-
         assert processing_job_desc["ProcessingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
 
@@ -172,6 +169,9 @@ def test_completed(self, kmeans_processing_job):
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(processing_job_arn, resource_tags)
+
         # Check that you can delete a completed resource from k8s
         _, deleted = k8s.delete_custom_resource(reference, 3, 10)
         assert deleted is True

test/e2e/tests/test_trainingjob.py

@@ -161,9 +161,6 @@ def test_completed(self, xgboost_training_job):
         training_job_arn = training_job_desc["TrainingJobArn"]
         assert k8s.get_resource_arn(resource) == training_job_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(training_job_arn, resource_tags)
-
         assert training_job_desc["TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
 
@@ -172,6 +169,9 @@ def test_completed(self, xgboost_training_job):
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(training_job_arn, resource_tags)
+
         # Check that you can delete a completed resource from k8s
         _, deleted = k8s.delete_custom_resource(reference, 3, 10)
         assert deleted is True

test/e2e/tests/test_trainingjob_debugger.py

@@ -191,9 +191,6 @@ def test_completed(self, xgboost_training_job_debugger):
         training_job_arn = training_job_desc["TrainingJobArn"]
         assert k8s.get_resource_arn(resource) == training_job_arn
 
-        resource_tags = resource["spec"].get("tags", None)
-        assert_tags_in_sync(training_job_arn, resource_tags)
-
         assert training_job_desc["TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
 
@@ -208,6 +205,9 @@ def test_completed(self, xgboost_training_job_debugger):
         )
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
 
+        resource_tags = resource["spec"].get("tags", None)
+        assert_tags_in_sync(training_job_arn, resource_tags)
+
         # Check that you can delete a completed resource from k8s
         _, deleted = k8s.delete_custom_resource(reference, 3, 10)
         assert deleted is True