Implement per region instance type config for canary and e2e tests (#94)

* Implement per region instance type config for canary and e2e tests (#87)

Description of changes:

Currently, we are using one fixed instance type across all AWS regions in our endpoint and training job tests. 
However, certain regions do not support the currently specified instance type or require a limit increase to use that instance type.
Specifically, canary tests in the eu-west-3 and eu-north-1 regions are failing due to this issue. 

This pull request updates the testing resource config file `replacement_values.py` to pass in the correct instance type depending on region. 
Regions that did not experience this issue will continue to use the previous instance type via the new config to avoid breaking canaries/e2e testing in those regions.

The changes have been tested in our eu-west-3 and eu-north-1 canary stacks and have resulted in passing canaries.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

* enable adopted resource test in canaries

* inc delete wait for endpoint

Co-authored-by: Rahul Kharse <[email protected]>

Author: surajkota

test/canary/Dockerfile.canary

@@ -30,7 +30,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.6/b
  && cp ./kubectl /bin
 
 # Install eksctl
-RUN curl --silent --location "https://github.com/weaveworks/eksctl/releases/download/latest_release/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && mv /tmp/eksctl /bin
+RUN curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && mv /tmp/eksctl /bin
 
 # Install Helm 
 RUN curl -q -L "https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz" | tar zxf - -C /usr/local/bin/ \

test/e2e/common/fixtures.py

@@ -91,7 +91,7 @@ def xgboost_churn_endpoint(sagemaker_client):
     yield endpoint_spec
 
     for cr in (model_reference, endpoint_config_reference, endpoint_reference):
-        _, deleted = k8s.delete_custom_resource(cr, 3, 10)
+        _, deleted = k8s.delete_custom_resource(cr, cfg.DELETE_WAIT_PERIOD, cfg.DELETE_WAIT_LENGTH)
         assert deleted
 
 

test/e2e/replacement_values.py

@@ -166,6 +166,16 @@
     "eu-south-1": "638885417683.dkr.ecr.eu-south-1.amazonaws.com",
 }
 
+ENDPOINT_INSTANCE_TYPES = {
+    "eu-west-3": "ml.m5.large",
+    "eu-north-1": "ml.m5.large",
+}
+
+TRAINING_JOB_INSTANCE_TYPES = {
+    "eu-west-3": "ml.m5.xlarge",
+    "eu-north-1": "ml.m5.xlarge",
+}
+
 REPLACEMENT_VALUES = {
     "SAGEMAKER_DATA_BUCKET": get_bootstrap_resources().DataBucketName,
     "XGBOOST_IMAGE_URI": f"{XGBOOST_IMAGE_URIS[get_region()]}/sagemaker-xgboost:1.0-1-cpu-py3",
@@ -175,4 +185,6 @@
     "SAGEMAKER_EXECUTION_ROLE_ARN": get_bootstrap_resources().ExecutionRoleARN,
     "MODEL_MONITOR_ANALYZER_IMAGE_URI": f"{MODEL_MONITOR_IMAGE_URIS[get_region()]}/sagemaker-model-monitor-analyzer",
     "CLARIFY_IMAGE_URI": f"{CLARIFY_IMAGE_URIS[get_region()]}/sagemaker-clarify-processing:1.0",
+    "ENDPOINT_INSTANCE_TYPE": ENDPOINT_INSTANCE_TYPES.get(get_region(), 'ml.c5.large'),
+    "TRAINING_JOB_INSTANCE_TYPE": TRAINING_JOB_INSTANCE_TYPES.get(get_region(), 'ml.m4.xlarge')
 }

test/e2e/resources/endpoint_config_data_capture_single_variant.yaml

@@ -7,7 +7,7 @@ spec:
   productionVariants:
   - modelName: $MODEL_NAME
     variantName: AllTraffic
-    instanceType: ml.c5.large
+    instanceType: $ENDPOINT_INSTANCE_TYPE
     initialVariantWeight: 1
     initialInstanceCount: 1
   dataCaptureConfig:

test/e2e/resources/endpoint_config_multi_variant.yaml

@@ -9,12 +9,12 @@ spec:
       modelName: $MODEL_NAME
       initialInstanceCount: 1
       # This is the smallest instance type which will support scaling
-      instanceType: ml.c5.large
+      instanceType: $ENDPOINT_INSTANCE_TYPE
       initialVariantWeight: 1
     - variantName: variant-2
       modelName: $MODEL_NAME
       initialInstanceCount: 1
-      instanceType: ml.c5.large
+      instanceType: $ENDPOINT_INSTANCE_TYPE
       initialVariantWeight: 1
   tags:
     - key: confidentiality

test/e2e/resources/endpoint_config_single_variant.yaml

@@ -10,7 +10,7 @@ spec:
       # instanceCount is 2 to test retainAllVariantProperties
       initialInstanceCount: 2
       # This is the smallest instance type which will support scaling
-      instanceType: ml.c5.large
+      instanceType: $ENDPOINT_INSTANCE_TYPE
       initialVariantWeight: 1
   tags:
     - key: confidentiality

test/e2e/resources/xgboost_trainingjob.yaml

@@ -21,7 +21,7 @@ spec:
     s3OutputPath: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/training/output
   resourceConfig:
     instanceCount: 1
-    instanceType: ml.m4.xlarge
+    instanceType: $TRAINING_JOB_INSTANCE_TYPE
     volumeSizeInGB: 5
   stoppingCondition:
     maxRuntimeInSeconds: 86400

test/e2e/resources/xgboost_trainingjob_debugger.yaml

@@ -21,7 +21,7 @@ spec:
     s3OutputPath: s3://$SAGEMAKER_DATA_BUCKET/sagemaker/training/debugger/output
   resourceConfig:
     instanceCount: 1
-    instanceType: ml.m4.xlarge
+    instanceType: $TRAINING_JOB_INSTANCE_TYPE
     volumeSizeInGB: 5
   stoppingCondition:
     maxRuntimeInSeconds: 86400

test/e2e/tests/test_adopt_endpoint.py

@@ -67,7 +67,7 @@ def sdk_make_endpoint_config(model_name, endpoint_config_name):
                 "VariantName": "variant-1",
                 "ModelName": model_name,
                 "InitialInstanceCount": 1,
-                "InstanceType": "ml.c5.large",
+                "InstanceType": REPLACEMENT_VALUES["ENDPOINT_INSTANCE_TYPE"],
             }
         ],
     }
@@ -170,6 +170,7 @@ def adopted_endpoint(sdk_endpoint):
 
 
 @service_marker
+@pytest.mark.canary
 class TestAdoptedEndpoint:
     def test_smoke(self, sdk_endpoint, adopted_endpoint):
         (