CLOUDP-339241 - Improve e2e on arm

.evergreen-functions.yml

@@ -527,6 +527,7 @@ functions:
         binary: scripts/release/pipeline.sh
         env:
           IMAGE_NAME: ${IMAGE_NAME}
+          FLAGS: ${FLAGS}
 
   pipeline_agent:
     - command: subprocess.exec

.evergreen.yml

@@ -346,6 +346,17 @@ tasks:
         vars:
           IMAGE_NAME: meko-tests
 
+  - name: build_test_image_arm
+    commands:
+      - func: clone
+      - func: setup_building_host
+      - func: build_multi_cluster_binary
+      - func: pipeline
+        vars:
+          IMAGE_NAME: meko-tests
+          FLAGS: "--platform linux/arm64 --architecture-suffix"
+
+
   - name: build_test_image_ibm
     commands:
       - func: clone
@@ -1378,10 +1389,6 @@ buildvariants:
         variant: init_test_run
       - name: build_database_image_ubi
         variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_power
     tasks:
@@ -1404,10 +1411,6 @@ buildvariants:
         variant: init_test_run
       - name: build_database_image_ubi
         variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_power
     tasks:
@@ -1430,10 +1433,6 @@ buildvariants:
         variant: init_test_run
       - name: build_database_image_ubi
         variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_z
     tasks:
@@ -1454,12 +1453,6 @@ buildvariants:
         variant: init_test_run
       - name: build_init_database_image_ubi
         variant: init_test_run
-      - name: build_database_image_ubi
-        variant: init_test_run
-      - name: build_init_appdb_images_ubi
-        variant: init_test_run
-      - name: build_init_om_images_ubi
-        variant: init_test_run
       - name: build_test_image_ibm
         variant: init_test_run_ibm_z
     tasks:
@@ -1471,10 +1464,18 @@ buildvariants:
     run_on:
       - ubuntu2404-arm64-large
     allowed_requesters: [ "patch", "commit" ]
+    depends_on:
+      - name: build_operator_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image_arm
+        variant: init_test_run_arm
     # TODO: Re-enable when staging is added to pipeline
     # https://jira.mongodb.org/browse/CLOUDP-349096
     disable: true
-    <<: *base_no_om_image_dependency
     tasks:
       - name: e2e_smoke_arm_task_group
 
@@ -1484,10 +1485,16 @@ buildvariants:
     run_on:
       - ubuntu2404-arm64-large
     allowed_requesters: [ "patch", "commit" ]
+    depends_on:
+      - name: build_operator_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image_arm
+        variant: init_test_run_arm
     # TODO: Re-enable when staging is added to pipeline
     # https://jira.mongodb.org/browse/CLOUDP-349096
     disable: true
-    <<: *base_no_om_image_dependency
     tasks:
       - name: e2e_smoke_arm_task_group
 
@@ -1719,6 +1726,15 @@ buildvariants:
     tasks:
       - name: build_test_image_ibm
 
+  - name: init_test_run_arm
+    display_name: init_test_run_arm
+    max_hosts: -1
+    tags: [ "staging" ]
+    run_on:
+      - ubuntu2204-arm64-small
+    tasks:
+      - name: build_test_image_arm
+
   - name: run_pre_commit
     priority: 70
     display_name: run_pre_commit

build_info.json

@@ -175,7 +175,6 @@
       "staging": {
         "repositories": ["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"],
         "platforms": [
-          "linux/arm64",
           "linux/amd64"
         ]
       },

scripts/dev/setup_evg_host.sh

@@ -47,6 +47,7 @@ download_kind() {
 download_kubectl() {
   kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
   echo "Downloading kubectl ${kubectl_version}..."
+  kubectl_version=$(echo "${kubectl_version}" | tail -n1 | tr -d '\n')
 
   curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
   chmod +x kubectl

scripts/evergreen/e2e/single_e2e.sh

@@ -28,14 +28,17 @@ deploy_test_app() {
     arch=$(uname -m)
 
     case "${arch}" in
+        aarch64|arm64)
+            meko_tests_version="${meko_tests_version}-arm64"
+            ;;
         ppc64le)
             meko_tests_version="${meko_tests_version}-ppc64le"
             ;;
         s390x)
             meko_tests_version="${meko_tests_version}-s390x"
             ;;
         *)
-            echo "Not IBM host, using default meko_tests_version"
+            echo "amd64 host, using default meko_tests_version"
             ;;
     esac
 

scripts/evergreen/setup_kubectl.sh

@@ -14,6 +14,7 @@ mkdir -p "${bindir}" "${tmpdir}"
 
 kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
 echo "Downloading kubectl ${kubectl_version} for ${ARCH}"
+kubectl_version=$(echo "${kubectl_version}" | tail -n1 | tr -d '\n')
 
 curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
 chmod +x kubectl

scripts/release/atomic_pipeline.py

@@ -103,6 +103,7 @@ def build_image(
         args=build_args,
         push=True,
         platforms=build_configuration.platforms,
+        architecture_suffix=build_configuration.architecture_suffix,
     )
 
     if build_configuration.sign:

scripts/release/build/build_info.py

@@ -27,6 +27,7 @@ class ImageInfo:
     latest_tag: bool = False
     olm_tag: bool = False
     skip_if_exists: bool = False
+    architecture_suffix: bool = False
 
 
 @dataclass
@@ -80,6 +81,7 @@ def load_build_info(scenario: BuildScenario) -> BuildInfo:
             latest_tag=scenario_data.get("latest-tag", False),
             olm_tag=scenario_data.get("olm-tag", False),
             skip_if_exists=scenario_data.get("skip-if-exists", False),
+            architecture_suffix=scenario_data.get("architecture_suffix", False),
         )
 
     binaries = {}

scripts/release/build/image_build_configuration.py

@@ -23,6 +23,7 @@ class ImageBuildConfiguration:
     parallel_factor: int = 0
     all_agents: bool = False
     currently_used_agents: bool = False
+    architecture_suffix: bool = False
 
     def is_release_scenario(self) -> bool:
         return self.scenario == BuildScenario.RELEASE

scripts/release/build/image_build_process.py

@@ -94,6 +94,7 @@ def execute_docker_build(
     Dict[str, str],
     push: bool,
     platforms: list[str],
+    architecture_suffix: bool = False,
     builder_name: str = DEFAULT_BUILDER_NAME,
 ):
     """
@@ -105,6 +106,7 @@ def execute_docker_build(
     :param args: Build arguments dictionary
     :param push: Whether to push the image after building
     :param platforms: List of target platforms (e.g., ["linux/amd64", "linux/arm64"])
+    :param architecture_suffix: Whether to add the architecture of the image as a suffix to the tag
     :param builder_name: Name of the buildx builder to use
     """
     # Login to ECR before building
@@ -126,6 +128,11 @@ def execute_docker_build(
         # Use buildx for multi-platform builds
         if len(platforms) > 1:
             logger.info(f"Multi-platform build for {len(platforms)} architectures")
+        elif architecture_suffix and len(platforms) == 1:
+            arch = platforms[0].split("/")[1]
+            tags = [f"{tag}-{arch}" for tag in tags]
+            logger.info(f"Using architecture suffix '{arch}' for tags: {tags}")
+
 
         # Build the image using buildx, builder must be already initialized
         docker_cmd.buildx.build(

scripts/release/pipeline.py

@@ -119,6 +119,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration:
     sign = args.sign or image_build_info.sign
     dockerfile_path = image_build_info.dockerfile_path
     skip_if_exists = image_build_info.skip_if_exists
+    architecture_suffix = args.architecture_suffix or image_build_info.architecture_suffix
 
     # Validate version - only agent can have None version as the versions are managed by the agent
     # which are externally retrieved from release.json
@@ -139,6 +140,7 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration:
         parallel_factor=args.parallel_factor,
         all_agents=args.all_agents,
         currently_used_agents=args.current_agents,
+        architecture_suffix=architecture_suffix,
     )
 
 
@@ -274,6 +276,11 @@ def main():
         action="store_true",
         help="Build all currently used agent images.",
     )
+    parser.add_argument(
+        "--architecture-suffix",
+        action="store_true",
+        help="Append architecture suffix to image tags for single platform builds",
+    )
 
     args = parser.parse_args()
 

scripts/release/pipeline.sh

@@ -4,4 +4,4 @@ set -Eeou pipefail
 
 source scripts/dev/set_env_context.sh
 
-scripts/dev/run_python.sh scripts/release/pipeline.py "${IMAGE_NAME}" --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}"
+scripts/dev/run_python.sh scripts/release/pipeline.py "${IMAGE_NAME}" --build-scenario "${BUILD_SCENARIO}" --version "${OPERATOR_VERSION}" ${FLAGS:+${FLAGS}}

scripts/release/tests/build_info_test.py

@@ -229,7 +229,7 @@ def test_load_build_info_staging():
             ),
             "meko-tests": ImageInfo(
                 repositories=["268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes-tests"],
-                platforms=["linux/arm64", "linux/amd64"],
+                platforms=["linux/amd64"],
                 dockerfile_path="docker/mongodb-kubernetes-tests/Dockerfile",
             ),
             "readiness-probe": ImageInfo(