HADOOP-19877: run s3a integration and scale tests in CI

.github/actions/build_image/action.yml

@@ -0,0 +1,62 @@
+name: Create hadoop build image
+
+inputs:
+  branch:
+    description: Git branch to use.
+    required: true
+  os:
+    description: Runner OS name
+    required: true
+  build_image_url:
+    description: URL of build (infra) image.
+    required: true
+
+outputs:
+  uid:
+    description: User ID this action ran as.
+
+runs:
+  using: composite
+  steps:
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ github.token }}
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+    - name: Build and push ${{ inputs.os }} base build image for ${{ inputs.branch }}
+      id: docker_build_base
+      uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+      with:
+        context: ./dev-support/docker/
+        file: ./dev-support/docker/Dockerfile_${{ inputs.os }}
+        push: true
+        tags: ${{ inputs.build_image_url }}-base
+        cache-from: type=registry,ref=ghcr.io/apache/hadoop/gha-build-${{ inputs.os }}-image-cache:${{ inputs.branch }}
+    - name: User-specific Dockerfile
+      shell: bash
+      run: |
+        USER_ID=$(id -u "${USER}")
+        GROUP_ID=$(id -g "${USER}")
+        cat > /tmp/Dockerfile.gha <<UserSpecificDocker
+        FROM ${{ inputs.build_image_url }}-base
+        RUN rm -f /var/log/faillog /var/log/lastlog
+        RUN groupadd --non-unique -g ${GROUP_ID} ${USER}
+        RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER}
+        RUN echo "${USER} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}"
+        UserSpecificDocker
+    - name: Build and push ${{ inputs.os }} build image for ${{ inputs.branch }}
+      id: docker_build_gha
+      uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+      with:
+        context: ./dev-support/docker/
+        file: /tmp/Dockerfile.gha
+        push: true
+        tags: ${{ inputs.build_image_url }}
+    - name: Set up Outputs
+      id: variables
+      shell: bash
+      run: |
+        echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT

.github/actions/build_image_url/action.yml

@@ -0,0 +1,27 @@
+name: Get Build Image URL
+description:
+
+inputs:
+  branch:
+    description: Git branch to use.
+    required: true
+  os:
+    description: Operating system to run the build on
+    required: true
+
+outputs:
+  build_image_url:
+    description: URL of build (infra) image.
+    value: ${{ steps.compute.outputs.image_url }}
+
+runs:
+  using: composite
+  steps:
+    - id: compute
+      shell: bash
+      run: |
+        # Convert to lowercase to meet Docker repo name requirement
+        REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+        echo "build_image_url=ghcr.io/${REPO_OWNER}/gha-build-${{ inputs.os }}:${{ inputs.branch }}-${{ github.run_id }}" \
+        >> $GITHUB_OUTPUT
+

.github/workflows/cloud_aws.yml

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Cloud - AWS integration tests"
+
+on:
+  # XXX TEMPORARY FOR TRIGGERING TEST
+  push:
+    branches:
+      - 'af/*'
+
+  pull_request:
+    paths:
+      - 'hadoop-tools/hadoop-aws/**'
+
+jobs:
+  run-aws-integration:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/tmpl_cloud_aws.yml
+    with:
+      java: 25
+      os: ubuntu_24

.github/workflows/tmpl_build_and_test.yml

@@ -71,6 +71,7 @@ env:
     -Dmaven.test.failure.ignore=false
 
 jobs:
+  # TODO: use common composite actions in .github/actions/
   precondition:
     name: Preparation
     runs-on: ubuntu-24.04

.github/workflows/tmpl_cloud_aws.yml

@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: AWS Integration
+
+on:
+  workflow_call:
+    inputs:
+      java:
+        required: false
+        type: string
+        default: 17
+      branch:
+        required: false
+        type: string
+        description: Branch to run the build against
+        default: trunk
+      os:
+        required: false
+        type: string
+        description: Operating system to run the build on
+        default: ubuntu_24
+
+# Security: Default to minimal permissions for workflow.
+permissions: {}
+
+concurrency:
+  group: >-
+    cloud-aws
+    ${{ github.workflow }}
+    ${{ github.repository == 'apache/hadoop' && github.run_id || github.ref }}
+    ${{ inputs.java }}
+    ${{ inputs.branch }}
+    ${{ inputs.os }}
+  cancel-in-progress: true
+
+env:
+  BUCKET_NAME: hadoop-ci
+
+jobs:
+  precondition:
+    runs-on: ubuntu-24.04
+    outputs:
+      build_image_url: ${{ steps.img.outputs.build_image_url }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/build_image_url
+        id: img
+        with:
+          branch: ${{ inputs.branch }}
+          os: ${{ inputs.os }}
+
+  build-image:
+    runs-on: ubuntu-24.04
+    needs: [ precondition ]
+    permissions:
+      packages: write
+    outputs:
+      uid: ${{ steps.build_img.outputs.uid }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/build_image
+        id: build_img
+        with:
+          branch: ${{ inputs.branch }}
+          os: ${{ inputs.os }}
+          build_image_url: ${{ needs.precondition.outputs.build_image_url }}
+
+  test:
+    name: S3A Integration Tests (Java ${{ inputs.java }})
+    needs: [ precondition, build-image ]
+    runs-on: ubuntu-24.04
+    container:
+      image: ${{ needs.precondition.outputs.build_image_url }}
+      options: --user ${{ needs.build-image.outputs.uid }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup JDK ${{ inputs.java }}
+        uses: actions/setup-java@v5
+        with:
+          distribution: zulu
+          java-version: ${{ inputs.java }}
+          cache: 'maven'
+      - name: Localstack setup
+        # Security: 0.3.2 is an immutable tag, but ASF Infra policy still
+        # requires full SHA hashes.
+        uses: LocalStack/setup-localstack@7c8a0cb3405bc58be4c8f763f812aa000bc46303  # 0.3.2
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+        env:
+          AWS_REGION: us-west-2
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_CI_KEY }}
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+
+      - name: Maven Build
+        shell: bash
+        run: ./mvnw install -B -DskipTests -am -pl hadoop-tools/hadoop-aws
+      - name: Create localstack s3 bucket
+        shell: bash
+        run: awslocal s3 mb s3://${{ env.BUCKET_NAME }}
+
+      - name: S3A Integration Tests
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+          AWS_REGION: us-west-2
+        run: |
+          ./mvnw verify -B -pl hadoop-tools/hadoop-aws \
+            -DskipITs=false \
+            -Dtest=none \
+            -Dscale=true \
+            -Dfs.s3a.access.key=test \
+            -Dfs.s3a.secret.key=test \
+            -Dfs.s3a.endpoint=http://localhost:4566 \
+            -Dfs.s3a.connection.ssl.enabled=false \
+            -Dfs.s3a.path.style.access=true \
+            -Dfs.s3a.endpoint.region=us-west-2 \
+            -Dtest.fs.s3a.name=s3a://${{ env.BUCKET_NAME }}/