diff --git a/.github/actions/build_image/action.yml b/.github/actions/build_image/action.yml new file mode 100644 index 00000000000000..538df067705889 --- /dev/null +++ b/.github/actions/build_image/action.yml @@ -0,0 +1,62 @@ +name: Create hadoop build image + +inputs: + branch: + description: Git branch to use. + required: true + os: + description: Runner OS name + required: true + build_image_url: + description: URL of build (infra) image. + required: true + +outputs: + uid: + description: User ID this action ran as. + +runs: + using: composite + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + - name: Build and push ${{ inputs.os }} base build image for ${{ inputs.branch }} + id: docker_build_base + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + with: + context: ./dev-support/docker/ + file: ./dev-support/docker/Dockerfile_${{ inputs.os }} + push: true + tags: ${{ inputs.build_image_url }}-base + cache-from: type=registry,ref=ghcr.io/apache/hadoop/gha-build-${{ inputs.os }}-image-cache:${{ inputs.branch }} + - name: User-specific Dockerfile + shell: bash + run: | + USER_ID=$(id -u "${USER}") + GROUP_ID=$(id -g "${USER}") + cat > /tmp/Dockerfile.gha < "/etc/sudoers.d/hadoop-build-${USER_ID}" + UserSpecificDocker + - name: Build and push ${{ inputs.os }} build image for ${{ inputs.branch }} + id: docker_build_gha + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + with: + context: ./dev-support/docker/ + file: /tmp/Dockerfile.gha + push: true + tags: ${{ inputs.build_image_url }} + - name: Set up Outputs + id: variables + shell: bash + run: | + echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT diff --git a/.github/actions/build_image_url/action.yml b/.github/actions/build_image_url/action.yml new file mode 100644 index 00000000000000..4c870d77697904 --- /dev/null +++ b/.github/actions/build_image_url/action.yml @@ -0,0 +1,27 @@ +name: Get Build Image URL +description: + +inputs: + branch: + description: Git branch to use. + required: true + os: + description: Operating system to run the build on + required: true + +outputs: + build_image_url: + description: URL of build (infra) image. + value: ${{ steps.compute.outputs.image_url }} + +runs: + using: composite + steps: + - id: compute + shell: bash + run: | + # Convert to lowercase to meet Docker repo name requirement + REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + echo "build_image_url=ghcr.io/${REPO_OWNER}/gha-build-${{ inputs.os }}:${{ inputs.branch }}-${{ github.run_id }}" \ + >> $GITHUB_OUTPUT + diff --git a/.github/workflows/cloud_aws.yml b/.github/workflows/cloud_aws.yml new file mode 100644 index 00000000000000..9f3bd6371fca12 --- /dev/null +++ b/.github/workflows/cloud_aws.yml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Cloud - AWS integration tests" + +on: + # XXX TEMPORARY FOR TRIGGERING TEST + push: + branches: + - 'af/*' + + pull_request: + paths: + - 'hadoop-tools/hadoop-aws/**' + +jobs: + run-aws-integration: + permissions: + packages: write + name: Run + uses: ./.github/workflows/tmpl_cloud_aws.yml + with: + java: 25 + os: ubuntu_24 diff --git a/.github/workflows/tmpl_build_and_test.yml b/.github/workflows/tmpl_build_and_test.yml index 1062c3e3f3d657..373a79a6a998f7 100644 --- a/.github/workflows/tmpl_build_and_test.yml +++ b/.github/workflows/tmpl_build_and_test.yml @@ -71,6 +71,7 @@ env: -Dmaven.test.failure.ignore=false jobs: + # TODO: use common composite actions in .github/actions/ precondition: name: Preparation runs-on: ubuntu-24.04 diff --git a/.github/workflows/tmpl_cloud_aws.yml b/.github/workflows/tmpl_cloud_aws.yml new file mode 100644 index 00000000000000..440aed08e9ba97 --- /dev/null +++ b/.github/workflows/tmpl_cloud_aws.yml @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: AWS Integration + +on: + workflow_call: + inputs: + java: + required: false + type: string + default: 17 + branch: + required: false + type: string + description: Branch to run the build against + default: trunk + os: + required: false + type: string + description: Operating system to run the build on + default: ubuntu_24 + +# Security: Default to minimal permissions for workflow. +permissions: {} + +concurrency: + group: >- + cloud-aws + ${{ github.workflow }} + ${{ github.repository == 'apache/hadoop' && github.run_id || github.ref }} + ${{ inputs.java }} + ${{ inputs.branch }} + ${{ inputs.os }} + cancel-in-progress: true + +env: + BUCKET_NAME: hadoop-ci + +jobs: + precondition: + runs-on: ubuntu-24.04 + outputs: + build_image_url: ${{ steps.img.outputs.build_image_url }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/build_image_url + id: img + with: + branch: ${{ inputs.branch }} + os: ${{ inputs.os }} + + build-image: + runs-on: ubuntu-24.04 + needs: [ precondition ] + permissions: + packages: write + outputs: + uid: ${{ steps.build_img.outputs.uid }} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/build_image + id: build_img + with: + branch: ${{ inputs.branch }} + os: ${{ inputs.os }} + build_image_url: ${{ needs.precondition.outputs.build_image_url }} + + test: + name: S3A Integration Tests (Java ${{ inputs.java }}) + needs: [ precondition, build-image ] + runs-on: ubuntu-24.04 + container: + image: ${{ needs.precondition.outputs.build_image_url }} + options: --user ${{ needs.build-image.outputs.uid }} + steps: + - uses: actions/checkout@v6 + - name: Setup JDK ${{ inputs.java }} + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: ${{ inputs.java }} + cache: 'maven' + - name: Localstack setup + # Security: 0.3.2 is an immutable tag, but ASF Infra policy still + # requires full SHA hashes. + uses: LocalStack/setup-localstack@7c8a0cb3405bc58be4c8f763f812aa000bc46303 # 0.3.2 + with: + image-tag: 'latest' + install-awslocal: 'true' + env: + AWS_REGION: us-west-2 + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_CI_KEY }} + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + + - name: Maven Build + shell: bash + run: ./mvnw install -B -DskipTests -am -pl hadoop-tools/hadoop-aws + - name: Create localstack s3 bucket + shell: bash + run: awslocal s3 mb s3://${{ env.BUCKET_NAME }} + + - name: S3A Integration Tests + shell: bash + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_REGION: us-west-2 + run: | + ./mvnw verify -B -pl hadoop-tools/hadoop-aws \ + -DskipITs=false \ + -Dtest=none \ + -Dscale=true \ + -Dfs.s3a.access.key=test \ + -Dfs.s3a.secret.key=test \ + -Dfs.s3a.endpoint=http://localhost:4566 \ + -Dfs.s3a.connection.ssl.enabled=false \ + -Dfs.s3a.path.style.access=true \ + -Dfs.s3a.endpoint.region=us-west-2 \ + -Dtest.fs.s3a.name=s3a://${{ env.BUCKET_NAME }}/ diff --git a/hadoop-tools/hadoop-aws/src/test/resources/XXX_REMOVE_ME b/hadoop-tools/hadoop-aws/src/test/resources/XXX_REMOVE_ME new file mode 100644 index 00000000000000..e69de29bb2d1d6