Data Source Integration Test for pulling External Data #59
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Source Integration Test for pulling External Data | |
| on: | |
| schedule: | |
| - cron: "43 20 * * 1,3" # UTC time | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| SERVICE_NAME: "rslearn_projects" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| ghcr_docker_image: ${{ steps.image-names.outputs.ghcr_image_name }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=sha,format=long | |
| type=sha,format=short | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| - name: Build and push Docker image | |
| id: build-push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Set Environment Variables | |
| run: | | |
| echo "TEST_BUCKET=test-bucket" >> $GITHUB_ENV | |
| echo "TEST_PREFIX=tests/" | |
| - name: Store Image Names | |
| # We need the docker image name downstream in test & deploy. This saves the full docker image names to outputs | |
| id: image-names | |
| run: |- | |
| GHCR_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}" | |
| GHCR_IMAGE=`echo ${GHCR_IMAGE} | tr '[:upper:]' '[:lower:]'` # docker requires that all image names be lowercase | |
| echo "ghcr.io Docker image name is ${GHCR_IMAGE}" | |
| echo "ghcr_image_name=\"${GHCR_IMAGE}\"" >> $GITHUB_OUTPUT | |
| test: | |
| needs: build | |
| runs-on: ubuntu-latest-m | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set Environment Variables | |
| run: | | |
| echo "TEST_BUCKET=test-bucket-rslearn" >> $GITHUB_ENV | |
| echo "GCSFS_DEFAULT_PROJECT=skylight-proto-1" >> $GITHUB_ENV | |
| - name: Authenticate into gcp | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| credentials_json: ${{secrets.GOOGLE_CREDENTIALS }} | |
| - name: Run Data Sources Integration Tests | |
| run: | | |
| docker run --rm \ | |
| -e TEST_BUCKET="${{ env.TEST_BUCKET }}" \ | |
| -v ${{env.GOOGLE_GHA_CREDS_PATH}}:/tmp/gcp-credentials.json:ro \ | |
| -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json \ | |
| -e GCSFS_DEFAULT_PROJECT="${GCSFS_DEFAULT_PROJECT}" \ | |
| -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ | |
| -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ | |
| -e AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" \ | |
| -e TEST_SERVICE_ACCOUNT_NAME="${TEST_SERVICE_ACCOUNT_NAME}" \ | |
| -e TEST_XYZ_TILES_TEMPLATE="${TEST_XYZ_TILES_TEMPLATE}" \ | |
| -e TEST_USGS_LANDSAT_USERNAME="${TEST_USGS_LANDSAT_USERNAME}" \ | |
| -e TEST_USGS_LANDSAT_TOKEN="${TEST_USGS_LANDSAT_TOKEN}" \ | |
| -e PL_API_KEY="${PL_API_KEY}" \ | |
| -e CDSAPI_KEY="${CDSAPI_KEY}" \ | |
| -e NASA_EARTHDATA_USERNAME="${NASA_EARTHDATA_USERNAME}" \ | |
| -e NASA_EARTHDATA_PASSWORD="${NASA_EARTHDATA_PASSWORD}" \ | |
| -e CI="true" \ | |
| --network host \ | |
| ${{ needs.build.outputs.ghcr_docker_image }} \ | |
| pytest tests/integration/data_sources/ | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| TEST_SERVICE_ACCOUNT_NAME: ${{ secrets.TEST_SERVICE_ACCOUNT_NAME }} | |
| TEST_XYZ_TILES_TEMPLATE: ${{ secrets.TEST_XYZ_TILES_TEMPLATE }} | |
| TEST_USGS_LANDSAT_USERNAME: ${{ secrets.TEST_USGS_LANDSAT_USERNAME }} | |
| TEST_USGS_LANDSAT_TOKEN: ${{ secrets.TEST_USGS_LANDSAT_TOKEN }} | |
| PL_API_KEY: ${{ secrets.PL_API_KEY }} | |
| CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} | |
| NASA_EARTHDATA_USERNAME: ${{ secrets.NASA_EARTHDATA_USERNAME }} | |
| NASA_EARTHDATA_PASSWORD: ${{ secrets.NASA_EARTHDATA_PASSWORD }} | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| docker image prune -f |