feat(snowflake-driver): support azure exports buckets #2274
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Drivers tests' | |
| on: | |
| push: | |
| branches: | |
| - 'master' | |
| paths: | |
| - '.github/workflows/drivers-tests.yml' | |
| - 'packages/cubejs-testing-drivers/**' | |
| - 'packages/cubejs-testing-shared/**' | |
| - 'packages/cubejs-query-orchestrator/src/**' | |
| - 'packages/cubejs-backend-shared/**' | |
| - 'packages/cubejs-server-core/**' | |
| - 'packages/cubejs-schema-compiler/**' | |
| - 'packages/cubejs-base-driver/src/**' | |
| - 'packages/cubejs-jdbc-driver/src/**' | |
| - 'packages/cubejs-athena-driver/**' | |
| - 'packages/cubejs-bigquery-driver/**' | |
| - 'packages/cubejs-clickhouse-driver/**' | |
| - 'packages/cubejs-databricks-jdbc-driver/**' | |
| - 'packages/cubejs-mssql-driver/**' | |
| - 'packages/cubejs-mysql-driver/**' | |
| - 'packages/cubejs-postgres-driver/**' | |
| - 'packages/cubejs-snowflake-driver/**' | |
| # To test SQL API Push down | |
| - 'packages/cubejs-backend-native/**' | |
| - 'rust/cubesql/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/drivers-tests.yml' | |
| - 'packages/cubejs-testing-drivers/**' | |
| - 'packages/cubejs-testing-shared/**' | |
| - 'packages/cubejs-query-orchestrator/src/**' | |
| - 'packages/cubejs-backend-shared/**' | |
| - 'packages/cubejs-server-core/**' | |
| - 'packages/cubejs-schema-compiler/**' | |
| - 'packages/cubejs-base-driver/src/**' | |
| - 'packages/cubejs-jdbc-driver/src/**' | |
| - 'packages/cubejs-athena-driver/**' | |
| - 'packages/cubejs-bigquery-driver/**' | |
| - 'packages/cubejs-clickhouse-driver/**' | |
| - 'packages/cubejs-databricks-jdbc-driver/**' | |
| - 'packages/cubejs-mssql-driver/**' | |
| - 'packages/cubejs-mysql-driver/**' | |
| - 'packages/cubejs-postgres-driver/**' | |
| - 'packages/cubejs-snowflake-driver/**' | |
| # To test SQL API Push down | |
| - 'packages/cubejs-backend-native/**' | |
| - 'rust/cubesql/**' | |
| jobs: | |
| latest-tag-sha: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| sha: ${{ steps.get-tag.outputs.sha }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - id: git-log | |
| run: git log HEAD~30..HEAD | |
| - id: get-tag-test | |
| run: echo "$SHA $(git rev-list -n 1 "$(git tag --contains "$SHA")")" | |
| env: | |
| SHA: ${{ github.sha }} | |
| - id: get-tag | |
| run: echo "sha=$(git rev-list -n 1 "$(git tag --contains "$SHA")")" >> "$GITHUB_OUTPUT" | |
| env: | |
| SHA: ${{ github.sha }} | |
| - id: get-tag-out | |
| run: echo "$OUT" | |
| env: | |
| OUT: ${{ steps.get-tag.outputs.sha }} | |
| native_linux: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| name: Build native Linux ${{ matrix.node-version }} ${{ matrix.target }} Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| node-version: [ 20 ] | |
| python-version: [ "fallback" ] | |
| target: [ "x86_64-unknown-linux-gnu" ] | |
| fail-fast: false | |
| container: | |
| image: cubejs/rust-cross:${{ matrix.target }}-15082024 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| target: ${{ matrix.target }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Install Yarn | |
| run: npm install -g yarn | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Install cargo-cp-artifact | |
| run: npm install -g [email protected] | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./packages/cubejs-backend-native | |
| key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu | |
| shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| run: cd packages/cubejs-backend-native && npm run native:build-release | |
| - name: Setup cross compilation | |
| if: (matrix.target == 'aarch64-unknown-linux-gnu') | |
| uses: allenevans/[email protected] | |
| with: | |
| PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| run: cd packages/cubejs-backend-native && npm run native:build-release-python | |
| - name: Upload native build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backend-native | |
| path: packages/cubejs-backend-native/index.node | |
| build: | |
| needs: [latest-tag-sha, native_linux] | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| # Building docker | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Download native build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: backend-native | |
| path: packages/cubejs-backend-native/ | |
| - name: Build and push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./packages/cubejs-docker/testing-drivers.Dockerfile | |
| tags: cubejs/cube:testing-drivers | |
| push: true | |
| tests: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 30 | |
| needs: [latest-tag-sha, build] | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| strategy: | |
| matrix: | |
| node: | |
| - 20.x | |
| database: | |
| - athena | |
| - bigquery | |
| - clickhouse | |
| - databricks-jdbc | |
| - databricks-jdbc-export-bucket | |
| - mssql | |
| - mysql | |
| - postgres | |
| - snowflake | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Node.js 20.x | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20.x | |
| - name: Configure `yarn` | |
| run: yarn policies set-version v1.22.22 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| # We don't want to save it on finish, restore only! | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Install dependencies | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Build client | |
| run: yarn build | |
| - name: Build packages | |
| run: yarn tsc | |
| - name: Build tests | |
| run: | | |
| cd packages/cubejs-testing-drivers | |
| yarn tsc | |
| - name: Run tests | |
| env: | |
| # Athena | |
| DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }} | |
| DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_SECRET }} | |
| # BigQuery | |
| DRIVERS_TESTS_CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_BQ_CREDENTIALS }} | |
| # Databricks | |
| DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_URL: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_URL }} | |
| DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN }} | |
| DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_KEY }} | |
| DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET }} | |
| # Snowflake | |
| DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }} | |
| DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }} | |
| run: | | |
| cd ./packages/cubejs-testing-drivers | |
| export DEBUG=testcontainers | |
| yarn ${{ matrix.database }}-full |