From 3f21fe8bbf18f41dcc968000dfca7092e18e37bf Mon Sep 17 00:00:00 2001 From: James Guthrie Date: Mon, 5 Dec 2022 13:18:12 +0100 Subject: [PATCH 1/2] Fix ci test workflow All matrix builds were using the same cache key, causing collisions. pgx puts the postgres data directory in `target/pgx-test-data-`, which the `Swatinem/rust-cache` action cleans, by removing all its content, and subsequently caches. This breaks pgx tests, which do not expect the directory to be empty. The combination of the cache collisions and incorrect caching caused a number of build failures, which could only be fixed by removing the cached artifacts from the GitHub cache. --- .github/workflows/ci.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 29b955d6..08b368d2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,6 +29,8 @@ jobs: sccache --show-stats - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.postgres.version }} - name: Install cargo-pgx run: | @@ -44,10 +46,18 @@ jobs: - name: Initialize pgx if: ${{ steps.cache-pgx.outputs.cache-hit != 'true' }} run: cargo pgx init --pg${{ matrix.postgres.version }} download - + - name: Run cargo test run: cargo pgx test pg${{ matrix.postgres.version }} + # Note: pgx puts the postgres test configuration data in target/pgx-test-data- + # Swatinem/rust-cache "cleans" and then caches this directory. This + # "cleaning" breaks pgx when the cache is restored. By removing the + # directory, we prevent it from being incorrectly cached. + - name: Remove pgx-test-data directory + run: | + rm -rf target/pgx-test-data* + - name: Show sccache stats run: sccache --show-stats From fe550b7f818ae8ff481c60a84fcf49c395f786af Mon Sep 17 00:00:00 2001 From: James Guthrie Date: Fri, 2 Dec 2022 11:38:15 +0100 Subject: [PATCH 2/2] Build arm64 packages This commit adds support to build arm64 packages. The steps taken are somewhat convoluted. As GitHub doesn't offer hosted arm64 runners, we have a self-hosted runner. For security reasons, GitHub doesn't allow self-hosted runners to be used with public repositories. For that purpose, we have created the `promscale_extension_private` repository, which is a private clone of the `promscale_extension` repository. The `release` GitHub workflow is designed to be run in _both_ the public and private repository. In the public repository its behaviour is unchanged. In the private repository, it uses our self-hosted arm64 runners and attempts to attach the packages to the draft release which was created in the `promscale_extension` repository. --- .github/workflows/docker.yaml | 4 +++- .github/workflows/release.yaml | 42 +++++++++++++++++++++++++++++----- RELEASING.md | 6 +++-- dist/deb.dockerfile | 4 +++- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 3bc6e122..7e4067d5 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -12,6 +12,7 @@ on: jobs: docker: runs-on: ubuntu-latest + if: github.repository == 'timescale/promscale_extension' outputs: branch_name: ${{steps.metadata.outputs.branch_name}} image_branch_name: ${{steps.metadata.outputs.image_branch_name}} @@ -125,6 +126,7 @@ jobs: pick-connector-branch: name: picking connector's repo branch + if: github.repository == 'timescale/promscale_extension' runs-on: ubuntu-latest needs: - docker @@ -165,7 +167,7 @@ jobs: # otherwise we have to keep updating them as we add or remove postgres versions etc. docker-result: name: docker result - if: always() + if: always() && github.repository == 'timescale/promscale_extension' needs: - docker - call-connector-e2e diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 648ae117..bdb63c81 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,15 +15,19 @@ on: jobs: package: - runs-on: ubuntu-latest + runs-on: ${{ matrix.arch.runner }} env: RUST_VERSION: "1.64.0" strategy: fail-fast: false matrix: arch: + - cpu: aarch64 + platform: arm64 + runner: [self-hosted, Linux, ARM64] - cpu: x86_64 platform: amd64 + runner: ubuntu-latest os: - name: linux distro: debian @@ -71,7 +75,15 @@ jobs: - version: "12" - version: "13" - version: "14" - + exclude: + # The centos build doesn't work on arm64 + - os: + distro: centos + arch: + platform: arm64 + # exclude arm64 on the promscale_extension repo, and exclude amd64 in the promscale_extension_private repo + - arch: + platform: ${{ github.repository == 'timescale/promscale_extension' && 'arm64' || 'amd64' }} steps: - name: Install PackageCloud if: ${{ startsWith(github.ref, 'refs/tags') }} @@ -80,6 +92,9 @@ jobs: - uses: actions/checkout@v3 + - name: Install rust + uses: dtolnay/rust-toolchain@1.64.0 + - name: Gather Metadata id: metadata run: | @@ -106,11 +121,15 @@ jobs: echo "ghopts=${ghopts}" >> ${GITHUB_OUTPUT} echo "image=promscale-extension:${tag}.pg${{ matrix.postgres.version }}.${{ matrix.os.distro }}${{ matrix.os.version }}" >> ${GITHUB_OUTPUT} - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 + # This is necessary on self-hosted runners + - name: Create build context + run: | + docker context create builder - name: Setup Docker Buildx uses: docker/setup-buildx-action@v2 + with: + endpoint: builder - name: Build Package uses: docker/build-push-action@v3 @@ -143,6 +162,8 @@ jobs: ls -la artifacts/* - name: Test package + # We can't test the package on arm64 because there is no timescaledb install for arm64 + if: matrix.arch.platform != 'arm64' env: DISTRO: ${{ matrix.os.distro }} DISTRO_VERSION: ${{ matrix.os.version }} @@ -226,8 +247,17 @@ jobs: cat dist/RELEASE_NOTES.md - name: Create Release - if: ${{ startsWith(github.ref, 'refs/tags') }} + if: ${{ startsWith(github.ref, 'refs/tags') && github.repository == 'timescale/promscale_extension' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create --repo timescale/promscale_extension --target ${GITHUB_SHA} ${{ needs.package.outputs.ghopts }} --notes-file dist/RELEASE_NOTES.md ${{ needs.package.outputs.version }} artifacts/* + + # Note: This assumes that the arm64 build reaches this point after the + # above `gh release create` command in the amd64 build + - name: Attach arm64 artifacts to release + if: ${{ startsWith(github.ref, 'refs/tags') && github.repository == 'timescale/promscale_extension_private' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release create -R timescale/promscale_extension --target ${GITHUB_SHA} ${{ needs.package.outputs.ghopts }} --notes-file dist/RELEASE_NOTES.md ${{ needs.package.outputs.version }} artifacts/* + gh release upload --repo timescale/promscale_extension ${{ needs.package.outputs.version }} artifacts/* diff --git a/RELEASING.md b/RELEASING.md index 6271c7f2..451410cc 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -19,10 +19,12 @@ Create a new issue titled "Release ``". Copy everyt ## Release - [ ] Create a tag on the master branch `git tag ` - [ ] Push the tag `git push origin ` -- [ ] CI will trigger and create a draft release with assets attached. +- [ ] CI will trigger the `release` job, which (when it completes) will create a draft release with assets attached. +- [ ] Push the tag to the `promscale_extension_private` repository `git push private `. +- [ ] Validate that the arm64 package build is triggered in the `promscale_extension_private` repo. - [ ] Prepare extension release notes, share with team asking for feedback. +- [ ] Wait for CI to generate the packages, validate that _both_ `x86_64` and `aarch64` packages are present. - [ ] Attach release notes to draft release created above. -- [ ] Wait for CI to generate the packages files - [ ] Create a PR to update the HA image in [timescaledb-docker-ha](https://github.com/timescale/timescaledb-docker-ha). [EXAMPLE](https://github.com/timescale/timescaledb-docker-ha/pull/285/files) - [ ] In the timescaledb-docker-ha repo, update `TIMESCALE_PROMSCALE_EXTENSIONS` in the `Makefile` to include the version just released - [ ] Update the CHANGELOG entry in the timescaledb-docker-ha repo and wait for the CI to complete and request review from the Cloud team and merge it when approved. diff --git a/dist/deb.dockerfile b/dist/deb.dockerfile index 03db8809..62832c0d 100644 --- a/dist/deb.dockerfile +++ b/dist/deb.dockerfile @@ -36,7 +36,9 @@ apt-get install -y \ git \ lintian \ pkg-config \ - rubygems + rubygems \ + clang \ + libclang-dev # Install FPM gem install fpm