From 9ccded514ab8dbc3b2d0453f71e6d159d872427c Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 21 Dec 2025 00:08:00 -0800 Subject: [PATCH 1/3] ci: fix release script --- .github/workflows/java-publish.yml | 21 +++++++++++---- .github/workflows/python-publish.yml | 14 +++++++--- .github/workflows/release.yml | 22 +++++++++++----- .github/workflows/rust-publish.yml | 14 +++++++--- ci/calculate_version.py | 5 ++-- docs/src/client/operations/index.md | 39 ++++++++++++++-------------- 6 files changed, 76 insertions(+), 39 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 54813b1d..3ad1d997 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -22,13 +22,17 @@ on: workflow_dispatch: inputs: mode: - description: "dry_run: build & package only, release: build & deploy to OSSRH" + description: 'Release mode' required: true - default: "dry_run" type: choice + default: dry_run options: - dry_run - release + ref: + description: 'The branch, tag or SHA to checkout' + required: false + type: string jobs: publish: @@ -37,8 +41,13 @@ jobs: run: working-directory: java steps: - - uses: actions/checkout@v4 - + - name: Checkout repository + uses: actions/checkout@v4 + with: + # When triggered by a release, use the release tag + # When triggered manually, use the provided ref + ref: ${{ github.event.release.tag_name || inputs.ref }} + - name: Set up Java sdk uses: actions/setup-java@v4 with: @@ -56,7 +65,9 @@ jobs: git config --global user.email "dev+gha@lance.org" - name: Dry run - if: github.event_name == 'pull_request' + if: | + github.event_name == 'pull_request' || + inputs.mode == 'dry_run' run: | ./mvnw --batch-mode -DskipTests package diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 3c30ce11..c2c55e38 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -22,13 +22,17 @@ on: workflow_dispatch: inputs: mode: - description: "dry_run: build & test only, release: build & publish to PyPI" + description: 'Release mode' required: true - default: "dry_run" type: choice + default: dry_run options: - dry_run - release + ref: + description: 'The branch, tag or SHA to checkout' + required: false + type: string jobs: publish: @@ -37,8 +41,12 @@ jobs: id-token: write # Required for PyPI trusted publishing contents: read steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@v4 + with: + # When triggered by a release, use the release tag + # When triggered manually, use the provided ref + ref: ${{ github.event.release.tag_name || inputs.ref }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7cd5ac3e..1841478c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,6 +99,7 @@ jobs: id: versions run: | BASE_VERSION="${{ steps.base_version.outputs.version }}" + CURRENT_VERSION="${{ steps.current_version.outputs.version }}" if [ "${{ inputs.release_channel }}" == "stable" ]; then TAG="v${BASE_VERSION}" POM_VERSION="${BASE_VERSION}" @@ -117,13 +118,22 @@ jobs: POM_VERSION="${BASE_VERSION}-beta.${BETA_NUM}" fi + # Check if version actually changes (needed for commit/push decisions) + if [ "$CURRENT_VERSION" != "$POM_VERSION" ]; then + VERSION_CHANGED="true" + else + VERSION_CHANGED="false" + fi + echo "tag=$TAG" >> $GITHUB_OUTPUT echo "pom_version=$POM_VERSION" >> $GITHUB_OUTPUT + echo "version_changed=$VERSION_CHANGED" >> $GITHUB_OUTPUT echo "Tag will be: $TAG" echo "POM version will be: $POM_VERSION" + echo "Version changed: $VERSION_CHANGED" - name: Update version (when version changes) - if: inputs.release_type != 'current' + if: steps.versions.outputs.version_changed == 'true' run: | python ci/bump_version.py --version "${{ steps.versions.outputs.pom_version }}" @@ -133,20 +143,20 @@ jobs: git config user.email 'dev+gha@lance.org' - name: Regenerate modules after version update - if: inputs.release_type != 'current' + if: steps.versions.outputs.version_changed == 'true' run: | git diff make clean make gen - name: Update Cargo lock version (when version changes) - if: inputs.release_type != 'current' + if: steps.versions.outputs.version_changed == 'true' working-directory: rust run: | make build - name: Create release commit (when version changes) - if: inputs.release_type != 'current' + if: steps.versions.outputs.version_changed == 'true' run: | git add -A git commit -m "chore: release version ${{ steps.versions.outputs.pom_version }}" || echo "No changes to commit" @@ -163,7 +173,7 @@ jobs: # Configure git to use the token for authentication git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git" - if [ "${{ inputs.release_type }}" != "current" ]; then + if [ "${{ steps.versions.outputs.version_changed }}" == "true" ]; then # Push the version bump commit git push origin main fi @@ -188,7 +198,7 @@ jobs: echo "- **Release Type:** ${{ inputs.release_type }}" >> $GITHUB_STEP_SUMMARY echo "- **Release Channel:** ${{ inputs.release_channel }}" >> $GITHUB_STEP_SUMMARY echo "- **Current Version:** ${{ steps.current_version.outputs.version }}" >> $GITHUB_STEP_SUMMARY - if [ "${{ inputs.release_type }}" != "current" ]; then + if [ "${{ steps.versions.outputs.version_changed }}" == "true" ]; then echo "- **New Version:** ${{ steps.versions.outputs.pom_version }}" >> $GITHUB_STEP_SUMMARY fi echo "- **Tag:** ${{ steps.versions.outputs.tag }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index 27390745..de74bf17 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -22,13 +22,17 @@ on: workflow_dispatch: inputs: mode: - description: "dry_run: build & test only, release: build & publish to crates.io" + description: 'Release mode' required: true - default: "dry_run" type: choice + default: dry_run options: - dry_run - release + ref: + description: 'The branch, tag or SHA to checkout' + required: false + type: string env: # This env var is used by Swatinem/rust-cache@v2 for the cache @@ -48,8 +52,12 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - name: Checkout code + - name: Checkout repository uses: actions/checkout@v4 + with: + # When triggered by a release, use the release tag + # When triggered manually, use the provided ref + ref: ${{ github.event.release.tag_name || inputs.ref }} - name: Install dependencies run: | sudo apt update diff --git a/ci/calculate_version.py b/ci/calculate_version.py index fe3fa80a..0cb39e5e 100755 --- a/ci/calculate_version.py +++ b/ci/calculate_version.py @@ -32,10 +32,11 @@ def calculate_next_version(current_version, release_type, channel): elif release_type == 'patch': new_version = f"{major}.{minor}.{patch + 1}" elif release_type == 'current': - # Keep current version - used for: + # Keep current base version - used for: # - Subsequent preview releases (v0.0.16-beta.2, beta.3, etc.) # - Finalizing preview to stable (v0.0.16-beta.X -> v0.0.16) - new_version = current_version + # Strip any pre-release suffix to get base version (e.g., 0.1.3-beta.1 -> 0.1.3) + new_version = f"{major}.{minor}.{patch}" else: raise ValueError(f"Unknown release type: {release_type}") diff --git a/docs/src/client/operations/index.md b/docs/src/client/operations/index.md index 450cacb8..f0b276d0 100644 --- a/docs/src/client/operations/index.md +++ b/docs/src/client/operations/index.md @@ -108,34 +108,33 @@ These operations provide the foundational metadata management capabilities neede without requiring data or index operation support. With the namespace able to provide basic information about the table, the Lance SDK can be used to fulfill the other operations. -### Why Not CreateTable and DropTable? +### Why Not `CreateTable` and `DropTable`? -`CreateTable` and `DropTable` are intentionally excluded from the recommended basic operations because they involve +`CreateTable` and `DropTable` are common in most catalog systems, +but are intentionally excluded from the recommended basic operations because they involve data operations that present challenges for catalog implementations: -**Data Operation Complexity:** -Both `CreateTable` and `DropTable` are considered data operations rather than pure metadata operations. -They can be long-running, especially when dealing with large datasets or remote storage systems. -This makes them difficult to implement reliably in catalog systems that are designed for fast metadata lookups. +- **Data Operation Complexity:** + Both `CreateTable` and `DropTable` are data operations rather than pure metadata operations. + They can be long-running, especially when dealing with large datasets or remote storage systems. + This makes them difficult to implement reliably in catalog systems designed for fast metadata lookups. -**Atomicity Guarantees:** -Data operations require careful handling of atomicity. A failed `CreateTable` or `DropTable` operation -can leave the system in an inconsistent state with partially created or deleted data files. -Catalog implementations would need to implement complex cleanup and recovery mechanisms. +- **Atomicity Guarantees:** + Data operations require careful handling of atomicity. A failed `CreateTable` or `DropTable` operation + can leave the system in an inconsistent state with partially created or deleted data files. + Catalog implementations would need to implement complex cleanup and recovery mechanisms. -**CreateTable Challenges:** -`CreateTable` is particularly difficult for catalogs to fully implement because features like -CREATE TABLE AS SELECT (CTAS) require either complicated staging mechanisms or multi-table -multi-statement transaction support. Most catalog systems are not designed to handle such complex workflows. +- **CreateTable Challenges:** + `CreateTable` is particularly difficult for catalogs to fully implement because features like + CREATE TABLE AS SELECT (CTAS) require either complicated staging mechanisms or multi-statement + transaction support. +While some catalog systems can handle these complex workflows, +doing so typically requires deep, dedicated integration. Lance Namespace aims to enable as many catalogs as possible to adopt Lance format. By focusing on `DeclareTable` and `DeregisterTable` instead of `CreateTable` and `DropTable`, namespace implementations only -need to handle metadata operations that are always fast and atomic across all catalog solutions. - -**Recommended Approach:** -- Use **DeclareTable** to reserve a table name and location, then use the Lance SDK to write data -- Use **DeregisterTable** to unregister a table while preserving its data for potential re-registration -- Use the Lance SDK directly for data operations when full control over the data lifecycle is needed +need to handle metadata operations that are simple, fast and atomic across all catalog solutions. +`CreateTable` and `DropTable` can then be fulfilled by combining these metadata operations with the Lance SDK. ## Operation Versioning From dfefe0dfd18422f6141f56396d283717e9c28723 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 21 Dec 2025 07:39:09 -0800 Subject: [PATCH 2/3] fix --- .github/workflows/java-publish.yml | 5 +++-- .github/workflows/python-publish.yml | 5 +++-- .github/workflows/rust-publish.yml | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 3ad1d997..26354a4c 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -45,8 +45,9 @@ jobs: uses: actions/checkout@v4 with: # When triggered by a release, use the release tag - # When triggered manually, use the provided ref - ref: ${{ github.event.release.tag_name || inputs.ref }} + # When triggered manually with a ref, use the provided ref + # Otherwise (PR or manual without ref), use the default branch + ref: ${{ github.event.release.tag_name || inputs.ref || '' }} - name: Set up Java sdk uses: actions/setup-java@v4 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index c2c55e38..9b6612a7 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -45,8 +45,9 @@ jobs: uses: actions/checkout@v4 with: # When triggered by a release, use the release tag - # When triggered manually, use the provided ref - ref: ${{ github.event.release.tag_name || inputs.ref }} + # When triggered manually with a ref, use the provided ref + # Otherwise (PR or manual without ref), use the default branch + ref: ${{ github.event.release.tag_name || inputs.ref || '' }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index de74bf17..39589f97 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -56,8 +56,9 @@ jobs: uses: actions/checkout@v4 with: # When triggered by a release, use the release tag - # When triggered manually, use the provided ref - ref: ${{ github.event.release.tag_name || inputs.ref }} + # When triggered manually with a ref, use the provided ref + # Otherwise (PR or manual without ref), use the default branch + ref: ${{ github.event.release.tag_name || inputs.ref || '' }} - name: Install dependencies run: | sudo apt update From 3b590e458ae0e3fdb901c749a734111ae89ec78f Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 21 Dec 2025 07:42:14 -0800 Subject: [PATCH 3/3] commit --- .github/workflows/rust-publish.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index 39589f97..11fa9c2b 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -72,10 +72,21 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: rust - - uses: katyo/publish-crates@v2 + - name: Dry run (build and package only) + if: | + github.event_name == 'pull_request' || + (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'dry_run') + working-directory: rust + run: | + cargo build --all-features + cargo package --all-features --allow-dirty + + - name: Publish to crates.io + if: | + (github.event_name == 'release' && github.event.action == 'released') || + (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'release') + uses: katyo/publish-crates@v2 with: - # registry-token: ${{ steps.auth.outputs.token }} registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} args: "--all-features" - path: rust - dry-run: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'dry_run') }} \ No newline at end of file + path: rust \ No newline at end of file