diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml new file mode 100644 index 0000000..1cd8be6 --- /dev/null +++ b/.github/workflows/build-python-wheels.yml @@ -0,0 +1,185 @@ +name: Build Python Wheels + +on: + workflow_call: + inputs: + upload-artifacts: + description: 'Upload wheel artifacts' + required: false + default: true + type: boolean + test-wheels: + description: 'Test the built wheels' + required: false + default: true + type: boolean + +jobs: + build-wheels: + name: Build wheel (${{ matrix.platform }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # macOS Apple Silicon + - os: macos-14 + platform: darwin-aarch64 + wheel_platform: macosx_14_0_arm64 + rust_target: aarch64-apple-darwin + + # Linux x86_64 glibc + - os: ubuntu-22.04 + platform: linux-x86_64-gnu + wheel_platform: manylinux_2_35_x86_64 + rust_target: x86_64-unknown-linux-gnu + + # Linux ARM64 glibc + - os: ubuntu-22.04-arm + platform: linux-aarch64-gnu + wheel_platform: manylinux_2_35_aarch64 + rust_target: aarch64-unknown-linux-gnu + + # Windows x64 + - os: windows-latest + platform: windows-x86_64 + wheel_platform: win_amd64 + rust_target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.rust_target }} + + - name: Build pg0 CLI (Unix) + if: runner.os != 'Windows' + env: + BUNDLE_POSTGRESQL: "true" + run: | + cargo build --release --target ${{ matrix.rust_target }} + ls -la target/${{ matrix.rust_target }}/release/pg0* + + - name: Build pg0 CLI (Windows) + if: runner.os == 'Windows' + env: + BUNDLE_POSTGRESQL: "true" + run: | + cargo build --release --target ${{ matrix.rust_target }} + dir target\${{ matrix.rust_target }}\release\pg0* + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Build wheel (Unix) + if: runner.os != 'Windows' + working-directory: sdk/python + env: + PG0_BINARY_PATH: ${{ github.workspace }}/target/${{ matrix.rust_target }}/release/pg0 + run: uv build --wheel + + - name: Build wheel (Windows) + if: runner.os == 'Windows' + working-directory: sdk/python + env: + PG0_BINARY_PATH: ${{ github.workspace }}\target\${{ matrix.rust_target }}\release\pg0.exe + run: uv build --wheel + + - name: Rename wheel to platform-specific + working-directory: sdk/python/dist + run: | + for f in *.whl; do + newname=$(echo "$f" | sed 's/py3-none-any/py3-none-${{ matrix.wheel_platform }}/g') + if [ "$f" != "$newname" ]; then + echo "Renaming $f -> $newname" + mv "$f" "$newname" + fi + done + ls -la + shell: bash + + - name: Test wheel (Unix) + if: inputs.test-wheels && runner.os != 'Windows' + working-directory: sdk/python + run: | + uv venv test-env + source test-env/bin/activate + pip install dist/*.whl + + python -c " + from pg0 import _get_bundled_binary, Pg0 + bundled = _get_bundled_binary() + assert bundled is not None, 'Bundled binary not found!' + assert bundled.exists(), f'Bundled binary does not exist: {bundled}' + print(f'Bundled binary found: {bundled}') + + pg = Pg0(name='wheel-test') + info = pg.start() + print(f'Started PostgreSQL: {info.uri}') + pg.stop() + pg.drop() + print('Wheel test passed!') + " + + - name: Test wheel (Windows) + if: inputs.test-wheels && runner.os == 'Windows' + working-directory: sdk/python + run: | + uv venv test-env + test-env\Scripts\activate + pip install (Get-ChildItem dist\*.whl).FullName + + python -c " + from pg0 import _get_bundled_binary, Pg0 + bundled = _get_bundled_binary() + assert bundled is not None, 'Bundled binary not found!' + assert bundled.exists(), f'Bundled binary does not exist: {bundled}' + print(f'Bundled binary found: {bundled}') + + pg = Pg0(name='wheel-test') + info = pg.start() + print(f'Started PostgreSQL: {info.uri}') + pg.stop() + pg.drop() + print('Wheel test passed!') + " + shell: pwsh + + - name: Upload wheel artifact + if: inputs.upload-artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.platform }} + path: sdk/python/dist/*.whl + + build-sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Build sdist + working-directory: sdk/python + run: uv build --sdist + + - name: Upload sdist artifact + if: inputs.upload-artifacts + uses: actions/upload-artifact@v4 + with: + name: sdist + path: sdk/python/dist/*.tar.gz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95211b7..cec7a01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: name: pg0-macos path: target/release/pg0 - sdk-python: - name: Python SDK Tests + sdk-tests: + name: SDK Tests (macOS) needs: build runs-on: macos-latest steps: @@ -49,77 +49,46 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 - - name: Install dependencies + - name: Run Python SDK tests working-directory: sdk/python - run: uv sync --dev - - - name: Run tests - working-directory: sdk/python - run: | - export PATH="$HOME/.local/bin:$PATH" - uv run pytest tests/ -v - - sdk-node: - name: Node.js SDK Tests - needs: build - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - - name: Download CLI - uses: actions/download-artifact@v4 - with: - name: pg0-macos - path: ~/.local/bin - - - name: Make CLI executable - run: chmod +x ~/.local/bin/pg0 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Install dependencies - working-directory: sdk/node - run: npm install - - - name: Run tests - working-directory: sdk/node run: | export PATH="$HOME/.local/bin:$PATH" - npm test + uv pip install --system -e ".[dev]" + pytest tests/ -v + # Docker tests - one job per platform, runs both CLI and Python SDK tests + # Note: ARM64 tests are skipped because QEMU emulation is too slow for PostgreSQL setup docker-tests: name: Docker Tests (${{ matrix.platform }}) - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - platform: debian-amd64 - runner: ubuntu-latest - script: docker-tests/test_debian_amd64.sh - - platform: debian-arm64 - runner: ubuntu-latest - script: docker-tests/test_debian_arm64.sh + cli_script: docker-tests/test_debian_amd64.sh + python_script: docker-tests/python/test_debian_amd64.sh - platform: alpine-amd64 - runner: ubuntu-latest - script: docker-tests/test_alpine_amd64.sh - - platform: alpine-arm64 - runner: ubuntu-latest - script: docker-tests/test_alpine_arm64.sh + cli_script: docker-tests/test_alpine_amd64.sh + python_script: docker-tests/python/test_alpine_amd64.sh steps: - uses: actions/checkout@v4 - - name: Set up QEMU (for ARM64 emulation) - if: contains(matrix.platform, 'arm64') - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 + - name: Run CLI Docker test + run: | + chmod +x ${{ matrix.cli_script }} + bash ${{ matrix.cli_script }} - - name: Run Docker test + - name: Run Python SDK Docker test run: | - chmod +x ${{ matrix.script }} - bash ${{ matrix.script }} + chmod +x ${{ matrix.python_script }} + bash ${{ matrix.python_script }} + + # Python wheel builds - test that wheel building works + python-wheels: + name: Python Wheels + uses: ./.github/workflows/build-python-wheels.yml + with: + upload-artifacts: false + test-wheels: true diff --git a/.github/workflows/release-cli.yml b/.github/workflows/release-cli.yml index a9b8ff8..7dd5799 100644 --- a/.github/workflows/release-cli.yml +++ b/.github/workflows/release-cli.yml @@ -7,6 +7,7 @@ on: permissions: contents: write + id-token: write jobs: build-cli: @@ -103,8 +104,123 @@ jobs: name: cli-${{ matrix.name }} path: pg0-${{ matrix.name }}.exe + build-python-wheels: + name: Build Python wheel (${{ matrix.platform }}) + needs: build-cli + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + # macOS Apple Silicon - reuse darwin-aarch64 CLI + - os: macos-14 + platform: darwin-aarch64 + wheel_platform: macosx_14_0_arm64 + cli_artifact: cli-darwin-aarch64 + cli_binary: pg0-darwin-aarch64 + + # Linux x86_64 glibc - reuse linux-x86_64-gnu CLI + - os: ubuntu-22.04 + platform: linux-x86_64-gnu + wheel_platform: manylinux_2_35_x86_64 + cli_artifact: cli-linux-x86_64-gnu + cli_binary: pg0-linux-x86_64-gnu + + # Linux ARM64 glibc - reuse linux-aarch64-gnu CLI + - os: ubuntu-22.04-arm + platform: linux-aarch64-gnu + wheel_platform: manylinux_2_35_aarch64 + cli_artifact: cli-linux-aarch64-gnu + cli_binary: pg0-linux-aarch64-gnu + + # Windows x64 - reuse windows-x86_64 CLI + - os: windows-latest + platform: windows-x86_64 + wheel_platform: win_amd64 + cli_artifact: cli-windows-x86_64 + cli_binary: pg0-windows-x86_64.exe + + steps: + - uses: actions/checkout@v4 + + - name: Download CLI artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.cli_artifact }} + path: cli-bin + + - name: Prepare CLI binary (Unix) + if: runner.os != 'Windows' + run: | + chmod +x cli-bin/${{ matrix.cli_binary }} + ls -la cli-bin/ + + - name: Prepare CLI binary (Windows) + if: runner.os == 'Windows' + run: | + dir cli-bin\ + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Build wheel + working-directory: sdk/python + env: + PG0_BINARY_PATH: ${{ github.workspace }}/cli-bin/${{ matrix.cli_binary }} + run: | + uv build --wheel + shell: bash + + - name: Rename wheel to platform-specific + working-directory: sdk/python/dist + run: | + for f in *.whl; do + newname=$(echo "$f" | sed 's/py3-none-any/py3-none-${{ matrix.wheel_platform }}/g') + if [ "$f" != "$newname" ]; then + echo "Renaming $f -> $newname" + mv "$f" "$newname" + fi + done + ls -la + shell: bash + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.platform }} + path: sdk/python/dist/*.whl + + build-python-sdist: + name: Build Python sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Build sdist + working-directory: sdk/python + run: uv build --sdist + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: python-sdist + path: sdk/python/dist/*.tar.gz + release: - needs: [build-cli] + name: Create GitHub Release + needs: [build-cli, build-python-wheels, build-python-sdist] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -124,7 +240,12 @@ jobs: - name: Prepare release files run: | mkdir release - find artifacts -type f -exec mv {} release/ \; + # CLI binaries + find artifacts/cli-* -type f -exec mv {} release/ \; + # Python wheels + find artifacts/wheel-* -type f -exec mv {} release/ \; + # Python sdist + find artifacts/python-sdist -type f -exec mv {} release/ \; ls -la release/ - name: Create Release @@ -133,23 +254,64 @@ jobs: PG_VERSION: ${{ steps.versions.outputs.PG_VERSION }} PGVECTOR_VERSION: ${{ steps.versions.outputs.PGVECTOR_VERSION }} run: | - echo "## pg0 - Embedded PostgreSQL CLI" > release_notes.md - echo "" >> release_notes.md - echo "### CLI Binaries" >> release_notes.md - echo "- \`pg0-darwin-aarch64\` - macOS Apple Silicon" >> release_notes.md - echo "- \`pg0-linux-x86_64-gnu\` - Linux x86_64 for Debian/Ubuntu (glibc)" >> release_notes.md - echo "- \`pg0-linux-x86_64-musl\` - Linux x86_64 for Alpine (musl, statically linked)" >> release_notes.md - echo "- \`pg0-linux-aarch64-gnu\` - Linux ARM64 for Debian/Ubuntu (glibc)" >> release_notes.md - echo "- \`pg0-linux-aarch64-musl\` - Linux ARM64 for Alpine (musl, statically linked)" >> release_notes.md - echo "- \`pg0-windows-x86_64.exe\` - Windows x64" >> release_notes.md - echo "" >> release_notes.md - echo "### Bundled Components" >> release_notes.md - echo "Everything is bundled directly in the binary - no downloads required, works completely offline!" >> release_notes.md + cat > release_notes.md << 'EOF' + ## pg0 - Embedded PostgreSQL CLI + + ### CLI Binaries + - `pg0-darwin-aarch64` - macOS Apple Silicon + - `pg0-linux-x86_64-gnu` - Linux x86_64 for Debian/Ubuntu (glibc) + - `pg0-linux-x86_64-musl` - Linux x86_64 for Alpine (musl, statically linked) + - `pg0-linux-aarch64-gnu` - Linux ARM64 for Debian/Ubuntu (glibc) + - `pg0-linux-aarch64-musl` - Linux ARM64 for Alpine (musl, statically linked) + - `pg0-windows-x86_64.exe` - Windows x64 + + ### Python Package + Install from PyPI: + ```bash + pip install pg0-embedded + ``` + + Pre-built wheels available for: + - macOS Apple Silicon (`macosx_14_0_arm64`) + - Linux x86_64 glibc (`manylinux_2_35_x86_64`) + - Linux ARM64 glibc (`manylinux_2_35_aarch64`) + - Windows x64 (`win_amd64`) + + ### Bundled Components + Everything is bundled directly in the binary - no downloads required, works completely offline! + EOF echo "- PostgreSQL ${PG_VERSION}" >> release_notes.md echo "- pgvector ${PGVECTOR_VERSION}" >> release_notes.md echo "" >> release_notes.md echo "### Installation (macOS/Linux)" >> release_notes.md - echo "\`\`\`bash" >> release_notes.md + echo '```bash' >> release_notes.md echo "curl -fsSL https://raw.githubusercontent.com/vectorize-io/pg0/main/install.sh | bash" >> release_notes.md - echo "\`\`\`" >> release_notes.md + echo '```' >> release_notes.md + gh release create "${{ github.ref_name }}" --title "${{ github.ref_name }}" --notes-file release_notes.md release/* + + publish-pypi: + name: Publish to PyPI + needs: [build-python-wheels, build-python-sdist, release] + runs-on: ubuntu-latest + steps: + - name: Download wheel artifacts + uses: actions/download-artifact@v4 + with: + pattern: wheel-* + path: dist + merge-multiple: true + + - name: Download sdist artifact + uses: actions/download-artifact@v4 + with: + name: python-sdist + path: dist + + - name: List artifacts + run: ls -la dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml deleted file mode 100644 index 570e4bf..0000000 --- a/.github/workflows/release-python.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Release Python - -on: - push: - tags: - - 'py-*' - -permissions: - contents: read - id-token: write - -jobs: - release: - name: Release Python Package - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Build package - working-directory: sdk/python - run: uv build - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: sdk/python/dist/ diff --git a/.gitignore b/.gitignore index a6d07dd..7f5ce7c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ Cargo.lock .idea/ node_modules/ .env -*.pyc \ No newline at end of file +*.pyc +sdk/python/pg0/bin/ \ No newline at end of file diff --git a/README.md b/README.md index 5d8554b..62d263a 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,13 @@ [![PyPI downloads](https://img.shields.io/pypi/dm/pg0-embedded.svg)](https://pypi.org/project/pg0-embedded/) [![Python versions](https://img.shields.io/pypi/pyversions/pg0-embedded.svg)](https://pypi.org/project/pg0-embedded/) -**Language-agnostic embedded PostgreSQL with zero dependencies.** +**Zero-config PostgreSQL with pgvector.** A single binary that runs PostgreSQL locally - no installation, no configuration, no Docker required. Includes **pgvector** for AI/vector workloads out of the box. ## Why pg0? -**Stop compromising on SQLite.** When building applications that need a database, developers often choose SQLite for simplicity - but then face painful migrations when they need PostgreSQL features in production. - -pg0 gives you **real PostgreSQL** with the same simplicity as SQLite: +PostgreSQL setup is painful. Docker adds complexity. Local installs conflict with system packages. pg0 gives you a real PostgreSQL server with zero friction: - **No installation** - download a single binary and run `pg0 start` - **No Docker** - no containers, no daemon, no complexity @@ -426,6 +424,36 @@ PostgreSQL and pgvector are **bundled directly** into the pg0 binary - no downlo Data is stored in `~/.pg0/instances//data/` (or your custom `--data-dir`) and persists between restarts. +## Runtime Dependencies + +pg0 bundles PostgreSQL but requires some shared libraries at runtime. These are typically pre-installed on most systems, but may need to be added in minimal environments like Docker. + +**macOS:** No additional dependencies required. + +**Linux (Debian/Ubuntu):** +```bash +apt-get install libxml2 libssl3 libgssapi-krb5-2 +``` + +**Linux (Alpine):** +```bash +apk add icu-libs lz4-libs libxml2 +``` + +### Why these dependencies? + +The bundled PostgreSQL binaries are compiled with these features enabled: + +| Library | Purpose | Can disable? | +|---------|---------|--------------| +| OpenSSL (`libssl`) | SSL/TLS connections | Not recommended | +| GSSAPI (`libgssapi-krb5`) | Kerberos authentication | Rarely needed locally | +| libxml2 | XML data type and functions | Rarely needed | +| ICU (`icu-libs`) | Unicode collation (Alpine only) | glibc builds don't need it | +| LZ4 (`lz4-libs`) | WAL/TOAST compression | Small impact | + +Most desktop Linux distributions and macOS have these libraries pre-installed. You only need to install them manually in minimal Docker images or bare-metal servers. + ## Troubleshooting ### PostgreSQL Cannot Run as Root diff --git a/docker-tests/README.md b/docker-tests/README.md index 61f01ea..195484e 100644 --- a/docker-tests/README.md +++ b/docker-tests/README.md @@ -2,7 +2,7 @@ Automated tests to verify pg0 works correctly across different platforms and distributions. -## Test Matrix +## CLI Tests | Test | Image | Platform | Architecture | libc | |------|-------|----------|--------------|------| @@ -11,6 +11,17 @@ Automated tests to verify pg0 works correctly across different platforms and dis | `test_alpine_amd64.sh` | python:3.12-alpine3.20 | linux/amd64 | x86_64 | musl | | `test_alpine_arm64.sh` | python:3.12-alpine3.20 | linux/arm64 | aarch64 | musl | +## Python SDK Tests + +| Test | Image | Platform | Architecture | libc | +|------|-------|----------|--------------|------| +| `python/test_debian_amd64.sh` | python:3.11-slim | linux/amd64 | x86_64 | glibc | +| `python/test_debian_arm64.sh` | python:3.11-slim | linux/arm64 | aarch64 | glibc | +| `python/test_alpine_amd64.sh` | python:3.12-alpine3.20 | linux/amd64 | x86_64 | musl | +| `python/test_alpine_arm64.sh` | python:3.12-alpine3.20 | linux/arm64 | aarch64 | musl | + +The Python SDK tests install the package via `pip install .` and verify the bundled binary works correctly. + ## What Each Test Does 1. **System Check** - Verifies architecture and OS @@ -55,7 +66,7 @@ chmod +x *.sh - ✅ All queries: Success ### Alpine (musl) -- ✅ PostgreSQL: Works (requires `icu-libs`, `lz4-libs`, `libxml2` packages and ICU 74 - use Alpine 3.20) +- ✅ PostgreSQL: Works (requires `icu-libs`, `lz4-libs`, `libxml2`, `zstd-libs`, `procps` packages and ICU 74 - use Alpine 3.20) - ⚠️ pgvector: Fails (no musl binaries available - glibc-only) - ✅ Basic queries: Success diff --git a/docker-tests/python/test_alpine_amd64.sh b/docker-tests/python/test_alpine_amd64.sh new file mode 100755 index 0000000..17c9a70 --- /dev/null +++ b/docker-tests/python/test_alpine_amd64.sh @@ -0,0 +1,114 @@ +#!/bin/bash +set -e + +echo "==================================" +echo "Testing pg0 Python SDK on Alpine AMD64" +echo "Image: python:3.12-alpine3.20" +echo "Platform: linux/amd64" +echo "==================================" + +# Get the script directory to find the SDK +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SDK_DIR="$SCRIPT_DIR/../../sdk/python" + +# Note: Using Alpine 3.20 because the musl PostgreSQL binary requires ICU 74 +# Alpine 3.22 has ICU 76 which is not compatible +docker run --rm --platform=linux/amd64 \ + -v "$SDK_DIR:/sdk-src:ro" \ + python:3.12-alpine3.20 sh -c ' +set -e + +echo "=== System Info ===" +uname -m +cat /etc/os-release | grep PRETTY_NAME + +echo "" +echo "=== Installing system dependencies ===" +# procps is needed for pg0 to check if postgres process is running +# zstd-libs is needed for PostgreSQL compression support +apk add --no-cache bash sudo shadow icu-libs lz4-libs libxml2 procps zstd-libs > /dev/null 2>&1 + +echo "" +echo "=== Creating non-root user ===" +adduser -D -s /bin/bash pguser + +# Copy SDK to writable location (excluding any existing bin directory with wrong-platform binary) +mkdir -p /home/pguser/sdk +cp -r /sdk-src/pg0 /home/pguser/sdk/ +cp /sdk-src/pyproject.toml /sdk-src/hatch_build.py /sdk-src/README.md /home/pguser/sdk/ +rm -rf /home/pguser/sdk/pg0/bin # Remove any existing binary +chown -R pguser:pguser /home/pguser/sdk + +echo "" +echo "=== Switching to non-root user ===" +su - pguser << EOF +set -e +export PATH="/usr/local/bin:\$PATH" + +echo "=== Installing Python SDK (will download correct binary) ===" +cd /home/pguser/sdk +python3 -m pip install --user . -q + +echo "" +echo "=== Testing Python SDK ===" +python3 << PYEOF +from pg0 import Pg0, _get_bundled_binary + +# Check bundled binary +bundled = _get_bundled_binary() +print(f"Bundled binary: {bundled}") +assert bundled is not None, "Bundled binary not found!" +assert bundled.exists(), f"Bundled binary does not exist: {bundled}" + +# Start PostgreSQL +print("") +print("=== Starting PostgreSQL ===") +pg = Pg0() +info = pg.start() +print(f"PostgreSQL running on port {info.port}") +print(f"URI: {info.uri}") + +# Test basic query +print("") +print("=== Testing basic SELECT query ===") +result = pg.execute("SELECT version();") +print(result.strip().split("\\n")[0][:80]) + +# Test table operations +print("") +print("=== Testing table creation and data ===") +pg.execute("CREATE TABLE test (id INT, name TEXT);") +pg.execute("INSERT INTO test VALUES (1, '\''Hello'\''), (2, '\''World'\'');") +result = pg.execute("SELECT * FROM test;") +print(result) + +# Test pgvector (expected to fail on Alpine/musl) +print("") +print("=== Testing pgvector extension ===") +try: + pg.execute("CREATE EXTENSION IF NOT EXISTS vector;") + print("✅ pgvector extension created successfully") + pg.execute("CREATE TABLE embeddings (id INT, vec vector(3));") + pg.execute("INSERT INTO embeddings VALUES (1, '\''[1,2,3]'\'');") + result = pg.execute("SELECT * FROM embeddings;") + print(result) + print("✅ pgvector working correctly") +except Exception as e: + print("⚠️ pgvector extension failed (known limitation on Alpine/musl)") + +# Stop PostgreSQL +print("") +print("=== Stopping PostgreSQL ===") +pg.stop() +pg.drop() + +print("") +print("==================================") +print("✅ ALL TESTS PASSED - Alpine AMD64 Python SDK") +print("==================================") +PYEOF +EOF +' + +echo "" +echo "✅ Test completed successfully!" diff --git a/docker-tests/python/test_alpine_arm64.sh b/docker-tests/python/test_alpine_arm64.sh new file mode 100755 index 0000000..fbca941 --- /dev/null +++ b/docker-tests/python/test_alpine_arm64.sh @@ -0,0 +1,114 @@ +#!/bin/bash +set -e + +echo "==================================" +echo "Testing pg0 Python SDK on Alpine ARM64" +echo "Image: python:3.12-alpine3.20" +echo "Platform: linux/arm64" +echo "==================================" + +# Get the script directory to find the SDK +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SDK_DIR="$SCRIPT_DIR/../../sdk/python" + +# Note: Using Alpine 3.20 because the musl PostgreSQL binary requires ICU 74 +# Alpine 3.22 has ICU 76 which is not compatible +docker run --rm --platform=linux/arm64 \ + -v "$SDK_DIR:/sdk-src:ro" \ + python:3.12-alpine3.20 sh -c ' +set -e + +echo "=== System Info ===" +uname -m +cat /etc/os-release | grep PRETTY_NAME + +echo "" +echo "=== Installing system dependencies ===" +# procps is needed for pg0 to check if postgres process is running +# zstd-libs is needed for PostgreSQL compression support +apk add --no-cache bash sudo shadow icu-libs lz4-libs libxml2 procps zstd-libs > /dev/null 2>&1 + +echo "" +echo "=== Creating non-root user ===" +adduser -D -s /bin/bash pguser + +# Copy SDK to writable location (excluding any existing bin directory with wrong-platform binary) +mkdir -p /home/pguser/sdk +cp -r /sdk-src/pg0 /home/pguser/sdk/ +cp /sdk-src/pyproject.toml /sdk-src/hatch_build.py /sdk-src/README.md /home/pguser/sdk/ +rm -rf /home/pguser/sdk/pg0/bin # Remove any existing binary +chown -R pguser:pguser /home/pguser/sdk + +echo "" +echo "=== Switching to non-root user ===" +su - pguser << EOF +set -e +export PATH="/usr/local/bin:\$PATH" + +echo "=== Installing Python SDK (will download correct binary) ===" +cd /home/pguser/sdk +python3 -m pip install --user . -q + +echo "" +echo "=== Testing Python SDK ===" +python3 << PYEOF +from pg0 import Pg0, _get_bundled_binary + +# Check bundled binary +bundled = _get_bundled_binary() +print(f"Bundled binary: {bundled}") +assert bundled is not None, "Bundled binary not found!" +assert bundled.exists(), f"Bundled binary does not exist: {bundled}" + +# Start PostgreSQL +print("") +print("=== Starting PostgreSQL ===") +pg = Pg0() +info = pg.start() +print(f"PostgreSQL running on port {info.port}") +print(f"URI: {info.uri}") + +# Test basic query +print("") +print("=== Testing basic SELECT query ===") +result = pg.execute("SELECT version();") +print(result.strip().split("\\n")[0][:80]) + +# Test table operations +print("") +print("=== Testing table creation and data ===") +pg.execute("CREATE TABLE test (id INT, name TEXT);") +pg.execute("INSERT INTO test VALUES (1, '\''Hello'\''), (2, '\''World'\'');") +result = pg.execute("SELECT * FROM test;") +print(result) + +# Test pgvector (expected to fail on Alpine/musl) +print("") +print("=== Testing pgvector extension ===") +try: + pg.execute("CREATE EXTENSION IF NOT EXISTS vector;") + print("✅ pgvector extension created successfully") + pg.execute("CREATE TABLE embeddings (id INT, vec vector(3));") + pg.execute("INSERT INTO embeddings VALUES (1, '\''[1,2,3]'\'');") + result = pg.execute("SELECT * FROM embeddings;") + print(result) + print("✅ pgvector working correctly") +except Exception as e: + print("⚠️ pgvector extension failed (known limitation on Alpine/musl)") + +# Stop PostgreSQL +print("") +print("=== Stopping PostgreSQL ===") +pg.stop() +pg.drop() + +print("") +print("==================================") +print("✅ ALL TESTS PASSED - Alpine ARM64 Python SDK") +print("==================================") +PYEOF +EOF +' + +echo "" +echo "✅ Test completed successfully!" diff --git a/docker-tests/python/test_debian_amd64.sh b/docker-tests/python/test_debian_amd64.sh new file mode 100755 index 0000000..98f43de --- /dev/null +++ b/docker-tests/python/test_debian_amd64.sh @@ -0,0 +1,118 @@ +#!/bin/bash +set -e + +echo "==================================" +echo "Testing pg0 Python SDK on Debian AMD64" +echo "Image: python:3.11-slim" +echo "Platform: linux/amd64" +echo "==================================" + +# Get the script directory to find the SDK +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SDK_DIR="$SCRIPT_DIR/../../sdk/python" + +docker run --rm --platform=linux/amd64 \ + -v "$SDK_DIR:/sdk-src:ro" \ + python:3.11-slim bash -c ' +set -e + +echo "=== System Info ===" +uname -m +cat /etc/os-release | grep PRETTY_NAME + +echo "" +echo "=== Installing system dependencies ===" +apt-get update -qq +# procps is needed for pg0 to check if postgres process is running +apt-get install -y -qq libxml2 libssl3 libgssapi-krb5-2 procps > /dev/null 2>&1 +apt-get install -y -qq libicu72 || apt-get install -y -qq libicu74 || apt-get install -y -qq libicu76 || apt-get install -y -qq "libicu*" > /dev/null 2>&1 + +echo "" +echo "=== Creating non-root user ===" +useradd -m -s /bin/bash pguser + +# Copy SDK to writable location (excluding any existing bin directory with wrong-platform binary) +mkdir -p /home/pguser/sdk +cp -r /sdk-src/pg0 /home/pguser/sdk/ +cp /sdk-src/pyproject.toml /sdk-src/hatch_build.py /sdk-src/README.md /home/pguser/sdk/ +rm -rf /home/pguser/sdk/pg0/bin # Remove any existing binary +chown -R pguser:pguser /home/pguser/sdk + +echo "" +echo "=== Switching to non-root user ===" +su - pguser << EOF +set -e + +echo "=== Installing Python SDK (will download correct binary) ===" +cd /home/pguser/sdk +pip install --user . -q + +echo "" +echo "=== Testing Python SDK ===" +python3 << PYEOF +from pg0 import Pg0, _get_bundled_binary + +# Check bundled binary +bundled = _get_bundled_binary() +print(f"Bundled binary: {bundled}") +assert bundled is not None, "Bundled binary not found!" +assert bundled.exists(), f"Bundled binary does not exist: {bundled}" + +# Start PostgreSQL +print("") +print("=== Starting PostgreSQL ===") +pg = Pg0() +info = pg.start() +print(f"PostgreSQL running on port {info.port}") +print(f"URI: {info.uri}") + +# Verify it is actually running +info = pg.info() +print(f"Verified running: {info.running}") +if not info.running: + raise RuntimeError("PostgreSQL is not running after start!") + +# Test basic query +print("") +print("=== Testing basic SELECT query ===") +result = pg.execute("SELECT version();") +print(result.strip().split("\\n")[0][:80]) + +# Test table operations +print("") +print("=== Testing table creation and data ===") +pg.execute("CREATE TABLE test (id INT, name TEXT);") +pg.execute("INSERT INTO test VALUES (1, '\''Hello'\''), (2, '\''World'\'');") +result = pg.execute("SELECT * FROM test;") +print(result) + +# Test pgvector +print("") +print("=== Testing pgvector extension ===") +try: + pg.execute("CREATE EXTENSION IF NOT EXISTS vector;") + print("pgvector extension created successfully") + pg.execute("CREATE TABLE embeddings (id INT, vec vector(3));") + pg.execute("INSERT INTO embeddings VALUES (1, '\''[1,2,3]'\'');") + result = pg.execute("SELECT * FROM embeddings;") + print(result) + print("pgvector working correctly") +except Exception as e: + print(f"pgvector failed: {e}") + +# Stop PostgreSQL +print("") +print("=== Stopping PostgreSQL ===") +pg.stop() +pg.drop() + +print("") +print("==================================") +print("ALL TESTS PASSED - Debian AMD64 Python SDK") +print("==================================") +PYEOF +EOF +' + +echo "" +echo "Test completed successfully!" diff --git a/docker-tests/python/test_debian_arm64.sh b/docker-tests/python/test_debian_arm64.sh new file mode 100755 index 0000000..20c33b6 --- /dev/null +++ b/docker-tests/python/test_debian_arm64.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -e + +echo "==================================" +echo "Testing pg0 Python SDK on Debian ARM64" +echo "Image: python:3.11-slim" +echo "Platform: linux/arm64" +echo "==================================" + +# Get the script directory to find the SDK +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SDK_DIR="$SCRIPT_DIR/../../sdk/python" + +docker run --rm --platform=linux/arm64 \ + -v "$SDK_DIR:/sdk-src:ro" \ + python:3.11-slim bash -c ' +set -e + +echo "=== System Info ===" +uname -m +cat /etc/os-release | grep PRETTY_NAME + +echo "" +echo "=== Installing system dependencies ===" +apt-get update -qq +# procps is needed for pg0 to check if postgres process is running +apt-get install -y -qq libxml2 libssl3 libgssapi-krb5-2 procps > /dev/null 2>&1 +apt-get install -y -qq libicu72 || apt-get install -y -qq libicu74 || apt-get install -y -qq libicu76 || apt-get install -y -qq "libicu*" > /dev/null 2>&1 + +echo "" +echo "=== Creating non-root user ===" +useradd -m -s /bin/bash pguser + +# Copy SDK to writable location (excluding any existing bin directory with wrong-platform binary) +mkdir -p /home/pguser/sdk +cp -r /sdk-src/pg0 /home/pguser/sdk/ +cp /sdk-src/pyproject.toml /sdk-src/hatch_build.py /sdk-src/README.md /home/pguser/sdk/ +rm -rf /home/pguser/sdk/pg0/bin # Remove any existing binary +chown -R pguser:pguser /home/pguser/sdk + +echo "" +echo "=== Switching to non-root user ===" +su - pguser << EOF +set -e + +echo "=== Installing Python SDK (will download correct binary) ===" +cd /home/pguser/sdk +pip install --user . -q + +echo "" +echo "=== Testing Python SDK ===" +python3 << PYEOF +from pg0 import Pg0, _get_bundled_binary + +# Check bundled binary +bundled = _get_bundled_binary() +print(f"Bundled binary: {bundled}") +assert bundled is not None, "Bundled binary not found!" +assert bundled.exists(), f"Bundled binary does not exist: {bundled}" + +# Start PostgreSQL +print("") +print("=== Starting PostgreSQL ===") +pg = Pg0() +info = pg.start() +print(f"PostgreSQL running on port {info.port}") +print(f"URI: {info.uri}") + +# Test basic query +print("") +print("=== Testing basic SELECT query ===") +result = pg.execute("SELECT version();") +print(result.strip().split("\\n")[0][:80]) + +# Test table operations +print("") +print("=== Testing table creation and data ===") +pg.execute("CREATE TABLE test (id INT, name TEXT);") +pg.execute("INSERT INTO test VALUES (1, '\''Hello'\''), (2, '\''World'\'');") +result = pg.execute("SELECT * FROM test;") +print(result) + +# Test pgvector +print("") +print("=== Testing pgvector extension ===") +try: + pg.execute("CREATE EXTENSION IF NOT EXISTS vector;") + print("✅ pgvector extension created successfully") + pg.execute("CREATE TABLE embeddings (id INT, vec vector(3));") + pg.execute("INSERT INTO embeddings VALUES (1, '\''[1,2,3]'\'');") + result = pg.execute("SELECT * FROM embeddings;") + print(result) + print("✅ pgvector working correctly") +except Exception as e: + print(f"⚠️ pgvector failed: {e}") + +# Stop PostgreSQL +print("") +print("=== Stopping PostgreSQL ===") +pg.stop() +pg.drop() + +print("") +print("==================================") +print("✅ ALL TESTS PASSED - Debian ARM64 Python SDK") +print("==================================") +PYEOF +EOF +' + +echo "" +echo "✅ Test completed successfully!" diff --git a/docker-tests/test_alpine_amd64.sh b/docker-tests/test_alpine_amd64.sh index 0c6c95d..90bd212 100755 --- a/docker-tests/test_alpine_amd64.sh +++ b/docker-tests/test_alpine_amd64.sh @@ -24,7 +24,7 @@ cat /etc/os-release | grep PRETTY_NAME echo "" echo "=== Installing dependencies ===" -apk add --no-cache curl bash sudo procps shadow icu-libs lz4-libs libxml2 > /dev/null 2>&1 +apk add --no-cache curl bash sudo procps shadow icu-libs lz4-libs libxml2 zstd-libs > /dev/null 2>&1 echo "" echo "=== Creating non-root user ===" diff --git a/docker-tests/test_alpine_arm64.sh b/docker-tests/test_alpine_arm64.sh index e143376..d411274 100755 --- a/docker-tests/test_alpine_arm64.sh +++ b/docker-tests/test_alpine_arm64.sh @@ -24,7 +24,7 @@ cat /etc/os-release | grep PRETTY_NAME echo "" echo "=== Installing dependencies ===" -apk add --no-cache curl bash sudo procps shadow icu-libs lz4-libs libxml2 > /dev/null 2>&1 +apk add --no-cache curl bash sudo procps shadow icu-libs lz4-libs libxml2 zstd-libs > /dev/null 2>&1 echo "" echo "=== Creating non-root user ===" diff --git a/sdk/node/README.md b/sdk/node/README.md index b9cff5d..3d9792a 100644 --- a/sdk/node/README.md +++ b/sdk/node/README.md @@ -1,8 +1,8 @@ -# pg0 - Embedded PostgreSQL for Node.js +# pg0 - PostgreSQL for Node.js [![npm](https://badge.fury.io/js/@vectorize-io%2Fpg0.svg)](https://www.npmjs.com/package/@vectorize-io/pg0) -Embedded PostgreSQL with pgvector. No installation, no Docker, no configuration. +Zero-config PostgreSQL with pgvector. No installation, no Docker, no configuration. ## Install diff --git a/sdk/python/README.md b/sdk/python/README.md index 12be2dd..383ccf5 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1,8 +1,8 @@ -# pg0 - Embedded PostgreSQL for Python +# pg0 - PostgreSQL for Python [![PyPI](https://badge.fury.io/py/pg0-embedded.svg)](https://pypi.org/project/pg0-embedded/) -Embedded PostgreSQL with pgvector. No installation, no Docker, no configuration. +Zero-config PostgreSQL with pgvector. No installation, no Docker, no configuration. ## Install @@ -79,6 +79,22 @@ print(info.username) # postgres print(info.database) # postgres ``` +## Supported Platforms + +Pre-built wheels are available for: + +| Platform | Architecture | Wheel Tag | +|----------|--------------|-----------| +| macOS | Apple Silicon (M1/M2/M3) | `macosx_14_0_arm64` | +| Linux | x86_64 (glibc) | `manylinux_2_35_x86_64` | +| Linux | ARM64 (glibc) | `manylinux_2_35_aarch64` | +| Windows | x64 | `win_amd64` | + +For other platforms, install from source (requires Rust toolchain): +```bash +pip install pg0-embedded --no-binary pg0-embedded +``` + ## Links - [GitHub](https://github.com/vectorize-io/pg0) diff --git a/sdk/python/build_binary.py b/sdk/python/build_binary.py new file mode 100644 index 0000000..6fb85e0 --- /dev/null +++ b/sdk/python/build_binary.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +""" +Build script to download the pg0 binary for the current platform. +This is run during wheel building to bundle the binary into the package. +""" + +import hashlib +import os +import platform +import stat +import subprocess +import sys +import urllib.request +from pathlib import Path + +# These are updated with each release +PG0_VERSION = "v0.9.0" +PG0_REPO = "vectorize-io/pg0" + +# SHA256 checksums for each binary (updated with each release) +# To generate: sha256sum pg0- +CHECKSUMS = { + "darwin-aarch64": "", # Will be populated by CI + "linux-x86_64-gnu": "", + "linux-x86_64-musl": "", + "linux-aarch64-gnu": "", + "linux-aarch64-musl": "", + "windows-x86_64": "", +} + + +def get_platform() -> str: + """Detect the current platform.""" + system = platform.system().lower() + machine = platform.machine().lower() + + if system == "darwin": + return "darwin-aarch64" + elif system == "linux": + if machine in ("x86_64", "amd64"): + arch = "x86_64" + elif machine in ("aarch64", "arm64"): + arch = "aarch64" + else: + raise RuntimeError(f"Unsupported architecture: {machine}") + + # Detect musl vs glibc + try: + result = subprocess.run( + ["ldd", "--version"], + capture_output=True, + text=True, + ) + if "musl" in (result.stdout + result.stderr).lower(): + return f"linux-{arch}-musl" + except FileNotFoundError: + pass + + # Check for musl loader + if Path(f"/lib/ld-musl-{arch}.so.1").exists(): + return f"linux-{arch}-musl" + + return f"linux-{arch}-gnu" + elif system == "windows": + return "windows-x86_64" + else: + raise RuntimeError(f"Unsupported platform: {system}") + + +def download_binary(target_dir: Path, plat: str | None = None) -> Path: + """Download the pg0 binary for the specified platform.""" + if plat is None: + plat = get_platform() + + ext = ".exe" if plat.startswith("windows") else "" + filename = f"pg0-{plat}{ext}" + url = f"https://github.com/{PG0_REPO}/releases/download/{PG0_VERSION}/{filename}" + + target_dir.mkdir(parents=True, exist_ok=True) + binary_path = target_dir / f"pg0{ext}" + + print(f"Downloading pg0 {PG0_VERSION} for {plat}...") + print(f" URL: {url}") + + # Download to temp file first + tmp_path = binary_path.with_suffix(".tmp") + urllib.request.urlretrieve(url, tmp_path) + + # Verify checksum if available + expected_checksum = CHECKSUMS.get(plat, "") + if expected_checksum: + with open(tmp_path, "rb") as f: + actual_checksum = hashlib.sha256(f.read()).hexdigest() + if actual_checksum != expected_checksum: + tmp_path.unlink() + raise RuntimeError( + f"Checksum mismatch for {plat}!\n" + f" Expected: {expected_checksum}\n" + f" Actual: {actual_checksum}" + ) + print(f" Checksum verified: {actual_checksum[:16]}...") + else: + print(" Warning: No checksum available for verification") + + # Move to final location + tmp_path.rename(binary_path) + + # Make executable on Unix + if not plat.startswith("windows"): + binary_path.chmod(binary_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + print(f" Saved to: {binary_path}") + return binary_path + + +def main(): + """Download binary for current platform into pg0/bin/.""" + script_dir = Path(__file__).parent + bin_dir = script_dir / "pg0" / "bin" + + # Allow overriding platform via environment variable (for CI cross-builds) + plat = os.environ.get("PG0_TARGET_PLATFORM") + + download_binary(bin_dir, plat) + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/sdk/python/hatch_build.py b/sdk/python/hatch_build.py new file mode 100644 index 0000000..360694b --- /dev/null +++ b/sdk/python/hatch_build.py @@ -0,0 +1,182 @@ +""" +Hatch build hook to include the pg0 binary in the wheel. + +The binary can come from: +1. PG0_BINARY_PATH env var - path to a pre-built binary (for CI/release) +2. Local cargo build - builds from source using cargo (default for local dev) +3. GitHub releases - downloads from releases (fallback, requires PG0_VERSION) +""" + +import hashlib +import os +import platform +import shutil +import stat +import subprocess +import urllib.request +from pathlib import Path +from typing import Any + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +# GitHub repo for downloading releases (fallback only) +PG0_REPO = "vectorize-io/pg0" + + +def get_platform() -> str: + """Detect the current platform.""" + system = platform.system().lower() + machine = platform.machine().lower() + + if system == "darwin": + return "darwin-aarch64" if machine == "arm64" else "darwin-x86_64" + elif system == "linux": + if machine in ("x86_64", "amd64"): + arch = "x86_64" + elif machine in ("aarch64", "arm64"): + arch = "aarch64" + else: + raise RuntimeError(f"Unsupported architecture: {machine}") + + # Detect musl vs glibc + try: + result = subprocess.run( + ["ldd", "--version"], + capture_output=True, + text=True, + ) + if "musl" in (result.stdout + result.stderr).lower(): + return f"linux-{arch}-musl" + except FileNotFoundError: + pass + + # Check for musl loader + if Path(f"/lib/ld-musl-{arch}.so.1").exists(): + return f"linux-{arch}-musl" + + return f"linux-{arch}-gnu" + elif system == "windows": + return "windows-x86_64" + else: + raise RuntimeError(f"Unsupported platform: {system}") + + +def build_binary_locally(target_dir: Path) -> Path: + """Build pg0 binary from source using cargo.""" + # Find the repo root (sdk/python -> repo root) + repo_root = Path(__file__).parent.parent.parent + + cargo_toml = repo_root / "Cargo.toml" + if not cargo_toml.exists(): + raise RuntimeError(f"Cargo.toml not found at {cargo_toml}") + + print("Building pg0 binary from source...") + print(f" Repo root: {repo_root}") + + # Build with cargo + env = os.environ.copy() + env["BUNDLE_POSTGRESQL"] = "true" + + result = subprocess.run( + ["cargo", "build", "--release"], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f" stdout: {result.stdout}") + print(f" stderr: {result.stderr}") + raise RuntimeError(f"Cargo build failed: {result.stderr}") + + # Find the built binary + system = platform.system().lower() + binary_name = "pg0.exe" if system == "windows" else "pg0" + built_binary = repo_root / "target" / "release" / binary_name + + if not built_binary.exists(): + raise RuntimeError(f"Built binary not found at {built_binary}") + + # Copy to target directory + target_dir.mkdir(parents=True, exist_ok=True) + target_path = target_dir / binary_name + shutil.copy2(built_binary, target_path) + + # Make executable on Unix + if system != "windows": + target_path.chmod(target_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + print(f" Built binary copied to: {target_path}") + return target_path + + +def download_binary(target_dir: Path, plat: str, version: str) -> Path: + """Download the pg0 binary from GitHub releases.""" + ext = ".exe" if plat.startswith("windows") else "" + filename = f"pg0-{plat}{ext}" + url = f"https://github.com/{PG0_REPO}/releases/download/{version}/{filename}" + + target_dir.mkdir(parents=True, exist_ok=True) + binary_path = target_dir / f"pg0{ext}" + + print(f"Downloading pg0 {version} for {plat}...") + print(f" URL: {url}") + + # Download to temp file first + tmp_path = binary_path.with_suffix(".tmp") + urllib.request.urlretrieve(url, tmp_path) + + # Move to final location + tmp_path.rename(binary_path) + + # Make executable on Unix + if not plat.startswith("windows"): + binary_path.chmod(binary_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + print(f" Saved to: {binary_path}") + return binary_path + + +class CustomBuildHook(BuildHookInterface): + """Build hook to include pg0 binary in wheel build.""" + + PLUGIN_NAME = "custom" + + def initialize(self, version: str, build_data: dict[str, Any]) -> None: + """Called before the build starts.""" + if self.target_name != "wheel": + # Only include binary for wheel builds, not sdist + return + + root = Path(self.root) + bin_dir = root / "pg0" / "bin" + system = platform.system().lower() + ext = ".exe" if system == "windows" else "" + binary_path = bin_dir / f"pg0{ext}" + + # Check if binary already exists + if binary_path.exists(): + print(f"Binary already exists: {binary_path}") + # Option 1: Use pre-built binary from env var (for CI) + elif os.environ.get("PG0_BINARY_PATH"): + src_path = Path(os.environ["PG0_BINARY_PATH"]) + if not src_path.exists(): + raise RuntimeError(f"PG0_BINARY_PATH does not exist: {src_path}") + print(f"Using pre-built binary from PG0_BINARY_PATH: {src_path}") + bin_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, binary_path) + if system != "windows": + binary_path.chmod(binary_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + # Option 2: Download from GitHub releases (requires PG0_VERSION) + elif os.environ.get("PG0_VERSION"): + plat = os.environ.get("PG0_TARGET_PLATFORM") or get_platform() + download_binary(bin_dir, plat, os.environ["PG0_VERSION"]) + # Option 3: Build locally from source (default for local dev) + else: + build_binary_locally(bin_dir) + + # Tell hatch to include the bin directory + if "force_include" not in build_data: + build_data["force_include"] = {} + build_data["force_include"][str(bin_dir)] = "pg0/bin" diff --git a/sdk/python/pg0/__init__.py b/sdk/python/pg0/__init__.py index a40030e..2e9fa2e 100644 --- a/sdk/python/pg0/__init__.py +++ b/sdk/python/pg0/__init__.py @@ -82,6 +82,16 @@ def from_dict(cls, data: dict) -> "InstanceInfo": ) +def _get_bundled_binary() -> Optional[Path]: + """Get the path to the bundled pg0 binary, if it exists.""" + package_dir = Path(__file__).parent + binary_name = "pg0.exe" if sys.platform == "win32" else "pg0" + bundled_path = package_dir / "bin" / binary_name + if bundled_path.exists(): + return bundled_path + return None + + def _get_install_dir() -> Path: """Get the directory where pg0 binary should be installed.""" # Use ~/.local/bin on Unix, or a pg0-specific dir @@ -96,17 +106,25 @@ def install(force: bool = False) -> Path: """ Install the pg0 binary using the official install script. + Note: If the package was installed from a platform-specific wheel, + the binary is already bundled and this function returns immediately. + Args: force: Force reinstall even if already installed Returns: Path to the installed binary """ + # Check for bundled binary first (from platform-specific wheel) + bundled = _get_bundled_binary() + if bundled and not force: + return bundled + install_dir = _get_install_dir() binary_name = "pg0.exe" if sys.platform == "win32" else "pg0" binary_path = install_dir / binary_name - # Check if already installed + # Check if already installed externally if binary_path.exists() and not force: return binary_path @@ -150,7 +168,12 @@ def install(force: bool = False) -> Path: def _find_pg0() -> str: """Find the pg0 binary, installing if necessary.""" - # Check PATH first + # Check for bundled binary first (from platform-specific wheel) + bundled = _get_bundled_binary() + if bundled: + return str(bundled) + + # Check PATH path = shutil.which("pg0") if path: return path @@ -163,8 +186,8 @@ def _find_pg0() -> str: if binary_path.exists(): return str(binary_path) - # Auto-install - installed_path = install(version=None) + # Auto-install as fallback + installed_path = install() return str(installed_path) @@ -482,4 +505,5 @@ def info(name: str = "default") -> InstanceInfo: "stop", "drop", "info", + "_get_bundled_binary", # for testing ] diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 4ac62db..8d42f97 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -31,3 +31,10 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["pg0"] +# Include the bundled binary in the wheel (artifacts are files that should be +# included even if they're in .gitignore) +artifacts = ["pg0/bin/*"] + +# Custom build hook to download the binary before building (wheel only) +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index c4f226d..f9de732 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -65,7 +65,7 @@ wheels = [ [[package]] name = "pg0-embedded" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } [package.optional-dependencies] diff --git a/src/main.rs b/src/main.rs index 1050475..ae3378b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -579,7 +579,7 @@ fn start( installation_dir: version_install_dir, configuration, trust_installation_dir: true, // Use our extracted files - timeout: Some(std::time::Duration::from_secs(300)), // 5 minute timeout for slow systems (ARM64 emulation) + timeout: Some(std::time::Duration::from_secs(600)), // 10 minute timeout for slow systems (ARM64 emulation under QEMU) ..Default::default() };