diff --git a/.github/workflows/generated-build-and-test.yaml b/.github/workflows/generated-build-and-publish-no-win.yaml similarity index 94% rename from .github/workflows/generated-build-and-test.yaml rename to .github/workflows/generated-build-and-publish-no-win.yaml index 4d067ca..23c32c1 100644 --- a/.github/workflows/generated-build-and-test.yaml +++ b/.github/workflows/generated-build-and-publish-no-win.yaml @@ -1,4 +1,4 @@ -name: Build and Test ๐Ÿ›  +name: Generated Build libpg-query no windows ๐Ÿ›  'on': workflow_dispatch: null jobs: @@ -17,10 +17,6 @@ jobs: with: node-version: 20.x cache: yarn - - name: Configure Git ๐Ÿ›  - run: | - git config user.name "Cosmology" - git config user.email "developers@cosmology.zone" - name: Install and Build ๐Ÿ“ฆ run: | yarn diff --git a/.github/workflows/generated-build-and-publish.yaml b/.github/workflows/generated-build-and-publish.yaml new file mode 100644 index 0000000..dd67718 --- /dev/null +++ b/.github/workflows/generated-build-and-publish.yaml @@ -0,0 +1,93 @@ +name: Generated Build libpg-query ๐Ÿ›  +'on': + workflow_dispatch: null +jobs: + build-artifacts: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - windows-latest + - macos-latest + - ubuntu-latest + steps: + - name: Checkout Repository ๐Ÿ“ฅ + uses: actions/checkout@v4 + - name: Setup Node.js ๐ŸŒ + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: yarn + - name: Install and Build ๐Ÿ“ฆ + run: | + yarn + yarn binary:build + - name: Save Artifacts For Supabase CDN ๐Ÿ— + uses: actions/upload-artifact@v4 + with: + name: build-supabase-artifact-${{ matrix.os }} + path: ./build/stage/libpg-query-node/ + - name: Save Artifacts For NPM ๐Ÿ— + uses: actions/upload-artifact@v4 + with: + name: build-npm-artifact-${{ matrix.os }} + path: | + ${{ matrix.os == 'macos-latest' && './libpg_query/osx/libpg_query.a' || + matrix.os == 'ubuntu-latest' && './libpg_query/linux/libpg_query.a' || + matrix.os == 'windows-latest' && './libpg_query/windows/pg_query.lib' }} + build-wasm: + needs: build-artifacts + runs-on: ubuntu-latest + steps: + - name: Checkout Repository ๐Ÿ“ฅ + uses: actions/checkout@v4 + - name: Setup Node.js ๐ŸŒ + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: yarn + - name: Install and Build ๐Ÿš€ + run: | + yarn + - name: Install Emscripten โœ๐Ÿป + run: | + sudo apt-get update + sudo apt-get install cmake python3 python3-pip + git clone --branch 3.1.59 --depth 1 https://github.com/emscripten-core/emsdk.git + cd emsdk + ./emsdk install 3.1.59 + ./emsdk activate 3.1.59 + source ./emsdk_env.sh + - name: Build with Emscripten ๐Ÿ— + run: | + source ./emsdk/emsdk_env.sh + emmake make + emmake make build + - name: Archive production artifacts ๐Ÿ› + uses: actions/upload-artifact@v4 + with: + name: wasm-artifacts + path: wasm + prepare-and-publish: + needs: build-wasm + runs-on: ubuntu-latest + steps: + - name: Checkout Repository ๐Ÿ“ฅ + uses: actions/checkout@v4 + - name: Get Artifacts ๐Ÿ“š + uses: actions/download-artifact@v4 + with: + path: downloaded-artifacts + - name: Prepare artifacts ๐Ÿ“ฆ + run: | + find ./downloaded-artifacts/ + cp ./downloaded-artifacts/build-npm-artifact-windows-latest/pg_query.lib ./libpg_query/windows/queryparser.lib + cp ./downloaded-artifacts/build-npm-artifact-ubuntu-latest/libpg_query.a ./libpg_query/linux/libpg_query.a + cp ./downloaded-artifacts/build-npm-artifact-macos-latest/libpg_query.a ./libpg_query/osx/libpg_query.a + cp ./downloaded-artifacts/wasm-artifacts/libpg-query.js ./wasm/libpg-query.js + cp ./downloaded-artifacts/wasm-artifacts/libpg-query.wasm ./wasm/libpg-query.wasm + rm -rf ./downloaded-artifacts + - name: Log + run: | + find ./libpg_query/ + find ./wasm diff --git a/.github/workflows/run-tests-linux.yml b/.github/workflows/run-tests-linux.yml index f55b061..21dbbb3 100644 --- a/.github/workflows/run-tests-linux.yml +++ b/.github/workflows/run-tests-linux.yml @@ -1,8 +1,9 @@ name: Linux ๐Ÿงช on: + push: pull_request: - types: [opened, push, reopened] + types: [opened, synchronize, reopened] workflow_dispatch: jobs: diff --git a/.github/workflows/run-tests-mac.yml b/.github/workflows/run-tests-mac.yml index fa26f4a..3bdb102 100644 --- a/.github/workflows/run-tests-mac.yml +++ b/.github/workflows/run-tests-mac.yml @@ -1,8 +1,9 @@ name: Mac ๐Ÿงช on: + push: pull_request: - types: [opened, push, reopened] + types: [opened, synchronize, reopened] workflow_dispatch: jobs: diff --git a/.github/workflows/run-tests-win.yml b/.github/workflows/run-tests-win.yml index 03f1fdc..ac6d83f 100644 --- a/.github/workflows/run-tests-win.yml +++ b/.github/workflows/run-tests-win.yml @@ -1,8 +1,9 @@ name: Windows ๐Ÿ›  on: + push: pull_request: - types: [opened, push, reopened] + types: [opened, synchronize, reopened] workflow_dispatch: jobs: diff --git a/.templates/buildAddon.bat b/.templates/buildAddon.bat new file mode 100644 index 0000000..c1c8c10 --- /dev/null +++ b/.templates/buildAddon.bat @@ -0,0 +1,90 @@ +@echo off + +set LIBPG_REPO=___LIBPG_REPO___ +set LIBPG_COMMIT=___LIBPG_COMMIT___ +set LIBPG_BRANCH=___LIBPG_BRANCH___ + +:: Check if each required variable is set +if "%LIBPG_REPO%"=="" ( + echo ERROR: LIBPG_REPO variable is not set. + exit /B 1 +) + +if "%LIBPG_COMMIT%"=="" ( + echo ERROR: LIBPG_COMMIT variable is not set. + exit /B 1 +) + +if "%LIBPG_BRANCH%"=="" ( + echo ERROR: LIBPG_BRANCH variable is not set. + exit /B 1 +) + +:: The environment variables must be set +echo Using repository: %LIBPG_REPO% +echo Using commit: %LIBPG_COMMIT% +echo Using branch: %LIBPG_BRANCH% + +setlocal enabledelayedexpansion + +rem Remember current's parent directory and create a new, unique, temporary directory +set buildDir=%cd% +set projectDir=%cd%\.. +set tmpDir=%temp%\tmpdir.libpg_query +rmdir /s /q %tmpDir% +md %tmpDir% + + +rem Define the make target +set makeTarget=build + +rem Change to the newly created temp directory +cd /D %tmpDir% + + +rem Clone the selected branch of the libpg_query Git repo +git clone -b %LIBPG_BRANCH% --single-branch %LIBPG_REPO% +cd libpg_query + +rem Checkout the desired commit +git checkout %LIBPG_COMMIT% + +rem needed if being invoked from within gyp +set MAKEFLAGS= +set MFLAGS= + +rem set path with Windows Developer Command Prompt +echo "please ensure you are running at Windows Developer Command Prompt environments" +nmake /F Makefile.msvc clean +nmake /F Makefile.msvc build + + +rem Terminate if build fails +if %errorlevel% NEQ 0 ( + echo ERROR: 'nmake' command failed +) + +rem Search for pg_query.obj (libpg_query.a), error if not found +for /f "delims=" %%f in ('dir /b /s pg_query.lib') do set file=%%f +if not defined file ( + echo "ERROR: pg_query.lib not found" + +) + +rem Error if pg_query.h is missing +for /f "delims=" %%f in ('dir /b /s pg_query.h') do set file=%%f +if not defined file ( + echo "ERROR: pg_query.h not found" + +) + +rem Copy pg_query.lib to windows dir +copy /Y pg_query.lib "%projectDir%\libpg_query\windows\" + +rem Copy header +copy /Y pg_query.h "%projectDir%\libpg_query\include\" + +rem Cleanup: revert to original directory +cd /D %buildDir% + +exit /B 0 \ No newline at end of file diff --git a/.templates/buildAddon.sh b/.templates/buildAddon.sh new file mode 100644 index 0000000..ee9a574 --- /dev/null +++ b/.templates/buildAddon.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +LIBPG_REPO=___LIBPG_REPO___ +LIBPG_COMMIT=___LIBPG_COMMIT___ +LIBPG_BRANCH=___LIBPG_BRANCH___ + +# Check if variables are set and exit if not +if [ -z "$LIBPG_COMMIT" ]; then + echo "ERROR: LIBPG_COMMIT variable is not set." + exit 1 +fi + +if [ -z "$LIBPG_BRANCH" ]; then + echo "ERROR: LIBPG_BRANCH variable is not set." + exit 1 +fi + +if [ -z "$LIBPG_REPO" ]; then + echo "ERROR: LIBPG_REPO variable is not set." + exit 1 +fi + +# Remember current directory and create a new, unique, temporary directory +rDIR=$(pwd) +tmpDir=$(mktemp -d 2>/dev/null || mktemp -d -t 'tmpdir.XXXX') + +# Define the make target +makeTarget=build + +# Change to the newly created temp directory +cd "$tmpDir" + +# Clone the selected branch of the libpg_query Git repo +git clone -b $LIBPG_BRANCH --single-branch $LIBPG_REPO +cd libpg_query + +# Checkout the desired commit +git checkout $LIBPG_COMMIT + +# needed if being invoked from within gyp +unset MAKEFLAGS +unset MFLAGS + +# Adaptively build for macOS or Linux +if [ "$(uname)" == "Darwin" ]; then + make CFLAGS='-mmacosx-version-min=10.7' PG_CFLAGS='-mmacosx-version-min=10.7' $makeTarget +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + make CFLAGS='' PG_CFLAGS='' $makeTarget +fi + +# Terminate if build fails +if [ $? -ne 0 ]; then + echo "ERROR: 'make' command failed"; + exit 1; +fi + +# Search for libpg_query.a, error if not found +file=$(ls | grep 'libpg_query.a') +if [ ! $file ]; then + echo "ERROR: libpg_query.a not found"; + exit 1; +fi + +# Error if pg_query.h is missing +file=$(ls | grep 'pg_query.h') +if [ ! $file ]; then + echo "ERROR: pg_query.h not found"; + exit 1; +fi + +# Copy queryparser.cc, binding.gyp to current directory +if [ "$(uname)" == "Darwin" ]; then + cp $(pwd)/libpg_query.a $rDIR/libpg_query/osx/ +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + cp $(pwd)/libpg_query.a $rDIR/libpg_query/linux/ +fi + +# Copy header +cp $(pwd)/pg_query.h $rDIR/libpg_query/include/ +cp $(pwd)/protobuf/*.proto $rDIR/libpg_query/protobuf/ + +# Cleanup: revert to original directory and remove the temp +cd "$rDIR" +rm -rf "$tmpDir" diff --git a/.yamlize/versions/14-latest.yaml b/.yamlize/versions/14-latest.yaml new file mode 100644 index 0000000..90ac91a --- /dev/null +++ b/.yamlize/versions/14-latest.yaml @@ -0,0 +1,7 @@ +env: + JOB: pg-14 + LIBPG_REPO: https://github.com/pganalyze/libpg_query.git + LIBPG_COMMIT: 1577ef7c6c349542149e34ffbfafc244ab942ec6 + LIBPG_BRANCH: 14-latest + PGSQL_TYPES: 14.0.0 + OPERATING_SYSTEMS: [linux,mac] \ No newline at end of file diff --git a/.yamlize/versions/15-latest.yaml b/.yamlize/versions/15-latest.yaml new file mode 100644 index 0000000..e587c57 --- /dev/null +++ b/.yamlize/versions/15-latest.yaml @@ -0,0 +1,7 @@ +env: + JOB: pg-15 + LIBPG_REPO: https://github.com/pganalyze/libpg_query.git + LIBPG_COMMIT: db39825bc7c1ddd45962ec6a626d740b7f8f027a + LIBPG_BRANCH: 15-latest + PGSQL_TYPES: 15.0.2 + OPERATING_SYSTEMS: [linux,mac] \ No newline at end of file diff --git a/.yamlize/versions/16-latest.yaml b/.yamlize/versions/16-latest.yaml new file mode 100644 index 0000000..b0abbe4 --- /dev/null +++ b/.yamlize/versions/16-latest.yaml @@ -0,0 +1,7 @@ +env: + JOB: pg-16 + LIBPG_REPO: https://github.com/pganalyze/libpg_query.git + LIBPG_COMMIT: 1ec38940e5c6f09a4c1d17a46d839a881c4f2db7 + LIBPG_BRANCH: 16-latest + PGSQL_TYPES: 16.0.0 + OPERATING_SYSTEMS: [linux,mac,win] \ No newline at end of file diff --git a/.yamlize/workflows/build-and-test.yaml b/.yamlize/workflows/build-and-publish-no-win.yaml similarity index 57% rename from .yamlize/workflows/build-and-test.yaml rename to .yamlize/workflows/build-and-publish-no-win.yaml index aaf9a11..73958f4 100644 --- a/.yamlize/workflows/build-and-test.yaml +++ b/.yamlize/workflows/build-and-publish-no-win.yaml @@ -1,11 +1,11 @@ -name: Build and Test ๐Ÿ›  +name: Build and Publish ๐Ÿš€ on: workflow_dispatch: jobs: build-artifacts: - import-yaml: yaml/build-artifacts.yaml + import-yaml: yaml/build-artifacts-no-win.yaml build-wasm: needs: build-artifacts @@ -13,4 +13,4 @@ jobs: prepare-and-publish: needs: build-wasm - import-yaml: yaml/prepare-and-log.yaml \ No newline at end of file + import-yaml: yaml/prepare-and-publish-no-win.yaml \ No newline at end of file diff --git a/.yamlize/workflows/yaml/artifacts/npm-no-win.yaml b/.yamlize/workflows/yaml/artifacts/npm-no-win.yaml new file mode 100644 index 0000000..f9b9fb1 --- /dev/null +++ b/.yamlize/workflows/yaml/artifacts/npm-no-win.yaml @@ -0,0 +1,7 @@ +name: Save Artifacts For NPM ๐Ÿ— +uses: actions/upload-artifact@v4 +with: + name: build-npm-artifact-${{ matrix.os }} + path: | + ${{ matrix.os == 'macos-latest' && './libpg_query/osx/libpg_query.a' || + matrix.os == 'ubuntu-latest' && './libpg_query/linux/libpg_query.a' }} \ No newline at end of file diff --git a/.yamlize/workflows/yaml/artifacts/npm.yaml b/.yamlize/workflows/yaml/artifacts/npm.yaml index f9b9fb1..a43be0d 100644 --- a/.yamlize/workflows/yaml/artifacts/npm.yaml +++ b/.yamlize/workflows/yaml/artifacts/npm.yaml @@ -4,4 +4,5 @@ with: name: build-npm-artifact-${{ matrix.os }} path: | ${{ matrix.os == 'macos-latest' && './libpg_query/osx/libpg_query.a' || - matrix.os == 'ubuntu-latest' && './libpg_query/linux/libpg_query.a' }} \ No newline at end of file + matrix.os == 'ubuntu-latest' && './libpg_query/linux/libpg_query.a' || + matrix.os == 'windows-latest' && './libpg_query/windows/pg_query.lib' }} \ No newline at end of file diff --git a/.yamlize/workflows/yaml/artifacts/prepare-no-win.yaml b/.yamlize/workflows/yaml/artifacts/prepare-no-win.yaml new file mode 100644 index 0000000..63f3305 --- /dev/null +++ b/.yamlize/workflows/yaml/artifacts/prepare-no-win.yaml @@ -0,0 +1,8 @@ +name: Prepare artifacts ๐Ÿ“ฆ +run: | + find ./downloaded-artifacts/ + cp ./downloaded-artifacts/build-npm-artifact-ubuntu-latest/libpg_query.a ./libpg_query/linux/libpg_query.a + cp ./downloaded-artifacts/build-npm-artifact-macos-latest/libpg_query.a ./libpg_query/osx/libpg_query.a + cp ./downloaded-artifacts/wasm-artifacts/libpg-query.js ./wasm/libpg-query.js + cp ./downloaded-artifacts/wasm-artifacts/libpg-query.wasm ./wasm/libpg-query.wasm + rm -rf ./downloaded-artifacts diff --git a/.yamlize/workflows/yaml/build-artifacts-no-win.yaml b/.yamlize/workflows/yaml/build-artifacts-no-win.yaml new file mode 100644 index 0000000..86cc04e --- /dev/null +++ b/.yamlize/workflows/yaml/build-artifacts-no-win.yaml @@ -0,0 +1,16 @@ +runs-on: ${{ matrix.os }} +strategy: + matrix: + os: [macos-latest, ubuntu-latest] +steps: + - import-yaml: git/checkout.yaml + - import-yaml: node/setup.yaml + # - import-yaml: git/configure.yaml + + - name: Install and Build ๐Ÿ“ฆ + run: | + yarn + yarn binary:build + + - import-yaml: artifacts/supabase.yaml + - import-yaml: artifacts/npm-no-win.yaml \ No newline at end of file diff --git a/.yamlize/workflows/yaml/build-artifacts.yaml b/.yamlize/workflows/yaml/build-artifacts.yaml index 13e1a9f..38d1fe0 100644 --- a/.yamlize/workflows/yaml/build-artifacts.yaml +++ b/.yamlize/workflows/yaml/build-artifacts.yaml @@ -1,11 +1,11 @@ runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest] + os: [windows-latest, macos-latest, ubuntu-latest] steps: - import-yaml: git/checkout.yaml - import-yaml: node/setup.yaml - - import-yaml: git/configure.yaml + # - import-yaml: git/configure.yaml - name: Install and Build ๐Ÿ“ฆ run: | diff --git a/.yamlize/workflows/yaml/prepare-and-log.yaml b/.yamlize/workflows/yaml/prepare-and-publish-no-win.yaml similarity index 63% rename from .yamlize/workflows/yaml/prepare-and-log.yaml rename to .yamlize/workflows/yaml/prepare-and-publish-no-win.yaml index b47a38c..65c6003 100644 --- a/.yamlize/workflows/yaml/prepare-and-log.yaml +++ b/.yamlize/workflows/yaml/prepare-and-publish-no-win.yaml @@ -10,3 +10,9 @@ steps: run: | find ./libpg_query/ find ./wasm + + # - name: Publish to NPM ๐Ÿš€ + # run: | + # npm publish + # env: + # NODE_AUTH_TOKEN: ${{secrets.NPM_API_KEY}} diff --git a/.yamlize/workflows/yaml/prepare-and-publish.yaml b/.yamlize/workflows/yaml/prepare-and-publish.yaml index 4efd548..65c6003 100644 --- a/.yamlize/workflows/yaml/prepare-and-publish.yaml +++ b/.yamlize/workflows/yaml/prepare-and-publish.yaml @@ -6,8 +6,13 @@ steps: - import-yaml: artifacts/prepare.yaml - - name: Publish to NPM ๐Ÿš€ + - name: Log run: | - npm publish - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_API_KEY}} \ No newline at end of file + find ./libpg_query/ + find ./wasm + + # - name: Publish to NPM ๐Ÿš€ + # run: | + # npm publish + # env: + # NODE_AUTH_TOKEN: ${{secrets.NPM_API_KEY}} diff --git a/package.json b/package.json index 8e5160f..f9dc6f0 100644 --- a/package.json +++ b/package.json @@ -34,12 +34,15 @@ "configure": "node-pre-gyp configure", "install": "node-pre-gyp install --fallback-to-build --loglevel verbose", "rebuild": "node-pre-gyp rebuild --loglevel verbose", - "make:wasm": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", - "build:wasm": "yarn make:wasm build", - "rebuild:wasm": "yarn make:wasm rebuild", - "clean:wasm": "yarn make:wasm clean", - "clean-cache:wasm": "yarn make:wasm clean-cache", - "workflows": "node script/workflows.js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "yarn wasm:make build", + "wasm:rebuild": "yarn wasm:make rebuild", + "wasm:clean": "yarn wasm:make clean", + "wasm:clean-cache": "yarn wasm:make clean-cache", + "generate:workflows:win": "node script/generate-workflows.js", + "generate:workflows:no-win": "node script/generate-non-win-workflows.js", + "generate:workflows": "npm run generate:workflows:win && npm run generate:workflows:no-win", + "generate:build": "node script/utils/generate.js", "test": "mocha --timeout 5000", "binary:build": "node-pre-gyp rebuild package", "binary:publish": "AWS_PROFILE=supabase-dev node-pre-gyp publish" @@ -54,6 +57,7 @@ "@yamlize/cli": "^0.8.0", "chai": "^3.5.0", "emnapi": "^0.43.1", + "js-yaml": "^4.1.0", "lodash": "^4.17.15", "mocha": "^5.2.0", "rimraf": "5.0.0" @@ -61,7 +65,7 @@ "dependencies": { "@emnapi/runtime": "^0.43.1", "@mapbox/node-pre-gyp": "^1.0.8", - "@pgsql/types": "^15.0.1", + "@pgsql/types": "15.0.2", "node-addon-api": "^7.0.0", "node-gyp": "^10.0.1" }, @@ -80,4 +84,4 @@ "host": "https://supabase-public-artifacts-bucket.s3.amazonaws.com", "remote_path": "./libpg-query-node/" } -} +} \ No newline at end of file diff --git a/script/buildAddon.bat b/script/buildAddon.bat index 66a81e4..bc0eb3c 100644 --- a/script/buildAddon.bat +++ b/script/buildAddon.bat @@ -1,7 +1,30 @@ +:: this file is auto-generated, use "yarn generate:build " to rebuild with an env (e.g., pg-15) @echo off -set commit=db39825bc7c1ddd45962ec6a626d740b7f8f027a -set branch=15-latest +set LIBPG_REPO=https://github.com/pganalyze/libpg_query.git +set LIBPG_COMMIT=db39825bc7c1ddd45962ec6a626d740b7f8f027a +set LIBPG_BRANCH=15-latest + +:: Check if each required variable is set +if "%LIBPG_REPO%"=="" ( + echo ERROR: LIBPG_REPO variable is not set. + exit /B 1 +) + +if "%LIBPG_COMMIT%"=="" ( + echo ERROR: LIBPG_COMMIT variable is not set. + exit /B 1 +) + +if "%LIBPG_BRANCH%"=="" ( + echo ERROR: LIBPG_BRANCH variable is not set. + exit /B 1 +) + +:: The environment variables must be set +echo Using repository: %LIBPG_REPO% +echo Using commit: %LIBPG_COMMIT% +echo Using branch: %LIBPG_BRANCH% setlocal enabledelayedexpansion @@ -21,11 +44,11 @@ cd /D %tmpDir% rem Clone the selected branch of the libpg_query Git repo -git clone -b %branch% --single-branch https://github.com/pganalyze/libpg_query.git +git clone -b %LIBPG_BRANCH% --single-branch %LIBPG_REPO% cd libpg_query rem Checkout the desired commit -git checkout %commit% +git checkout %LIBPG_COMMIT% rem needed if being invoked from within gyp set MAKEFLAGS= diff --git a/script/buildAddon.sh b/script/buildAddon.sh index e6c1fbf..eff6309 100755 --- a/script/buildAddon.sh +++ b/script/buildAddon.sh @@ -1,8 +1,25 @@ #!/usr/bin/env bash +# this file is auto-generated, use "yarn generate:build " to rebuild with an env (e.g., pg-15) -# Set the desired commit hash and branch -commit=db39825bc7c1ddd45962ec6a626d740b7f8f027a -branch=15-latest +LIBPG_REPO=https://github.com/pganalyze/libpg_query.git +LIBPG_COMMIT=db39825bc7c1ddd45962ec6a626d740b7f8f027a +LIBPG_BRANCH=15-latest + +# Check if variables are set and exit if not +if [ -z "$LIBPG_COMMIT" ]; then + echo "ERROR: LIBPG_COMMIT variable is not set." + exit 1 +fi + +if [ -z "$LIBPG_BRANCH" ]; then + echo "ERROR: LIBPG_BRANCH variable is not set." + exit 1 +fi + +if [ -z "$LIBPG_REPO" ]; then + echo "ERROR: LIBPG_REPO variable is not set." + exit 1 +fi # Remember current directory and create a new, unique, temporary directory rDIR=$(pwd) @@ -15,11 +32,11 @@ makeTarget=build cd "$tmpDir" # Clone the selected branch of the libpg_query Git repo -git clone -b $branch --single-branch https://github.com/pganalyze/libpg_query.git +git clone -b $LIBPG_BRANCH --single-branch $LIBPG_REPO cd libpg_query # Checkout the desired commit -git checkout $commit +git checkout $LIBPG_COMMIT # needed if being invoked from within gyp unset MAKEFLAGS @@ -61,6 +78,7 @@ fi # Copy header cp $(pwd)/pg_query.h $rDIR/libpg_query/include/ +cp $(pwd)/protobuf/*.proto $rDIR/libpg_query/protobuf/ # Cleanup: revert to original directory and remove the temp cd "$rDIR" diff --git a/script/env.generated.json b/script/env.generated.json new file mode 100644 index 0000000..c167d8b --- /dev/null +++ b/script/env.generated.json @@ -0,0 +1,11 @@ +{ + "JOB": "pg-15", + "LIBPG_REPO": "https://github.com/pganalyze/libpg_query.git", + "LIBPG_COMMIT": "db39825bc7c1ddd45962ec6a626d740b7f8f027a", + "LIBPG_BRANCH": "15-latest", + "PGSQL_TYPES": "15.0.2", + "OPERATING_SYSTEMS": [ + "linux", + "mac" + ] +} \ No newline at end of file diff --git a/script/generate-non-win-workflows.js b/script/generate-non-win-workflows.js new file mode 100644 index 0000000..6293361 --- /dev/null +++ b/script/generate-non-win-workflows.js @@ -0,0 +1,47 @@ +const { exec } = require('child_process'); +const { join } = require('path'); +const fs = require('fs'); +const yaml = require('js-yaml'); + +// if (typeof process.argv[2] !== 'string') { +// throw new Error('branchName not provided'); +// } + +const yamldir = (s) => join(__dirname, '/../.yamlize/', s); +const workflowDir = (s) => join(__dirname, '/../.github/workflows/', s); + +const cmd = (config, workflow) => ([ + 'yamlize', + '--config', + yamldir(`config/${config}`), + + '--inFile', + yamldir(`workflows/${workflow}`), + + '--outFile', + workflowDir(`generated-${workflow}`), +].join(' ')); + + +exec(cmd('config.yaml', 'build-and-publish-no-win.yaml'), (error, _stdout, _stderr) => { + if (error) { + console.error(`Error: ${error.message}`); + return; + } + + // Read the generated YAML file + const outputPath = workflowDir(`generated-build-and-publish-no-win.yaml`); + try { + const fileContents = fs.readFileSync(outputPath, 'utf8'); + const data = yaml.load(fileContents); + + // Modify the top-level 'name' property + data.name = 'Generated Build libpg-query no windows ๐Ÿ› '; + + // Write the modified YAML back to the file + const newYamlContent = yaml.dump(data, { lineWidth: -1 }); + fs.writeFileSync(outputPath, newYamlContent, 'utf8'); + } catch (readOrWriteError) { + console.error(`Error processing YAML file: ${readOrWriteError}`); + } +}); \ No newline at end of file diff --git a/script/generate-workflows.js b/script/generate-workflows.js new file mode 100644 index 0000000..972ca95 --- /dev/null +++ b/script/generate-workflows.js @@ -0,0 +1,48 @@ +const { exec } = require('child_process'); +const { join } = require('path'); +const fs = require('fs'); +const yaml = require('js-yaml'); + +// if (typeof process.argv[2] !== 'string') { +// throw new Error('branchName not provided'); +// } + +const yamldir = (s) => join(__dirname, '/../.yamlize/', s); +const workflowDir = (s) => join(__dirname, '/../.github/workflows/', s); + +const cmd = (config, workflow) => ([ + 'yamlize', + '--config', + yamldir(`config/${config}`), + + '--inFile', + yamldir(`workflows/${workflow}`), + + '--outFile', + workflowDir(`generated-${workflow}`), +].join(' ')); + + +exec(cmd('config.yaml', 'build-and-publish.yaml'), (error, _stdout, _stderr) => { + if (error) { + console.error(`Error: ${error.message}`); + return; + } + + + // Read the generated YAML file + const outputPath = workflowDir(`generated-build-and-publish.yaml`); + try { + const fileContents = fs.readFileSync(outputPath, 'utf8'); + const data = yaml.load(fileContents); + + // Modify the top-level 'name' property + data.name = 'Generated Build libpg-query ๐Ÿ› '; + + // Write the modified YAML back to the file + const newYamlContent = yaml.dump(data, { lineWidth: -1 }); + fs.writeFileSync(outputPath, newYamlContent, 'utf8'); + } catch (readOrWriteError) { + console.error(`Error processing YAML file: ${readOrWriteError}`); + } +}); \ No newline at end of file diff --git a/script/utils/config.js b/script/utils/config.js new file mode 100644 index 0000000..4f39db3 --- /dev/null +++ b/script/utils/config.js @@ -0,0 +1,6 @@ +const path = require('path'); +const rootDir = path.join(__dirname, '/../../'); +module.exports.rootDir = rootDir; +module.exports.templatesDir = path.join(rootDir, '.templates'); +module.exports.yamlizeDir = path.join(rootDir, '.yamlize'); +module.exports.configDir = path.join(rootDir, '.yamlize/versions'); \ No newline at end of file diff --git a/script/utils/generate.js b/script/utils/generate.js new file mode 100644 index 0000000..6b8d3e4 --- /dev/null +++ b/script/utils/generate.js @@ -0,0 +1,107 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const path = require('path'); +const { + configDir, + templatesDir, + rootDir +} = require('./config'); + +const { + getConfig +} = require('./get-config'); + +const { + getTemplates +} = require('./get-templates'); + +// Read the JOB from the command line arguments +const jobFilter = process.argv[2]; +console.log({jobFilter}) + +if (!jobFilter) { + console.error('Usage: node script.js '); + console.error('ENV is PG version, e.g. pg-15'); + process.exit(1); +} + +// Load templates and configurations +const templates = getTemplates(templatesDir); +const configs = getConfig(configDir); + +// Filter configurations based on the command line input +const filteredConfigs = configs.filter(config => config.JOB === jobFilter); + +if (filteredConfigs.length === 0) { + console.error(`No configurations found for ENV: ${jobFilter}`); + process.exit(1); +} + +// Directory to output the processed templates +const outputDir = path.join(rootDir, 'script'); +if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir); +} + +// get config +const config = filteredConfigs[0]; + +// update the package.json +const packageJsonPath = path.join(__dirname, '../../package.json'); +let packageJson = fs.readFileSync(packageJsonPath, 'utf8'); +packageJson = JSON.parse(packageJson); +packageJson.dependencies['@pgsql/types'] = config.PGSQL_TYPES; +fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); + +// Generate build files from the templates +templates.forEach(template => { + // Replace placeholders in the template content + let content = template.content; + // Dynamic replacement for each key in the config + Object.keys(config).forEach(key => { + const placeholder = new RegExp(`___${key.toUpperCase()}___`, 'g'); + content = content.replace(placeholder, config[key]); + }); + + // Define the output path for the processed template + const outputFile = path.join(outputDir, `${template.name}`); + const generatedEnv = path.join(outputDir, `env.generated.json`); + + // Write the processed template to the output directory + fs.writeFileSync(generatedEnv, JSON.stringify(config, null, 2), 'utf8'); + + + // Determine the file extension + const extension = path.extname(outputFile).toLowerCase(); + + // Mapping of file extensions to single-line comment syntax + const commentMap = { + '.bat': '::', + '.sh': '#', + '.js': '//', + '.ts': '//', + }; + + // Determine the comment prefix from the map or use default + const commentPrefix = commentMap.hasOwnProperty(extension) ? commentMap[extension] : defaultComment; + let header = ''; + + let newContent = ''; + + if (commentPrefix) { + const commentLine = `${commentPrefix} this file is auto-generated, use "yarn generate:build " to rebuild with an env (e.g., pg-15)\n`; + if (content.startsWith('#!')) { + // Find the end of the first line (shebang) + const firstLineEnd = content.indexOf('\n') + 1; + newContent = content.substring(0, firstLineEnd) + commentLine + content.substring(firstLineEnd); + } else { + newContent = commentLine + content; + } + } else { + newContent = content; // No comment to add + } + + fs.writeFileSync(outputFile, newContent, 'utf8'); + console.log(`Written: ${outputFile}`); +}); diff --git a/script/utils/get-config.js b/script/utils/get-config.js new file mode 100644 index 0000000..888a0d0 --- /dev/null +++ b/script/utils/get-config.js @@ -0,0 +1,35 @@ +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); + +module.exports.getConfig = (directory) => { + let result = [] + + // Read all files in the directory recursively + const readDirectory = (dir) => { + const files = fs.readdirSync(dir, { withFileTypes: true }); + + for (const file of files) { + const fullPath = path.join(dir, file.name); + if (file.isDirectory()) { + // Recursively read subdirectory + readDirectory(fullPath); + } else if (file.name.endsWith('.yaml') || file.name.endsWith('.yml')) { + // Read and parse YAML file + + try { + const fileContents = fs.readFileSync(fullPath, 'utf8'); + const parsedYaml = yaml.load(fileContents); + const key = path.relative(directory, fullPath); // Use relative path as key + result.push(parsedYaml.env); + } catch (error) { + console.error(`Error reading file ${fullPath}:`, error); + } + } + } + }; + + readDirectory(directory); + + return result; +} \ No newline at end of file diff --git a/script/utils/get-templates.js b/script/utils/get-templates.js new file mode 100644 index 0000000..09b9db5 --- /dev/null +++ b/script/utils/get-templates.js @@ -0,0 +1,20 @@ +const fs = require('fs'); +const path = require('path'); + +module.exports.getTemplates = (templatesDir) => { + try { + const files = fs.readdirSync(templatesDir); + return files.map(file => { + const filePath = path.join(templatesDir, file); + const content = fs.readFileSync(filePath, 'utf8'); + return { + name: path.basename(filePath), + filePath, + content + } + }); + } catch (err) { + console.error('Error processing the directory:', err); + } + +}; \ No newline at end of file diff --git a/script/workflows.js b/script/workflows.js deleted file mode 100644 index fe34703..0000000 --- a/script/workflows.js +++ /dev/null @@ -1,29 +0,0 @@ -const { exec } = require('child_process'); -const { join } = require('path'); - -// if (typeof process.argv[2] !== 'string') { -// throw new Error('branchName not provided'); -// } - -const yamldir = (s) => join(__dirname, '/../.yamlize/', s); -const workflowDir = (s) => join(__dirname, '/../.github/workflows/', s); - -const cmd = (config, workflow) => ([ - 'yamlize', - '--config', - yamldir(`config/${config}`), - - '--inFile', - yamldir(`workflows/${workflow}`), - - '--outFile', - workflowDir(`generated-${workflow}`), -].join(' ')); - - -exec(cmd('config.yaml', 'build-and-test.yaml'), (error, _stdout, _stderr) => { - if (error) { - console.error(`Error: ${error.message}`); - return; - } -}); \ No newline at end of file