Feature/auscope

.github/workflows/bump-version.yml

@@ -1,4 +1,6 @@
 name: Bump Version
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 on:
   pull_request:
     types:
@@ -8,7 +10,7 @@ on:
       - development
 jobs:
   tag:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:

.github/workflows/dev.yml

@@ -12,6 +12,7 @@ env:
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
 jobs:
   prerelease:

.github/workflows/features.yml

@@ -15,6 +15,7 @@ env:
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
 jobs:
   slug:
@@ -28,7 +29,7 @@ jobs:
           submodules: true
       - name: GitHub Slug Action
         uses: rlespinasse/github-slug-action@v4.4.0
-  feature:
+  step:
     uses: ./.github/workflows/main.yml
     needs: slug
     with:

.github/workflows/main.yml

@@ -14,6 +14,7 @@ env:
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
 jobs:
   version:
@@ -26,8 +27,8 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: GitHub Slug Action
-        uses: rlespinasse/github-slug-action@v4.4.0
+      # - name: GitHub Slug Action
+      #   uses: rlespinasse/github-slug-action@v4.4.0
       - name: version
         uses: paulhatch/semantic-version@v5.0.3
         id: version
@@ -51,23 +52,10 @@ jobs:
           # The output method used to generate list of users, 'csv' or 'json'.
           user_format_type: "csv"
 
-  test:
+  unit-test:
+  # NOTE: ubuntu-latest equates to Description:	Ubuntu 24.04.3 LTS on 12 MAR 2026 (while I'm running Ubuntu 25.10)
     runs-on: ubuntu-latest
     needs: [version]
-    # services:
-    #   mongodb:
-    #     image: mongo:4.0.28
-    #     #env:
-    #       #MONGO_INITDB_ROOT_USERNAME: admin
-    #       #MONGO_INITDB_ROOT_PASSWORD: admin
-    #       #MONGO_INITDB_DATABASE: APP-DATABASE
-    #     ports:
-    #       - 27017:27017
-    #     options: >-
-    #       --health-cmd mongo
-    #       --health-interval 10s
-    #       --health-timeout 5s
-    #       --health-retries 5
     steps:
       - uses: actions/checkout@v3
         with:
@@ -76,19 +64,55 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.22.0"
+          go-version: "1.24.5"
+      - name: Setup VIPS lib
+        run: |
+          echo "Installing libvips quietly..."
+          sudo apt-get update
+          sudo apt-get -y -qq install libvips-dev
+          sudo apt list --installed | grep libvips
+          echo "Running genvips..."
+          ./genvips.sh
       - name: Start MongoDB
         uses: supercharge/mongodb-github-action@1.10.0
         with:
-          mongodb-version: 4.0.28
-          mongodb-replica-set: test
+          mongodb-version: 8
+          mongodb-replica-set: testu
           mongodb-port: 27017
       - name: Unit Test
         run: make unittest
         env:
           AWS_DEFAULT_REGION: us-east-1
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+  integration-test:
+  # NOTE: ubuntu-latest equates to Description:	Ubuntu 24.04.3 LTS on 12 MAR 2026 (while I'm running Ubuntu 25.10)
+    runs-on: ubuntu-latest
+    needs: [version]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.24.5"
+      - name: Setup VIPS lib
+        run: |
+          echo "Installing libvips quietly..."
+          sudo apt-get update
+          sudo apt-get -y -qq install libvips-dev
+          sudo apt list --installed | grep libvips
+          echo "Running genvips..."
+          ./genvips.sh
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.10.0
+        with:
+          mongodb-version: 8
+          mongodb-replica-set: testi
+          mongodb-port: 27017
       - name: Integration Test
         # NOTE: integration-test-config.json must have EnvironmentName at the end, so we can append a suffix
         #       to it, to make DB names unique for builds to not clash!
@@ -101,7 +125,7 @@ jobs:
           sleep 5
           cd ./internal/cmd-line-tools/api-integration-test
           echo "unittest_${UNIX_MS}"
-          ./tester -apiHost localhost:8080 -envName "unittest_${UNIX_MS}" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          ./tester -apiHost localhost:8080 -envName "unittest_${UNIX_MS}" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -jobsBucket "integration-test-jobs-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           AWS_DEFAULT_REGION: us-east-1
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -110,7 +134,8 @@ jobs:
           PIXLISE_CONFIG_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_MGMT_AUTH0_CLIENT_ID }}
           PIXLISE_CONFIG_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_MGMT_AUTH0_SECRET }}
           PIXLISE_CONFIG_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
-  build:
+
+  build-lambdas:
     runs-on: ubuntu-latest
     needs: [version]
     env:
@@ -131,14 +156,16 @@ jobs:
           echo "BUILDING VERSION ${{ needs.version.outputs.version }}"
           echo "Tag name: ${{ needs.version.outputs.version_tag }}"
       - name: Build Package
-        run: BUILD_VERSION="${{ needs.version.outputs.version }}" make build-linux
+        run: |
+          mkdir -p _out
+          BUILD_VERSION="${{ needs.version.outputs.version }}" make build-linux-aux
       - name: Copy to S3 Archive
         run: |
           cd _out
-          rm pixlise-api-linux
+          #rm pixlise-api-linux
           # WAS USING: wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem
           wget https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem
-          mkdir dataimport integrationtest
+          mkdir -p dataimport integrationtest
           zip -q dataimport/dataimport-linux-${{ needs.version.outputs.version }}.zip global-bundle.pem bootstrap
           rm bootstrap
           # mkdir test-data
@@ -148,11 +175,11 @@ jobs:
           # zip -q importtest/importtest-linux-${{ needs.version.outputs.version }}.zip importtest-linux test-data/000000001-21-10-2022-15-37-00.zip
           # rm importtest-linux
           mv job-runner job-runner-${{ needs.version.outputs.version }}
-          mkdir job-runner
+          mkdir -p job-runner
           cp job-runner-${{ needs.version.outputs.version }} ./job-runner
           aws s3 cp . s3://corestack-buildartifactsf774a77d-105on4pno9pjm/ --recursive --region us-east-1
 
-  docker:
+  build-api-docker:
     needs: [version]
     runs-on: ubuntu-latest
     permissions:
@@ -197,6 +224,7 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: type=raw,value=${{ needs.version.outputs.version }}
+
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
@@ -210,6 +238,7 @@ jobs:
           build-args: |
             VERSION=${{ needs.version.outputs.version }}
             GITHUB_SHA=${{ needs.version.outputs.github_sha_short }}
+
       # Sign the resulting Docker image digest except on PRs.
       # This will only write to the public Rekor transparency log when the Docker
       # repository is public to avoid leaking data.  If you would like to publish

.github/workflows/release.yml

@@ -8,6 +8,7 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
 jobs:
   release:

.github/workflows/staging.yml

@@ -14,6 +14,7 @@ env:
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
 jobs:
   staging:

.gitignore

@@ -4,20 +4,13 @@ __debug_bin.exe
 
 # Build output from make for exes, coverage reports, etc
 /_out/
+/vips/
 #generated-protos/
 api/ws/wsMessage.go
-/OLDCODE
-auth0dummy-LOCAL-ONLY.go-
-/local-mongo/dbseed/from-prod-Jan2024/
-/local-mongo/dbseed/migrated/
-/local-mongo/dbseed/migration-source/
-/local-mongo/dbseed/old/
-/local-mongo/dbseed/*.json
-/local-mongo/dbseed/*.sh
-/local-mongo/dbseed/json/
-/local-mongo/3repl/
 
 db-restore/
+mongo-db-local-*/
+mongo-db-pixlise-*/
 
 local-api-config*.json
 local-api-config-unittest.json

Dockerfile

@@ -14,30 +14,39 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM golang:1.22-alpine AS builder
+
+FROM golang:1.24-alpine AS builder
+
 ARG VERSION
 ARG GITHUB_SHA
 
-RUN apk --no-cache add ca-certificates libc6-compat wget make bash
+RUN apk add --no-cache ca-certificates libc6-compat wget make bash build-base pkgconfig vips-dev
 
 COPY . /build
+ENV CGO_ENABLED=1
+ENV BUILD_VERSION=${VERSION}
+ENV GITHUB_SHA=${GITHUB_SHA}
+
+RUN cd /build && ./genvips.sh
+RUN cd /build && make build-linux-api
 
-RUN cd /build && export BUILD_VERSION=${VERSION} && export GITHUB_SHA=${GITHUB_SHA} && make build-linux
 
 FROM alpine:latest
+
 RUN apk --no-cache add ca-certificates libc6-compat wget
 WORKDIR /root
-# Copy the Pre-built binary file from the previous stage
 
+# Copy the Pre-built binary file from the previous stage
 COPY --from=builder /build/_out/pixlise-api-linux ./
-#COPY ./_out/pixlise-api-linux ./
-#RUN ls -al
+
 COPY beam-tool/BGT ./
 COPY beam-tool/Geometry_PIXL_EM_Landing_25Jan2021.csv ./
-#RUN ls -al
+
+RUN apk add --no-cache vips-dev
 
 RUN chmod +x ./pixlise-api-linux && wget https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -O global-bundle.pem
 RUN chmod +x ./BGT
+
 # Expose port 8080 to the outside world
 EXPOSE 8080
 

Makefile

@@ -14,34 +14,42 @@ lint: ## Lint the files
 	golint ${PKG_LIST}
 
 unittest: ## Run unittests
-	pwd
-	cd ..
 	mkdir -p _out
 	go install github.com/favadi/protoc-go-inject-tag@latest
 	go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
 	protoc-go-inject-tag -remove_tag_comment -input="./generated-protos/*.pb.go"
-	go test -v ./...
+	go test ./...
 
 integrationtest:
 	mkdir -p _out
 	echo "version: ${BUILD_VERSION}"
 	echo "sha: ${GITHUB_SHA}"
-	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./api-service ./internal/api
-	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./internal/cmd-line-tools/api-integration-test/tester ./internal/cmd-line-tools/api-integration-test
+	go install github.com/favadi/protoc-go-inject-tag@latest
+	go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
+	protoc-go-inject-tag -remove_tag_comment -input="./generated-protos/*.pb.go"
+	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -o ./api-service ./internal/api
+	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -o ./internal/cmd-line-tools/api-integration-test/tester ./internal/cmd-line-tools/api-integration-test
 
 codegen:
 	./genproto.sh checkgen
 
-build: build-linux build-mac
+build: build-linux-api build-linux-aux build-mac
+
+build-linux-api:
+	mkdir -p _out
+	echo "version: ${BUILD_VERSION}"
+	echo "sha: ${GITHUB_SHA}"
+
+	GOOS=linux GOARCH=amd64 go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
+	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -o ./_out/pixlise-api-linux ./internal/api
 
-build-linux:
+build-linux-aux:
 	mkdir -p _out
 	echo "version: ${BUILD_VERSION}"
 	echo "sha: ${GITHUB_SHA}"
 	GOOS=linux GOARCH=amd64 go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
-	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/pixlise-api-linux ./internal/api
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/bootstrap ./internal/lambdas/data-import
-	GOOS=linux GOARCH=amd64 go build -v -o ./_out/job-runner ./internal/cmd-line-tools/job-runner
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -o ./_out/bootstrap ./internal/lambdas/data-import
+	GOOS=linux GOARCH=amd64 go build -o ./_out/job-runner ./internal/cmd-line-tools/job-runner
 #	GOOS=linux GOARCH=amd64 go build -v -o ./_out/importtest-linux ./internal/cmdline-tools/import-integration-test
 #	GOOS=linux GOARCH=amd64 go build -v -o ./_out/integrationtest-linux ./internal/cmdline-tools/api-integration-test
 

api/config/config.go

@@ -70,14 +70,9 @@ type APIConfig struct {
 	PiquantDockerImage string // PIQUANT docker image to use to run a job
 	PiquantJobsBucket  string // PIQUANT job scratch drive
 
-	PosterImage             string
-	QuantDestinationPackage string
-
 	QuantExecutor  string
 	QuantNamespace string // Used for running large multi-node quants
 
-	QuantObjectType string
-
 	SentryEndpoint string
 
 	UsersBucket string
@@ -121,6 +116,8 @@ type APIConfig struct {
 	RestoreEnabled   bool
 
 	ImpersonateEnabled bool
+
+	MongoDebug bool
 }
 
 func homeDir() string {