Pgbouncer config healthcheck

.devcontainer/Dockerfile

@@ -0,0 +1,34 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3/.devcontainer/base.Dockerfile
+
+
+# [Choice] Ubuntu version (use ubuntu-22.04 or ubuntu-18.04 on local arm64/Apple Silicon): ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+ARG VARIANT=ubuntu-24.04
+FROM mcr.microsoft.com/vscode/devcontainers/base:${VARIANT}
+
+# Postgres & our packages. Currently not customizable via VERSION param.
+# RUN apt-get update \
+#     && apt-get -y install --no-install-recommends curl ca-certificates gnupg
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends postgresql-plpython3-17 postgresql-17-postgis-3 libpq-dev
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -yq --no-install-recommends \
+    libhdf5-dev \
+    libnetcdf-dev \
+    libyaml-dev \
+    libgdal-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    cython3 && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
+ENV C_INCLUDE_PATH=/usr/include/gdal
+
+RUN pip install numpy --break-system-packages
+
+RUN pip install gdal==3.8.4 h5py netCDF4 psycopg2 PyYAML pillow --break-system-packages

.devcontainer/devcontainer-lock.json

@@ -0,0 +1,24 @@
+{
+  "features": {
+    "ghcr.io/devcontainers-extra/features/poetry:2": {
+      "version": "2.0.18",
+      "resolved": "ghcr.io/devcontainers-extra/features/poetry@sha256:7b5f2c66d25d8b3cc5d601af6b81b720c77dd315f24d5e6f84b342cccc3c9228",
+      "integrity": "sha256:7b5f2c66d25d8b3cc5d601af6b81b720c77dd315f24d5e6f84b342cccc3c9228"
+    },
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "2.17.0",
+      "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c",
+      "integrity": "sha256:25b9f05705ffba7dbe503230ac76081419306f8c8bc88e0ce78c4ecd99a0c78c"
+    },
+    "ghcr.io/devcontainers/features/git:1": {
+      "version": "1.3.5",
+      "resolved": "ghcr.io/devcontainers/features/git@sha256:27905dc196c01f77d6ba8709cb82eeaf330b3b108772e2f02d1cd0d826de1251",
+      "integrity": "sha256:27905dc196c01f77d6ba8709cb82eeaf330b3b108772e2f02d1cd0d826de1251"
+    },
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "1.8.0",
+      "resolved": "ghcr.io/devcontainers/features/python@sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511",
+      "integrity": "sha256:fbcad6955caeecc5ad3f7886baf652e25cba5225a6c4c2287c536de2e5607511"
+    }
+  }
+}

.devcontainer/devcontainer.json

@@ -0,0 +1,67 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3
+{
+  "name": "Python3 & Poetry & Postgres",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "args": {
+    }
+  },
+  // Configure tool-specific properties.
+  "customizations": {
+    // Configure properties specific to VS Code.
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "remote.autoForwardPorts": false,
+        "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
+        "python.linting.enabled": true,
+        "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+        "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+        "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+        "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+        "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+        "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+        "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+      },
+      // VSCODE ONLY: Add the IDs of extensions you want installed when the container is created.
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  //"forwardPorts": [48423],
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "bash ./.devcontainer/post-install.sh",
+  "postStartCommand": "bash ./.devcontainer/post-start.sh",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": "latest",
+    "ghcr.io/devcontainers/features/git:1": "latest",
+    // add python to container
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.14"
+    },
+    // add poetry to container
+    "ghcr.io/devcontainers-extra/features/poetry:2": {
+      "version": "2.4"
+    }
+  },
+  // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "vscode",
+  "remoteEnv": {
+    //		"PATH": "${containerEnv:PATH}:${containerEnv:HOME}/.local/bin"
+  },
+  "runArgs": [
+    // allow container to be treated with no network isolation
+    "--network=host",
+    // give a nicer name to the container
+    "--name",
+    "pcex_backend_devcontainer_${localEnv:USER}"
+  ],
+  "mounts": [
+    "source=/storage/data/projects/comp_support/daccs/test-data/,target=/storage/data/projects/comp_support/daccs/test-data/,type=bind,consistency=cached"
+  ]
+}

.devcontainer/post-install.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -ex
+
+##
+## Create some aliases
+##
+echo 'alias ll="ls -alF"' >> $HOME/.bashrc
+echo 'alias la="ls -A"' >> $HOME/.bashrc
+echo 'alias l="ls -CF"' >> $HOME/.bashrc
+
+# Convenience workspace directory for later use
+WORKSPACE_DIR=$(pwd)
+
+# Change some Poetry settings to better deal with working in a container
+poetry config cache-dir ${WORKSPACE_DIR}/.cache
+poetry config virtualenvs.in-project true
+
+# Now install all dependencies
+poetry install --all-extras
+
+echo "Done!"

.devcontainer/post-start.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -ex
+
+##
+## Create some aliases
+##
+echo 'alias ll="ls -alF"' >> $HOME/.bashrc
+echo 'alias la="ls -A"' >> $HOME/.bashrc
+echo 'alias l="ls -CF"' >> $HOME/.bashrc
+
+# Convenience workspace directory for later use
+WORKSPACE_DIR=$(pwd)
+
+# Change some Poetry settings to better deal with working in a container
+poetry config cache-dir ${WORKSPACE_DIR}/.cache
+poetry config virtualenvs.in-project true
+
+# Now install all dependencies
+poetry install --all-extras
+
+echo "Done!"

.github/workflows/black.yml

@@ -14,7 +14,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
-          python-version: "3.12" # Python ≥3.11 required for use_pyproject
+          python-version: "3.14" # Python ≥3.11 required for use_pyproject
       - uses: psf/black@35ea67920b7f6ac8e09be1c47278752b1e827f76 # stable
         with:
           use_pyproject: true

.github/workflows/pypi-publish.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
       with:
-        python-version: '3.12'
+        python-version: '3.14'
     - name: Install Poetry
       run: |
         pipx install "poetry==2.4.0"

.github/workflows/python-ci.yml

@@ -8,6 +8,17 @@ jobs:
   test:
     runs-on: ubuntu-24.04
     container: pcic/geospatial-python:3.8.4
+    timeout-minutes: 15
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.python-version }}
+      cancel-in-progress: true
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+          - "3.12"
+          - "3.13"
+          - "3.14"
     env:
       HOME: /root
 

ce/__init__.py

@@ -1,7 +1,9 @@
 import os
+from typing import Any
 
 from flask import Flask
 from flask_cors import CORS
+from sqlalchemy.pool import NullPool
 
 from ce.api.multimeta_cache import (
     DEFAULT_PRELOAD_MULTIMETA_PARAMS,
@@ -11,6 +13,31 @@
 from ce.views import add_routes
 
 
+def _build_engine_options():
+    """Build SQLAlchemy engine options from environment variables.
+
+    Set SQLALCHEMY_POOL_CLASS=null to use NullPool (recommended when deploying
+    behind pgbouncer). Optional pool tuning via SQLALCHEMY_POOL_SIZE,
+    SQLALCHEMY_MAX_OVERFLOW, SQLALCHEMY_POOL_TIMEOUT, SQLALCHEMY_POOL_RECYCLE.
+    """
+    if os.getenv("SQLALCHEMY_POOL_CLASS", "").lower() == "null":
+        return {"poolclass": NullPool}
+
+    options: dict[str, Any] = {"pool_pre_ping": True}
+
+    for env_var, key, cast in [
+        ("SQLALCHEMY_POOL_SIZE", "pool_size", int),
+        ("SQLALCHEMY_MAX_OVERFLOW", "max_overflow", int),
+        ("SQLALCHEMY_POOL_TIMEOUT", "pool_timeout", float),
+        ("SQLALCHEMY_POOL_RECYCLE", "pool_recycle", float),
+    ]:
+        val = os.getenv(env_var)
+        if val is not None:
+            options[key] = cast(val)
+
+    return options
+
+
 def get_app(config=None):
     app = Flask(__name__)
     CORS(app)
@@ -19,7 +46,7 @@ def get_app(config=None):
         "SQLALCHEMY_DATABASE_URI": os.getenv(
             "MDDB_DSN", "postgresql://httpd_meta@db.pcic.uvic.ca/pcic_meta"
         ),
-        "SQLALCHEMY_ENGINE_OPTIONS": {"pool_pre_ping": True},
+        "SQLALCHEMY_ENGINE_OPTIONS": _build_engine_options(),
         "SQLALCHEMY_TRACK_MODIFICATIONS": False,
         "MULTIMETA_CACHE_DIR": os.getenv("MULTIMETA_CACHE_DIR", "/tmp/multimeta-cache"),
         "MULTIMETA_CACHE_ENABLED": True,

ce/api/geo.py

@@ -249,8 +249,9 @@ def polygon_to_masked_array_helper(raster):
 
         return array * scale_factor + add_offset
 
-    if "dodsC" in resource:
-        with rasterio_thredds_helper(resource, variable) as raster:
+    if "http" in resource:
+        fs_resource = resource.replace("/dodsC/", "/fileServer/", 1)
+        with rasterio_thredds_helper(fs_resource, variable) as raster:
             return polygon_to_masked_array_helper(raster)
 
     # Local file path case
@@ -263,10 +264,10 @@ def polygon_to_masked_array_helper(raster):
 @contextmanager
 def rasterio_thredds_helper(resource, variable):
     """
-    Opens a remote NetCDF resource for Rasterio, handling OpenDAP (dodsC) or fileServer as needed.
-    Assumes Rasterio has GDAL with OpenDAP support.
+    Opens a remote NetCDF resource for Rasterio via GDAL's vsicurl.
+    Expects a fileServer URL; GDAL's NetCDF driver requires direct HTTP
+    file access and cannot use OPeNDAP.
     """
-    # Construct proper NetCDF subdataset path
-    dataset_url = f'NETCDF:"{resource}":{variable}'
+    dataset_url = f'NETCDF:"/vsicurl/{resource}":{variable}'
     with rasterio.open(dataset_url, mode="r", driver="NetCDF") as raster:
         yield raster

ce/api/util.py

@@ -68,12 +68,9 @@ def open_nc(resource):
             f"The metadata database is out of sync with the filesystem. I was told to open file with name {resource}, but it does not exist."
         )
 
-    try:
-        nc = Dataset(resource, "r")
+    with Dataset(resource, "r") as nc:
         nc.set_always_mask(False)
         yield nc
-    finally:
-        nc.close()
 
 
 def get_array(nc, resource, time, area, variable):