diff --git a/.github/workflows/deploy-test-studio-crc.yml b/.github/workflows/deploy-test-studio-crc.yml index 2af19893..0b9b0614 100644 --- a/.github/workflows/deploy-test-studio-crc.yml +++ b/.github/workflows/deploy-test-studio-crc.yml @@ -23,7 +23,7 @@ jobs: uses: lewagon/wait-on-check-action@v1.3.4 with: ref: ${{ github.event.pull_request.head.sha }} - check-name: 'deploy-and-test' + check-name: 'Deploy and Test Studio on Kind' repo-token: ${{ secrets.GITHUB_TOKEN }} wait-interval: 30 allowed-conclusions: success,failure,cancelled,skipped diff --git a/.github/workflows/deploy-test-studio-kind.yml b/.github/workflows/deploy-test-studio-kind.yml index 0d282817..e6ffaf3d 100644 --- a/.github/workflows/deploy-test-studio-kind.yml +++ b/.github/workflows/deploy-test-studio-kind.yml @@ -201,6 +201,24 @@ jobs: kill $MONITOR_PID 2>/dev/null || true echo "=== Deployment script completed ===" + - name: Inspect gateway/ui resources + run: | + echo "=== All resources in default ===" + kubectl get all -n default || true + + echo "" + echo "=== Deployments with labels ===" + kubectl get deploy -n default --show-labels || true + + echo "" + echo "=== Pods with labels ===" + kubectl get pods -n default --show-labels || true + + echo "" + echo "=== Services with selectors ===" + kubectl get svc -n default -o wide || true + kubectl describe svc geofm-gateway -n default || true + kubectl describe svc geofm-ui -n default || true - name: Verify Deployment run: | @@ -332,6 +350,29 @@ jobs: exit 1 fi + - name: Run Integration Tests + env: + GATEWAY_TLS_VERIFY: "0" + BASE_GATEWAY_URL: "https://localhost:4181" + run: | + echo "=== Running Integration Tests ===" + + # Extract API key + if [[ -f ".studio-api-key" ]]; then + source .studio-api-key + export API_KEY=$STUDIO_API_KEY + echo "✅ Loaded API key" + else + echo "❌ Error: .studio-api-key file not found" + exit 1 + fi + + # Install test dependencies if not already installed + pip install -r requirements-dev.txt + + # Run integration tests + python -m pytest -q -m integration --no-cov --log-file=run.log --log-file-level=INFO tests/integration/test_inference_models.py + - name: Run Workshop Labs if: success() env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 737bd40d..983576cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: # You are encouraged to use static refs such as tags, instead of branch name # # Running "pre-commit autoupdate" automatically updates rev to latest tag - rev: 0.13.1+ibm.62.dss + rev: 0.13.1+ibm.64.dss hooks: - id: detect-secrets # pragma: whitelist secret # Add options for detect-secrets-hook binary. You can run `detect-secrets-hook --help` to list out all possible options. diff --git a/docs/deployment/detailed_deployment_local.md b/docs/deployment/detailed_deployment_local.md index fe3f2f79..b38a8516 100644 --- a/docs/deployment/detailed_deployment_local.md +++ b/docs/deployment/detailed_deployment_local.md @@ -7,6 +7,7 @@ Below we provide two different deployment options, which are similar during depl * Lima VM * Minikube * OpenShift Local (formerly CodeReady Containers) +<<<<<<< HEAD ## Clone Repository @@ -25,10 +26,39 @@ source .venv/bin/activate # On Windows: .venv\Scripts\activate # Install dependencies pip install -r requirements.txt ``` +======= +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +<<<<<<< HEAD ## VM cluster initialisation Here you need to follow the Lima VM *or* the Minikube *or* the Openshift local(CRC) instructions. +======= +## Clone Repository + +```bash +git clone https://github.com/IBM/geospatial-studio.git +cd geospatial-studio +``` + +## Install Python Dependencies + +```bash +# Create virtual environment +python3 -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt +``` +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) + +## VM cluster initialisation +Here you need to follow the Lima VM *or* the Minikube *or* the Openshift local(CRC) instructions. + +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) ### [Option 1] Lima VM setup **Prerequisites** @@ -45,7 +75,26 @@ Here you need to follow the Lima VM *or* the Minikube *or* the Openshift local( 1. Install [Lima VM](https://github.com/lima-vm/lima). Needs to be *v1.2.1* (not yet compatible with v2) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +2. Start the Lima VM cluster: +======= +2. Install Python dependencies: +```shell +pip install -r requirements.txt +``` + +3. Start the Lima VM cluster: +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= 2. Start the Lima VM cluster: +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ```shell limactl start --name=studio deployment-scripts/lima/studio.yaml ``` @@ -121,6 +170,11 @@ kubectl config current-context minikube dashboard ``` +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ### [Option 3] OpenShift Local setup (formerly CodeReady Containers) **Prerequisites** @@ -129,6 +183,47 @@ minikube dashboard - **Memory**: 32GB RAM minimum (48GB recommended) - **Disk**: 100GB free space minimum - **OS**: Linux +<<<<<<< HEAD + +#### Required Software +- [Red Hat OpenShift Local (CRC)](https://developers.redhat.com/products/openshift-local/overview) +- [oc CLI](https://docs.openshift.com/container-platform/latest/cli_reference/openshift_cli/getting-started-cli.html) +- [kubectl](https://kubernetes.io/docs/tasks/tools/) +- [Helm 3](https://helm.sh/docs/intro/install/) +- [Python 3.9+](https://www.python.org/downloads/) +- [jq](https://github.com/jqlang/jq) - json command-line processor +- [yq](https://github.com/mikefarah/yq) - yaml command-line processor + +### CRC Cluster setup + +```bash +# Download from https://developers.redhat.com/products/openshift-local/overview +# Or use package manager (macOS example): +brew install --cask openshift-local + +# Verify installation +crc version + +# Set up your host machine for CRC (one-time operation): +crc setup + +# Start with recommended resources for geospatial workloads +crc start --cpus 8 --memory 32768 --disk-size 100 +======= +### OpenShift Local setup (formerly CodeReady Containers) +<<<<<<< HEAD +======= +### [Option 3] OpenShift Local setup (formerly CodeReady Containers) +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) + +**Prerequisites** +#### System Requirements +- **CPU**: 8+ cores (12+ recommended) +- **Memory**: 32GB RAM minimum (48GB recommended) +- **Disk**: 100GB free space minimum +- **OS**: macOS, Linux, or Windows +======= +>>>>>>> 0d4180c (docs: update OS requirements in crc deployment instructions) #### Required Software - [Red Hat OpenShift Local (CRC)](https://developers.redhat.com/products/openshift-local/overview) @@ -153,11 +248,47 @@ crc version crc setup # Start with recommended resources for geospatial workloads +<<<<<<< HEAD +crc start --cpus 8 --memory 16384 --disk-size 100 +>>>>>>> 31c028f (feat: Test Openshift local deployment) +======= crc start --cpus 8 --memory 32768 --disk-size 100 +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= + +**Prerequisites** +* A [RedHat OpenShift account](https://console.redhat.com/openshift/create/local) +* [OpenShift Local(crc)](https://console.redhat.com/openshift/create/local) installed and running +* [Helm](https://helm.sh/docs/v3/) - v3.19 (*currently incompatible with v4*) +* [OpenShift CLI](https://docs.okd.io/4.18/cli_reference/openshift_cli/getting-started-cli.html) +* Kubectl (bundled with above) +* [jq](https://github.com/jqlang/jq) - json command-line processor +* [yq](https://github.com/mikefarah/yq) - yaml command-line processor +* Minimum 8GB RAM and 4 CPUs available for the VM (more recommended) + +**VM cluster initialization** +1. Follow the [Getting started](https://console.redhat.com/openshift/create/local) guide to install your local OpenShift instance. + +2. [Start the local OpenShift cluster instance](https://crc.dev/docs/using/). Ensure your container machine configuration has resource allocation for memory > 8g and cpu > 4 and disk-size 100 +```bash +# Set up your host machine for CRC: +crc setup + +# Start with recommended resources for geospatial workloads +crc start --cpus 8 --memory 16384 --disk-size 100 +>>>>>>> 31c028f (feat: Test Openshift local deployment) +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # Verify cluster is running crc status +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # Login to CRC # Use the credentials from crc start output eval $(crc oc-env) @@ -172,14 +303,60 @@ k9s # Useful commands: +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 31c028f (feat: Test Openshift local deployment) +======= +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +======= +>>>>>>> 31c028f (feat: Test Openshift local deployment) +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # stop the instance crc stop # Remove previous cluster (if present) crc delete +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # view the password for the developer and kubeadmin users crc console --credentials +======= +``` + +3. To monitor deployment on the cluster, you can access the cluster running in the CRC instance by using the OpenShift Container Platform web console or OpenShift CLI (oc). +```bash +# Access the OpenShift Container Platform web console with your default web browser. Log in as the `developer` user with the password printed in the output of the crc start command. +crc console + +# view the password for the developer and kubeadmin users +crc console --credentials + +# Alternatively, access the OpenShift Container Platform cluster by using the OpenShift CLI (oc) +# add the cached oc executable to your $PATH +eval $(crc oc-env) + +# Log in as the admin user +oc login -u kubeadmin -p https://api.crc.testing:6443 +``` + +4. Alternatively, you can use a tool such as [k9s](https://k9scli.io). +```sh +k9s +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= + +# view the password for the developer and kubeadmin users +crc console --credentials +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ``` @@ -267,9 +444,34 @@ export DEPLOYMENT_ENV=minikube export DEPLOYMENT_ENV=crc ``` +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +Use the `default` namespace +>>>>>>> 31c028f (feat: Test Openshift local deployment) +======= +Use the `default` namespace for *Lima VM* and *Minikube:* +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> c15dba1 (Update crc instructions) +======= +======= +Use the `default` namespace +>>>>>>> 31c028f (feat: Test Openshift local deployment) +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ```bash export OC_PROJECT=default ``` +```bash +export IMAGE_REGISTRY=geospatial-studio +``` + + +```bash +export IMAGE_REGISTRY=geospatial-studio +``` ```bash @@ -296,7 +498,21 @@ OC_PROJECT=default # cluster_url # For OpenShift local: +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) export CLUSTER_URL='apps-crc.testing' +======= +export CLUSTER_URL='https://api.crc.testing:6443' +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= +export CLUSTER_URL='apps-crc.testing' +>>>>>>> c15dba1 (Update crc instructions) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # Otherwise use: export CLUSTER_URL=localhost @@ -328,6 +544,142 @@ source workspace/$DEPLOYMENT_ENV/env/env.sh ### Set up S3 compatible storage #### For Openshift local(CRC): +<<<<<<< HEAD +```bash +# Label the CRC node with required topology labels: +oc label nodes crc topology.kubernetes.io/region=us-east --overwrite +oc label nodes crc topology.kubernetes.io/zone=us-east --overwrite +oc label nodes crc ibm-cloud.kubernetes.io/region=us-east --overwrite + +# Add IBM Helm repository: +helm repo add ibm-helm https://raw.githubusercontent.com/IBM/charts/master/repo/ibm-helm +helm repo update + +# Fetch the IBM Object Storage Plugin: +helm fetch --untar ibm-helm/ibm-object-storage-plugin +# Make the plugin script executable +chmod +x ./ibm-object-storage-plugin/helm-ibmc/ibmc.sh +# Install Helm plugin +helm plugin install ./ibm-object-storage-plugin/helm-ibmc +# Install IBM Object Storage Plugin +helm ibmc install ibm-object-storage-plugin ibm-helm/ibm-object-storage-plugin \ + --set license=true \ + --set workerOS="redhat" \ + --set region="us-east" + +# Wait for the plugin deployment to be ready: +kubectl wait --for=condition=available deployment/ibmcloud-object-storage-plugin \ + -n ibm-object-s3fs --timeout=300s + +# Create a ConfigMap for OpenShift TLS certificates: +kubectl apply -f - < workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml + +# Apply MinIO deployment +kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml -n ${OC_PROJECT} + +# Wait for MinIO to be ready +kubectl wait --for=condition=ready pod -l app=minio -n ${OC_PROJECT} --timeout=300s +``` + +* Update MinIO Connection details: + ```bash + export MINIO_API_URL="https://minio-api-$OC_PROJECT.$CLUSTER_URL" + # Update `workspace/${DEPLOYMENT_ENV}/env/.env` with MinIO details for external connection + sed -i -e "s/access_key_id=.*/access_key_id=minioadmin/g" workspace/${DEPLOYMENT_ENV}/env/.env + sed -i -e "s/secret_access_key=.*/secret_access_key=minioadmin/g" workspace/${DEPLOYMENT_ENV}/env/.env + sed -i -e "s|endpoint=.*|endpoint=$MINIO_API_URL|g" workspace/${DEPLOYMENT_ENV}/env/.env + sed -i -e "s/region=.*/region=us-east-1/g" workspace/${DEPLOYMENT_ENV}/env/.env + ``` + +* Configure Host Modifier DaemonSet: This step ensures MinIO is accessible from within pods + ```bash + # Get MinIO cluster IP and internal URL + export MINIO_CLUSTER_IP=$(oc get svc minio -n "${OC_PROJECT}" -o jsonpath='{.spec.clusterIP}') + export MINIO_INTERNAL_URL="minio.${OC_PROJECT}.svc.cluster.local" + export LOCAL_CA_CRT=$(oc get configmap trusted-ca-bundle -n ibm-object-s3fs -o jsonpath='{.data.service-ca\.crt}') + + # Generate hosts modifier DaemonSet + cat deployment-scripts/crc-hosts-modifier-daemonset.yaml | \ + sed -e "s/\$MINIO_CLUSTER_IP/$MINIO_CLUSTER_IP/g" | \ + sed -e "s/\$MINIO_INTERNAL_URL/$MINIO_INTERNAL_URL/g" \ + > workspace/$DEPLOYMENT_ENV/initialisation/crc-hosts-modifier-daemonset-tmp.yaml + + # Use common function to inject CA certificate + source ./common_functions.sh + auto_indent_and_replace \ + workspace/$DEPLOYMENT_ENV/initialisation/crc-hosts-modifier-daemonset-tmp.yaml \ + SELF_CA_CRT \ + "$LOCAL_CA_CRT" \ + workspace/$DEPLOYMENT_ENV/initialisation/crc-hosts-modifier-daemonset.yaml + + # Apply DaemonSet + oc apply -f workspace/$DEPLOYMENT_ENV/initialisation/crc-hosts-modifier-daemonset.yaml -n default + + # Clean up temporary file + rm workspace/$DEPLOYMENT_ENV/initialisation/crc-hosts-modifier-daemonset-tmp.yaml + ``` + + +#### Otherwise, for Lima VM and Minikube: + +```bash +### Install cloud object storage drivers in the cluster +# Ensure node has labels required by drivers +kubectl label nodes lima-studio topology.kubernetes.io/region=us-east-1 topology.kubernetes.io/zone=us-east-1a + +# Install the drivers +cp -R deployment-scripts/ibm-object-csi-driver workspace/$DEPLOYMENT_ENV/initialisation +sed -e "s/default/$OC_PROJECT/g" deployment-scripts/template/cos-s3-csi-s3fs-sc.yaml > workspace/$DEPLOYMENT_ENV/initialisation/ibm-object-csi-driver/cos-s3-csi-s3fs-sc.yaml +sed -e "s/default/$OC_PROJECT/g" deployment-scripts/template/cos-s3-csi-sc.yaml > workspace/$DEPLOYMENT_ENV/initialisation/ibm-object-csi-driver/cos-s3-csi-sc.yaml +kubectl apply -k workspace/$DEPLOYMENT_ENV/initialisation/ibm-object-csi-driver/ + +======= ```bash # Label the CRC node with required topology labels: oc label nodes crc topology.kubernetes.io/region=us-east --overwrite @@ -452,6 +804,9 @@ kubectl wait --for=condition=ready pod -l app=minio -n ${OC_PROJECT} --timeout=3 #### Otherwise, for Lima VM and Minikube: ```bash +<<<<<<< HEAD +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= ### Install cloud object storage drivers in the cluster # Ensure node has labels required by drivers kubectl label nodes lima-studio topology.kubernetes.io/region=us-east-1 topology.kubernetes.io/zone=us-east-1a @@ -462,6 +817,7 @@ sed -e "s/default/$OC_PROJECT/g" deployment-scripts/template/cos-s3-csi-s3fs-sc. sed -e "s/default/$OC_PROJECT/g" deployment-scripts/template/cos-s3-csi-sc.yaml > workspace/$DEPLOYMENT_ENV/initialisation/ibm-object-csi-driver/cos-s3-csi-sc.yaml kubectl apply -k workspace/$DEPLOYMENT_ENV/initialisation/ibm-object-csi-driver/ +>>>>>>> c15dba1 (Update crc instructions) # Create TLS for MinIO openssl genrsa -out minio-private.key 2048 mkdir -p workspace/$DEPLOYMENT_ENV/initialisation @@ -477,29 +833,121 @@ kubectl create configmap minio-public-config --from-file=minio-public.crt -n kub kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/minio-public-config.yaml -n kube-system # Install MinIO +# For Openshift local: +python ./deployment-scripts/update-deployment-template.py --storageclass crc-csi-hostpath-provisioner --disable-route --filename deployment-scripts/minio-deployment.yaml > workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml + +# Otherwise use: python ./deployment-scripts/update-deployment-template.py --disable-route --filename deployment-scripts/minio-deployment.yaml > workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD # Apply MinIO deployment +======= + +>>>>>>> 31c028f (feat: Test Openshift local deployment) kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml -n ${OC_PROJECT} +<<<<<<< HEAD # Wait for MinIO to be ready kubectl wait --for=condition=ready pod -l app=minio -n ${OC_PROJECT} --timeout=300s +<<<<<<< HEAD # Access MinIO Console: # Port forward to access MinIO console at https://localhost:9001 +======= +#### Access MinIO Console +To access the MinIO console: +```bash +# Port forward to access MinIO console at https://localhost:9001 +# For Openshift local: +kubectl port-forward -n ${OC_PROJECT} svc/minio 9001:9001 & + +# Otherwise use: +>>>>>>> 31c028f (feat: Test Openshift local deployment) kubectl port-forward -n ${OC_PROJECT} svc/minio-console 9001:9001 & kubectl port-forward -n ${OC_PROJECT} svc/minio 9000:9000 & +<<<<<<< HEAD # Login with username: `minioadmin`, password: `minioadmin` +======= +#### Install cloud object storage drivers in the cluster +```bash +# Ensure node has labels required by drivers +NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +kubectl label node $NODE topology.kubernetes.io/region=us-east-1 topology.kubernetes.io/zone=us-east-1a +>>>>>>> 31c028f (feat: Test Openshift local deployment) +======= +# Wait for MinIO to be ready: +======= +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +# Apply MinIO deployment +======= + +>>>>>>> 31c028f (feat: Test Openshift local deployment) +kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml -n ${OC_PROJECT} +# Wait for MinIO to be ready +>>>>>>> c15dba1 (Update crc instructions) +kubectl wait --for=condition=ready pod -l app=minio -n ${OC_PROJECT} --timeout=300s + +<<<<<<< HEAD +# Access MinIO Console: +# Port forward to access MinIO console at https://localhost:9001 +======= +#### Access MinIO Console +To access the MinIO console: +```bash +# Port forward to access MinIO console at https://localhost:9001 +# For Openshift local: +kubectl port-forward -n ${OC_PROJECT} svc/minio 9001:9001 & +# Otherwise use: +>>>>>>> 31c028f (feat: Test Openshift local deployment) +kubectl port-forward -n ${OC_PROJECT} svc/minio-console 9001:9001 & +kubectl port-forward -n ${OC_PROJECT} svc/minio 9000:9000 & + +<<<<<<< HEAD +# Login with username: `minioadmin`, password: `minioadmin` +======= +#### Install cloud object storage drivers in the cluster +```bash +# Ensure node has labels required by drivers +NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') +kubectl label node $NODE topology.kubernetes.io/region=us-east-1 topology.kubernetes.io/zone=us-east-1a +>>>>>>> 31c028f (feat: Test Openshift local deployment) + +<<<<<<< HEAD +### Install cloud object storage drivers in the cluster +# Ensure node has labels required by drivers +kubectl label nodes lima-studio topology.kubernetes.io/region=us-east-1 topology.kubernetes.io/zone=us-east-1a +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) + + +======= + +>>>>>>> c15dba1 (Update crc instructions) # Also at this point update `workspace/${DEPLOYMENT_ENV}/env/.env.sh` with... export COS_STORAGE_CLASS=cos-s3-csi-s3fs-sc export NON_COS_STORAGE_CLASS=local-path ``` +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD + +======= +>>>>>>> 31c028f (feat: Test Openshift local deployment) +======= + +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +======= +>>>>>>> 31c028f (feat: Test Openshift local deployment) +>>>>>>> 9018b46 (feat: Test Openshift local deployment) * Once the S3 instance has been created, you can add the credentials and endpoint to the `workspace/${DEPLOYMENT_ENV}/env/.env` file as shown below. ``` @@ -509,13 +957,50 @@ export NON_COS_STORAGE_CLASS=local-path region=us-east ``` +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +======= +* Also at this point update `workspace/${DEPLOYMENT_ENV}/env/.env.sh` with... + ```bash + # Storage classes + export COS_STORAGE_CLASS=cos-s3-csi-s3fs-sc +<<<<<<< HEAD +======= + # For Openshift local: + export NON_COS_STORAGE_CLASS=crc-csi-hostpath-provisioner + # Otherwise use: +>>>>>>> 9018b46 (feat: Test Openshift local deployment) + export NON_COS_STORAGE_CLASS=local-path + ``` + +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= +>>>>>>> c15dba1 (Update crc instructions) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ### Create the required buckets Source the environment variables: +<<<<<<< HEAD +======= ```bash source workspace/${DEPLOYMENT_ENV}/env/env.sh ``` +<<<<<<< HEAD +Run the following script to create the buckets (For Lima VM and minikube only): +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) + +```bash +source workspace/${DEPLOYMENT_ENV}/env/env.sh +``` + +======= +>>>>>>> c15dba1 (Update crc instructions) Create required S3 buckets ```bash python deployment-scripts/create_buckets.py --env-path workspace/${DEPLOYMENT_ENV}/env/.env @@ -555,6 +1040,11 @@ Install postgres: ***Note*** If you have an instance of postgres already installed, following this guide to [uninstall](postgres-uninstall.md). ```bash +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # Export postgres password export POSTGRES_PASSWORD=devPostgresql123 @@ -562,6 +1052,28 @@ export POSTGRES_PASSWORD=devPostgresql123 ./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV # For Lima/Minikube: +======= +# For openshift local(crc): +<<<<<<< HEAD +./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV DO_NOT_SET_SCC + +# Otherwise use: +>>>>>>> 31c028f (feat: Test Openshift local deployment) +======= +# Export postgres password +export POSTGRES_PASSWORD=devPostgresql123 + +# For OpenShift local(CRC): +./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV + +# For Lima/Minikube: +>>>>>>> c15dba1 (Update crc instructions) +======= +./deployment-scripts/install-postgres.sh UPDATE_STORAGE + +# Otherwise use: +>>>>>>> 31c028f (feat: Test Openshift local deployment) +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ./deployment-scripts/install-postgres.sh ``` @@ -571,6 +1083,22 @@ kubectl wait --for=condition=ready pod/postgresql-0 -n ${OC_PROJECT} --timeout=3 ``` Once completed, in terminal you will find some notes on the created postgres database. To prepare for the [create databases](#create-databases) section below, follow these steps.. +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +======= +* Export postgres password: +```bash +export POSTGRES_PASSWORD=devPostgresql123 +``` +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= +>>>>>>> c15dba1 (Update crc instructions) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) * To connect to your database from outside the cluster for [create databases](#create-databases) section below execute the following commands: @@ -589,6 +1117,22 @@ Once completed, in terminal you will find some notes on the created postgres dat pg_port=5432 pg_original_db_name='postgres' ``` +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +======= + > Note: after completing [create databases](#create-databases) section below update `pg_uri` in `workspace/${DEPLOYMENT_ENV}/env/.env` with... + ```bash + pg_uri=postgresql.default.svc.cluster.local + ``` +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ### Create databases @@ -603,7 +1147,22 @@ python deployment-scripts/create_studio_dbs.py --env-path workspace/${DEPLOYMENT Once you create the databases update the pg_uri in `workspace/${DEPLOYMENT_ENV}/env/.env` with ``` +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +pg_uri=postgresql.${OC_PROJECT}.svc.cluster.local +======= +pg_uri=postgresql.default.svc.cluster.local +#pg_uri=127.0.0.1 +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= pg_uri=postgresql.${OC_PROJECT}.svc.cluster.local +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) ``` ## 4. Authenticator setup @@ -619,12 +1178,21 @@ source workspace/$DEPLOYMENT_ENV/env/env.sh #### 1. Keycloak Deploy Keycloak for authentication: +# TODO: Will --disable-route work with openshift local?? ```bash # For Openshift local(CRC): python ./deployment-scripts/update-keycloak-deployment.py --filename deployment-scripts/keycloak-deployment.yaml --env-path workspace/${DEPLOYMENT_ENV}/env/.env > workspace/$DEPLOYMENT_ENV/initialisation/keycloak-deployment.yaml +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= + + +>>>>>>> c15dba1 (Update crc instructions) # Otherwise use: python ./deployment-scripts/update-keycloak-deployment.py --disable-route --filename deployment-scripts/keycloak-deployment.yaml --env-path workspace/${DEPLOYMENT_ENV}/env/.env > workspace/$DEPLOYMENT_ENV/initialisation/keycloak-deployment.yaml @@ -791,10 +1359,40 @@ For Lima and minikube: ```bash export GEOSERVER_URL=http://localhost:3000/geoserver +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +python ./deployment-scripts/update-deployment-template.py --filename deployment-scripts/geoserver-deployment.yaml --proxy-base-url $(printf "http://geofm-geoserver-%s.svc.cluster.local:3000/geoserver" "$OC_PROJECT") --disable-route > workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml +======= +# For openshift local(crc): +python ./deployment-scripts/update-deployment-template.py --storageclass ${NON_COS_STORAGE_CLASS} --filename deployment-scripts/geoserver-deployment.yaml --proxy-base-url $(printf "http://geofm-geoserver-%s.svc.cluster.local:3000/geoserver" "$OC_PROJECT") --disable-route > workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml + + +# Otherwise use: +======= +>>>>>>> c15dba1 (Update crc instructions) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +python ./deployment-scripts/update-deployment-template.py --filename deployment-scripts/geoserver-deployment.yaml --proxy-base-url $(printf "http://geofm-geoserver-%s.svc.cluster.local:3000/geoserver" "$OC_PROJECT") --disable-route > workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml +======= +# For openshift local(crc): +python ./deployment-scripts/update-deployment-template.py --storageclass ${NON_COS_STORAGE_CLASS} --filename deployment-scripts/geoserver-deployment.yaml --proxy-base-url $(printf "http://geofm-geoserver-%s.svc.cluster.local:3000/geoserver" "$OC_PROJECT") --disable-route > workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml + + +# Otherwise use: python ./deployment-scripts/update-deployment-template.py --filename deployment-scripts/geoserver-deployment.yaml --proxy-base-url $(printf "http://geofm-geoserver-%s.svc.cluster.local:3000/geoserver" "$OC_PROJECT") --disable-route > workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml -n ${OC_PROJECT} +>>>>>>> 31c028f (feat: Test Openshift local deployment) +kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml -n ${OC_PROJECT} +>>>>>>> 31c028f (feat: Test Openshift local deployment) + +<<<<<<< HEAD +kubectl apply -f workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml -n ${OC_PROJECT} + +======= +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) ``` Wait for Geoserver to be ready: @@ -854,16 +1452,43 @@ Update `workspace/${DEPLOYMENT_ENV}/env/env.sh` ```bash # Environment vars +<<<<<<< HEAD +<<<<<<< HEAD +export ENVIRONMENT=local # set to 'crc' for Openshift Local(CRC) +======= +export ENVIRONMENT=local +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= export ENVIRONMENT=local # set to 'crc' for Openshift Local(CRC) +>>>>>>> c15dba1 (Update crc instructions) export ROUTE_ENABLED=false # set to true for Openshift Local(CRC) # storage config export SHARE_PIPELINE_PVC=true # set to false for Openshift Local(CRC) export STORAGE_PVC_ENABLED=true +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) +export STORAGE_FILESYSTEM_ENABLED=true # set to false for Openshift Local(CRC) +export CREATE_TUNING_FOLDERS_FLAG=false # set to true for Openshift Local(CRC) +export PIPELINES_V2_INFERENCE_ROOT_FOLDER_VALUE= +export PIPELINES_TERRATORCH_INFERENCE_CREATE_FT_PVC=false +======= +export STORAGE_FILESYSTEM_ENABLED=true +export CREATE_TUNING_FOLDERS_FLAG=false +export PIPELINES_V2_INFERENCE_ROOT_FOLDER_VALUE=/data +>>>>>>> 31c028f (feat: Test Openshift local deployment) +<<<<<<< HEAD +======= export STORAGE_FILESYSTEM_ENABLED=true # set to false for Openshift Local(CRC) export CREATE_TUNING_FOLDERS_FLAG=false # set to true for Openshift Local(CRC) export PIPELINES_V2_INFERENCE_ROOT_FOLDER_VALUE= export PIPELINES_TERRATORCH_INFERENCE_CREATE_FT_PVC=false +>>>>>>> 36f1fcd (Update manual local deployment steps with CRC option) +======= +>>>>>>> 9018b46 (feat: Test Openshift local deployment) # switch off oauth config (optional) export OAUTH_PROXY_ENABLED=false # set to true for Openshift Local(CRC) diff --git a/geospatial-studio/charts/gfm-studio-gateway/values.yaml b/geospatial-studio/charts/gfm-studio-gateway/values.yaml index 5ee5073a..e4476231 100644 --- a/geospatial-studio/charts/gfm-studio-gateway/values.yaml +++ b/geospatial-studio/charts/gfm-studio-gateway/values.yaml @@ -113,7 +113,15 @@ hooks: # Wait for gateway service to be ready before seeding waitForGateway: enabled: true +<<<<<<< HEAD +<<<<<<< HEAD image: docker.io/library/busybox:latest +======= + image: busybox:1.36 +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) +======= + image: docker.io/library/busybox:latest +>>>>>>> 430fb82 (Update busybox image tag) sleepTime: 5 # Mount JSON payloads volumeMounts: - name: sandbox-models-payloads diff --git a/geospatial-studio/charts/pgbouncer/templates/deployment.yaml b/geospatial-studio/charts/pgbouncer/templates/deployment.yaml index b93f9f8a..f7d3c85a 100644 --- a/geospatial-studio/charts/pgbouncer/templates/deployment.yaml +++ b/geospatial-studio/charts/pgbouncer/templates/deployment.yaml @@ -20,6 +20,10 @@ spec: securityContext: runAsUser: {{ .Values.podSecurityContext.runAsUser }} {{- end }} +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) initContainers: - name: setup-config image: {{ .Values.image.repository }}:{{ .Values.image.tag }} @@ -52,6 +56,11 @@ spec: mountPath: /config-source - name: pgbouncer-conf mountPath: /opt/bitnami/pgbouncer/conf +<<<<<<< HEAD +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) containers: - name: pgbouncer image: {{ .Values.image.repository }}:{{ .Values.image.tag }} @@ -62,6 +71,10 @@ spec: securityContext: runAsUser: {{ .Values.containerSecurityContext.runAsUser }} {{- end }} +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) # Use custom command to bypass Bitnami's auto-configuration command: - /bin/bash @@ -69,10 +82,24 @@ spec: - | # Start PgBouncer with our custom config exec pgbouncer /opt/bitnami/pgbouncer/conf/pgbouncer.ini +<<<<<<< HEAD +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) ports: - name: pgbouncer containerPort: {{ .Values.service.port }} protocol: TCP +<<<<<<< HEAD +<<<<<<< HEAD +======= + envFrom: + - configMapRef: + name: {{ .Values.fullnameOverride }}-config +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) env: - name: POSTGRESQL_PASSWORD valueFrom: @@ -91,6 +118,10 @@ spec: port: pgbouncer initialDelaySeconds: 5 periodSeconds: 5 +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) volumeMounts: - name: pgbouncer-conf mountPath: /opt/bitnami/pgbouncer/conf @@ -103,4 +134,9 @@ spec: path: pgbouncer.ini - name: pgbouncer-conf emptyDir: {} +<<<<<<< HEAD +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) {{- end }} diff --git a/geospatial-studio/charts/pgbouncer/values.yaml b/geospatial-studio/charts/pgbouncer/values.yaml index 59fa98e0..452b3e77 100644 --- a/geospatial-studio/charts/pgbouncer/values.yaml +++ b/geospatial-studio/charts/pgbouncer/values.yaml @@ -61,7 +61,15 @@ service: resources: limits: cpu: "1" +<<<<<<< HEAD +<<<<<<< HEAD memory: "1Gi" +======= + memory: "512Mi" +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= + memory: "1Gi" +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) requests: cpu: "250m" memory: "256Mi" diff --git a/geospatial-studio/templates/_helpers.tpl b/geospatial-studio/templates/_helpers.tpl index 3b4f1d5e..32b58938 100644 --- a/geospatial-studio/templates/_helpers.tpl +++ b/geospatial-studio/templates/_helpers.tpl @@ -45,7 +45,15 @@ Used for jobs that need direct database access {{/* Build DATABASE_URI for gateway database +<<<<<<< HEAD +<<<<<<< HEAD Uses pgbouncer pool if enabled, otherwise direct postgres +======= +Uses pgbouncer if enabled, otherwise direct postgres +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +Uses pgbouncer pool if enabled, otherwise direct postgres +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) */}} {{- define "geospatial-studio.postgres.gateway.uri" -}} {{- if .Values.global.postgres.in_cluster_db -}} @@ -53,6 +61,10 @@ postgresql+pg8000://{{ .Values.global.postgresql.auth.username }}:{{ .Values.glo {{- else -}} {{- $host := include "geospatial-studio.postgres.host" . -}} {{- $port := include "geospatial-studio.postgres.port" . -}} +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) {{- if .Values.global.pgbouncer.enabled -}} {{- /* Use PgBouncer pool for gateway API */ -}} postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port }}/geostudio_api_pool @@ -61,11 +73,26 @@ postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.globa postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port }}/{{ .Values.global.postgres.dbs.gateway }} {{- end -}} {{- end -}} +<<<<<<< HEAD +======= +postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port }}/{{ .Values.global.postgres.dbs.gateway }} +{{- end -}} +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) {{- end }} {{/* Build AUTH_DATABASE_URI for auth database +<<<<<<< HEAD +<<<<<<< HEAD Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +======= +Uses pgbouncer if enabled, otherwise direct postgres +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) */}} {{- define "geospatial-studio.postgres.auth.uri" -}} {{- if .Values.global.postgres.in_cluster_db -}} @@ -73,14 +100,29 @@ postgresql+pg8000://{{ .Values.global.postgresql.auth.username }}:{{ .Values.glo {{- else -}} {{- $host := include "geospatial-studio.postgres.host" . -}} {{- $port := include "geospatial-studio.postgres.port" . -}} +<<<<<<< HEAD +<<<<<<< HEAD +{{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= {{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port }}/{{ .Values.global.postgres.dbs.auth }} {{- end -}} {{- end }} {{/* Build MLFLOW_DATABASE_URI for mlflow database +<<<<<<< HEAD +<<<<<<< HEAD +Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +======= +Uses pgbouncer if enabled, otherwise direct postgres +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) */}} {{- define "geospatial-studio.postgres.mlflow.uri" -}} {{- if .Values.global.postgres.in_cluster_db -}} @@ -88,30 +130,67 @@ postgresql+pg8000://{{ .Values.global.postgresql.auth.username }}:{{ .Values.glo {{- else -}} {{- $host := include "geospatial-studio.postgres.host" . -}} {{- $port := include "geospatial-studio.postgres.port" . -}} +<<<<<<< HEAD +<<<<<<< HEAD {{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +{{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port }}/{{ .Values.global.postgres.dbs.mlflow }} {{- end -}} {{- end }} {{/* Build orchestration database URI for pipelines +<<<<<<< HEAD +<<<<<<< HEAD +Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +======= +Uses pgbouncer if enabled, otherwise direct postgres +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) */}} {{- define "geospatial-studio.postgres.orchestrate.uri" -}} {{- $host := include "geospatial-studio.postgres.host" . -}} {{- $port := include "geospatial-studio.postgres.port" . -}} +<<<<<<< HEAD +<<<<<<< HEAD +{{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= {{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) postgresql+pg8000://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port | int }}/{{ .Values.global.postgres.dbs.gateway }} {{- end }} {{/* Build MLflow backend URI (without +pg8000 driver) +<<<<<<< HEAD +<<<<<<< HEAD Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +======= +Uses pgbouncer if enabled, otherwise direct postgres +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= +Uses pgbouncer if enabled (pool name matches database name), otherwise direct postgres +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) */}} {{- define "geospatial-studio.postgres.mlflow.backend.uri" -}} {{- $host := include "geospatial-studio.postgres.host" . -}} {{- $port := include "geospatial-studio.postgres.port" . -}} +<<<<<<< HEAD +<<<<<<< HEAD +{{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +======= +>>>>>>> 42c9d82 (✨ feat(pgbouncer): add connection pooling with deployment improvements) +======= {{- /* Pool name matches database name, so same URI works for both PgBouncer and direct */ -}} +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) postgresql://{{ .Values.global.postgres.postgres_user }}:{{ .Values.global.postgres.postgres_password }}@{{ $host }}:{{ $port }}/{{ .Values.global.postgres.dbs.mlflow }} {{- end }} diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..b831c9e2 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +# © Copyright IBM Corporation 2025 +# SPDX-License-Identifier: Apache-2.0 + + +# For Python 3.11 +-r requirements.txt + +black +flake8 +hunter==3.7.0 +IPython>=8.18,<9.0.0 +rich>=14.0.0 + +# Needed by conftest +fastapi +httpx +sqlalchemy +sqlalchemy-utils + +# Test +pytest +pytest-cov +pre-commit diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..195567e0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,163 @@ +import os +import shlex +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import pytest +from dotenv import find_dotenv, load_dotenv +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy_utils import create_database, database_exists, drop_database + +from tests.integration.gateway import GatewayApiClient + +from .integration.utils import make_timestamped_name + + +# ------------------------------- +# Common configurations for Integration and Unit tests +# ------------------------------- +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + # # --- load env and set test DB (unit tests support) --- + # load_dotenv(".env", override=True) + # db_url = os.environ.get("DATABASE_URI", str(settings.DATABASE_URI)) + "_test" + # settings.DATABASE_URI = db_url + # settings.AUTH_ENABLED = False + # + # --- register markers (integration tests support) --- + config.addinivalue_line( + "markers", "integration: marks tests that hit live external services" + ) + + +# ------------------------------- +# Unit Tests Support +# ------------------------------- +def _db_session(): + db_url = str(settings.DATABASE_URI) + if not database_exists(db_url): + create_database(db_url) + + engine = create_engine(db_url) + TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + return db_url, TestingSessionLocal, engine + + +@pytest.fixture(scope="session") +def db(): + """Fixture sets up and tears down PostgreSQL test database.""" + db_url, TestingSessionLocal, engine = _db_session() + Base.metadata.create_all(bind=engine) + + session = TestingSessionLocal() + yield session + + session.close_all() + drop_database(db_url) + + +def override_get_db(): + try: + _, TestingSessionLocal, _ = _db_session() + db = TestingSessionLocal() + yield db + finally: + db.close() + + +@pytest.fixture(scope="module") +def client(db): + """Sets up FastAPI test client for sending HTTP requests during testing.""" + app.dependency_overrides[get_db] = override_get_db + client = TestClient(app) + return client + + +@pytest.fixture(scope="session") +def repo_root() -> Path: + toplevel = ( + subprocess.check_output(shlex.split("git rev-parse --show-toplevel")) + .decode() + .strip() + ) + return Path(toplevel) + + +def pytest_addoption(parser): + # https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser.add_argument + parser.addoption( + "--tune-id", + action="store", + default=None, + help="A ID of a geotune" "from the Fine Tune Service", + ) + + +@pytest.fixture(scope="session") +def tune_id(pytestconfig): + tune_id = pytestconfig.getoption("tune_id") + if not tune_id: + pytest.skip("[unit] skipped due to missing tune-id") + return tune_id + + +@pytest.fixture(scope="session") +def token() -> Optional[str]: + """ + Returns a valid option for IBM Verify to authenticate. Only used for interactive + integration tests. This value can be provided by .envrc or .env + """ + token = os.environ.get("TOKEN") + if not token: + pytest.skip("[unit] skipped due to missing authentication TOKEN in environment") + return token + + +# ------------------------------- +# Integration Tests Support (APIs) +# ------------------------------- +def _int_env(name: str, default: str | None = None, required: bool = False) -> str: + v = os.getenv(name, default) + if required and not v: + pytest.skip(f"[integration] Missing env var {name}; skipping") + return v or "" + + +@pytest.fixture(scope="session") +def gateway() -> GatewayApiClient: + """ + External Gateway client. + Requires: + BASE_GATEWAY_URL and API_KEY in .env file + """ + + # Make sure .env is found whether you run from repo root or a subfolder + load_dotenv(find_dotenv(usecwd=True), override=True) + + base_url = os.getenv("BASE_GATEWAY_URL") + api_key = os.getenv("API_KEY") + + if not base_url or not api_key: + pytest.skip("[integration] Missing BASE_GATEWAY_URL or API_KEY; skipping") + + # Normalize possible CRLF or stray quotes from copy/paste + api_key = api_key.strip().strip('"').strip("'") + if api_key.endswith("\r"): + api_key = api_key[:-1] + + return GatewayApiClient(base_url=base_url, api_key=api_key) + + +@pytest.fixture(scope="function") +def name_factory(): + """One fixed timestamp per test so related names match.""" + fixed_now = datetime.now(timezone.utc) + + def _make(base: str, *, prefix: str | None = None, ext: str = "") -> str: + return make_timestamped_name(base, prefix=prefix, ext=ext, now=fixed_now) + + return _make diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/data/__init__.py b/tests/integration/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/data/api_inference_models_and_inference.py b/tests/integration/data/api_inference_models_and_inference.py new file mode 100644 index 00000000..964e0a31 --- /dev/null +++ b/tests/integration/data/api_inference_models_and_inference.py @@ -0,0 +1,50 @@ +""" +Example payloads for integration tests. +Each payload is a Python dict, instead of JSON files. +""" + +# Base valid model payload +SANDBOX_MODEL = { + "display_name": "integration-test-sandbox-model", + "description": "[Integration Test 175933_01we_10oct_25] Early-access test model made available for demonstration or limited user evaluation. These models may include incomplete features or evolving performance characteristics and are intended for feedback and experimentation before full deployment.", + "pipeline_steps": [ + {"status": "READY", "process_id": "url-connector", "step_number": 0}, + {"status": "WAITING", "process_id": "push-to-geoserver", "step_number": 1}, + ], + "geoserver_push": [], + "model_input_data_spec": [ + { + "bands": [], + "connector": "sentinelhub", + "collection": "hls_s30", + "file_suffix": "S2Hand", + } + ], + "postprocessing_options": {}, + "sharable": False, + "model_onboarding_config": { + "fine_tuned_model_id": "", + "model_configs_url": "", + "model_checkpoint_url": "", + }, + "latest": True, + "version": 1.0, +} + +DEFAULT_ONBOARD_INFERENCE_MODEL = { + "model_framework": "terratorch", + "model_id": "string", + "model_name": "string", + "model_configs_url": "https://example.com/", + "model_checkpoint_url": "https://example.com/", + "deployment_type": "gpu", + "resources": { + "requests": {"cpu": "6", "memory": "16G"}, + "limits": {"cpu": "12", "memory": "32G"}, + }, + "gpu_resources": { + "requests": {"nvidia.com/gpu": "1"}, + "limits": {"nvidia.com/gpu": "1"}, + }, + "inference_container_image": "", +} diff --git a/tests/integration/gateway.py b/tests/integration/gateway.py new file mode 100644 index 00000000..458b705d --- /dev/null +++ b/tests/integration/gateway.py @@ -0,0 +1,126 @@ +import os +from typing import Any, Dict, Optional +from urllib.parse import urljoin, urlparse + +import requests +from dotenv import find_dotenv, load_dotenv +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + + +class GatewayApiClient: + def __init__( + self, + base_url: str, + api_key: Optional[str] = None, + timeout: int = 60, + verify: Optional[bool | str] = False, + proxies: Optional[Dict[str, str]] = None, + ): + self.base_url = base_url.rstrip("/") + "/" + self.timeout = timeout + self.verify = verify + self.proxies = proxies + + self.session = requests.Session() + retry = Retry( + total=5, + backoff_factor=0.3, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset(("GET", "POST", "PATCH", "DELETE")), + ) + adapter = HTTPAdapter(max_retries=retry) + self.session.mount("https://", adapter) + self.session.mount("http://", adapter) + + self.session.headers.update({"Accept": "application/json"}) + + if api_key: + key = api_key.strip().strip('"').strip("'") + if key.endswith("\r"): + key = key[:-1] + self.session.headers.update({"X-Api-Key": key}) + + def _url(self, path: str) -> str: + return urljoin(self.base_url, path.lstrip("/")) + + # Gateway methods + def get(self, path: str, **kwargs) -> requests.Response: + return self.session.get( + self._url(path), + timeout=self.timeout, + verify=self.verify, + proxies=self.proxies, + **kwargs, + ) + + def post(self, path: str, json: Any | None = None, **kwargs) -> requests.Response: + headers = {**self.session.headers, "Content-Type": "application/json"} + return self.session.post( + self._url(path), + headers=headers, + json=json or {}, + timeout=self.timeout, + verify=self.verify, + proxies=self.proxies, + **kwargs, + ) + + def put(self, path: str, json: Any | None = None, **kwargs) -> requests.Response: + user_headers = kwargs.pop("headers", None) or {} + files = kwargs.pop("files", None) + + headers = {**self.session.headers, **user_headers} + + if files is not None: + # multipart/form-data PUT + return self.session.put( + self._url(path), + headers=headers, + files=files, + data=kwargs.pop("data", {}), + timeout=self.timeout, + verify=self.verify, + proxies=self.proxies, + **kwargs, + ) + else: + # JSON PUT + headers.setdefault("Content-Type", "application/json") + return self.session.put( + self._url(path), + headers=headers, + json=json or {}, + timeout=self.timeout, + verify=self.verify, + proxies=self.proxies, + **kwargs, + ) + + def patch(self, path: str, json: Any | None = None, **kwargs) -> requests.Response: + headers = {**self.session.headers, "Content-Type": "application/json"} + return self.session.patch( + self._url(path), + headers=headers, + json=json or {}, + timeout=self.timeout, + verify=self.verify, + proxies=self.proxies, + **kwargs, + ) + + def delete(self, path: str, **kwargs) -> requests.Response: + return self.session.delete( + self._url(path), + timeout=self.timeout, + verify=self.verify, + proxies=self.proxies, + **kwargs, + ) + + @classmethod + def from_env(cls) -> "GatewayApiClient": + load_dotenv(find_dotenv(usecwd=True)) + base = os.environ["BASE_GATEWAY_URL"] + api_key = os.environ["API_KEY"] # uses the key *value* (pak-...), not the id + return cls(base_url=base, api_key=api_key) diff --git a/tests/integration/test_inference_models.py b/tests/integration/test_inference_models.py new file mode 100644 index 00000000..269dc6ae --- /dev/null +++ b/tests/integration/test_inference_models.py @@ -0,0 +1,579 @@ +import logging +import os + +import pytest + +from .data import api_inference_models_and_inference as payloads +from .utils import redacted_response_text + +log = logging.getLogger("gateway_tests") +log.setLevel(logging.INFO) + +pytestmark = pytest.mark.integration + + +# ====================== Helper Functions ======================== +def env_eval(name: str, default: str = "") -> bool: + val = os.getenv(name, default).strip().lower() + return val in {"1", "true", "yes", "y", "on"} + + +# ====================== Fixtures ================================ +@pytest.fixture() +def amo_tests_activate(): + if not env_eval("ALLOW_AMO_TESTS"): + msg = "\nSet ALLOW_AMO_TESTS=1 to run AMO tasks" + log.info(msg) + pytest.skip(msg) + + +@pytest.fixture() +def create_model(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _create_model(model_payload): + + # PAYLOAD + payload = model_payload + + # QUERY + r = gateway.post("/v2/models", json=model_payload) + log.info( + "POST /v2/models \nPayload:\n%s \nResponse (redacted)(%s):\n%s", + payload, + r.status_code, + redacted_response_text(r), + ) + assert r.status_code == 201 + body = r.json() + model_id = body.get("id") + assert model_id + return body + + return _create_model + + +@pytest.fixture() +def list_models(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _list_models(**overrides): + + # PARAMS + params = {"limit": 25, "skip": 0, **overrides} + + # QUERY + r = gateway.get("/v2/models", params=params) + log.info( + "GET /v2/models -> (%s)\nPARAMS\n%s \nResponse (redacted)(%s)", + r.status_code, + params, + redacted_response_text(r), + ) + assert r.status_code == 200 + body = r.json() + assert isinstance(body, dict), type(body) + assert "results" in body and isinstance(body["results"], list) + return body + + return _list_models + + +@pytest.fixture() +def deploy_model_with_amo(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _deploy_model_with_amo(model_id, **overrides): + + # PARAMS + # model_id + params = { + "fine_tuned_model_id": model_id, + "model_configs_url": "", # Need url + "model_checkpoint_url": "", # Need url + **overrides, + } + + # QUERY + r = gateway.post(f"/v2/models/{model_id}/deploy", params=params) + log.info( + "POST /v2/models/%s/deploy -> (%s)\nPARAMS\n%s \nResponse (redacted)(%s)", + model_id, + r.status_code, + params, + redacted_response_text(r), + ) + + body = r.json() + assert isinstance(body, dict), type(body) + return body, r + + return _deploy_model_with_amo + + +@pytest.fixture() +def update_model(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _update_model(model_id: str, *, replace: bool = False, **overrides): + + # PARAMS + # model_id + + # PAYLOAD + defaults = { + "display_name": "", + "description": "string", + "model_url": "https://example.com/", + "pipeline_steps": [{"additionalProp1": {}}], + "geoserver_push": [{"additionalProp1": {}}], + "model_input_data_spec": [{"additionalProp1": {}}], + "postprocessing_options": {"additionalProp1": {}}, + "sharable": True, + "model_onboarding_config": { + "fine_tuned_model_id": "string", + "model_configs_url": "string", + "model_checkpoint_url": "string", + }, + "latest": True, + **overrides, + } + payload = overrides if replace else {**defaults, **overrides} + + # QUERY + r = gateway.patch(f"/v2/models/{model_id}", json=payload) + log.info( + "PATCH /v2/models/%s -> (%s)\nPayload\n%s \nResponse (redacted)(%s)", + model_id, + r.status_code, + payload, + redacted_response_text(r), + ) + + body = r.json() + assert isinstance(body, dict), type(body) + return body, r + + return _update_model + + +@pytest.fixture() +def get_model(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _get_model(model_id): + + # PARAMS + # model_id + + # QUERY + r = gateway.get(f"/v2/models/{model_id}") + log.info( + "GET /v2/models/%s -> (%s)\nResponse (redacted)(%s)", + model_id, + r.status_code, + redacted_response_text(r), + ) + + body = r.json() + assert isinstance(body, dict), type(body) + return body, r + + return _get_model + + +@pytest.fixture() +def delete_model(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _delete_model(model_id): + + # PARAMS + # model_id + + # QUERY + r = gateway.delete(f"/v2/models/{model_id}") + log.info( + "\nDELETE /v2/models/%s -> (%s)\nResponse (redacted)(%s)", + model_id, + r.status_code, + redacted_response_text(r), + ) + return r + + return _delete_model + + +@pytest.fixture() +def retrieve_amo_task(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _retrieve_amo_task(model_id): + + # PARAMS + # model_id + + # QUERY + r = gateway.get(f"/v2/amo-tasks/{model_id}") + log.info( + "\nGET /v2/amo-tasks/%s -> (%s)\nResponse (redacted)(%s)", + model_id, + r.status_code, + redacted_response_text(r), + ) + body = r.json() + + return body, r + + return _retrieve_amo_task + + +@pytest.fixture() +def offboard_inference_model(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _offboard_inference_model(model_id): + + # PARAMS + # model_id + + # QUERY + r = gateway.delete(f"/v2/amo-tasks/{model_id}") + log.info( + "\nDELETE /v2/amo-tasks/%s -> (%s)\nResponse (redacted)(%s)", + model_id, + r.status_code, + redacted_response_text(r), + ) + + body = r.json() + assert isinstance(body, dict), type(body) + return body, r + + return _offboard_inference_model + + +@pytest.fixture() +def onboard_inference_model(gateway): + """ + Call /v2/datasets and return the parsed body. + Each call will hit the endpoint afresh. + """ + + def _onboard_inference_model(*, replace: bool = False, **overrides): + + # PAYLOAD + DEFAULT_ONBOARD_INFERENCE_MODEL = { + "model_framework": "terratorch", + "model_id": "string", + "model_name": "string", + "model_configs_url": "https://example.com/", + "model_checkpoint_url": "https://example.com/", + "deployment_type": "gpu", + "resources": { + "requests": {"cpu": "6", "memory": "16G"}, + "limits": {"cpu": "12", "memory": "32G"}, + }, + "gpu_resources": { + "requests": {"nvidia.com/gpu": "1"}, + "limits": {"nvidia.com/gpu": "1"}, + }, + "inference_container_image": "", + **overrides, + } + payload = ( + overrides if replace else {**DEFAULT_ONBOARD_INFERENCE_MODEL, **overrides} + ) + + # QUERY + r = gateway.post("/v2/amo-tasks", json=payload) + log.info( + "\nPOST /v2/amo-tasks -> (%s)\nPayload\n%s \nResponse (redacted)(%s)", + r.status_code, + payload, + redacted_response_text(r), + ) + + body = r.json() + assert isinstance(body, dict), type(body) + return body, r + + return _onboard_inference_model + + +# ======================== Tests ================================= +# --------[create_model] Tests ----------------------------------- +def test_create_model_fixture(create_model, caplog): + caplog.set_level(logging.INFO, logger="gateway_tests") + + body = create_model(payloads.SANDBOX_MODEL) + assert "PENDING" == body["status"] + + +# --------[list_models] Tests ------------------------------------ +def test_list_models_fixture(list_models, caplog): + caplog.set_level(logging.INFO, logger="gateway_tests") + body = list_models(limit=1) + assert "total_records" in body + assert isinstance(body["results"], list) + + +# --------[deploy_model_with_amo] Tests -------------------------- +def test_deploy_model_with_amo_fixture_no_urls( + deploy_model_with_amo, list_models, caplog +): + # PARAMS + list_model_body = list_models() + model_id = list_model_body["results"][0][ + "id" + ] # [TODO] Check different models for errors + + params = { + "fine_tuned_model_id": model_id, + "model_configs_url": "", # Intentional missing url for testing + "model_checkpoint_url": "", # Intentional missing url for testing + } + + caplog.set_level(logging.INFO, logger="gateway_tests") + body, r = deploy_model_with_amo(model_id, **params) + assert isinstance(body, dict) + assert r.status_code == 422 + assert "detail" in body and isinstance(body["detail"], list) + assert "both model_checkpoint_url" in body["detail"][0]["msg"].lower() + assert "model_config" in body["detail"][0]["msg"].lower() + assert "should be" in body["detail"][0]["msg"].lower() + + +def test_deploy_model_with_amo_fixture_no_urls_and_missing_model_id( + deploy_model_with_amo, caplog +): + # PARAMS + # Sample nonexistent model_id + model_id = "e436969b-24bf-46e5-ac6f-7d0653da0f14" + + params = { + "fine_tuned_model_id": model_id, + "model_configs_url": "", # Intentional missing url for testing + "model_checkpoint_url": "", # Intentional missing url for testing + } + + caplog.set_level(logging.INFO, logger="gateway_tests") + body, r = deploy_model_with_amo(model_id, **params) + assert isinstance(body, dict) + assert r.status_code == 404 + assert "Model not found" in body["detail"] + + +# --------[update_model] Tests ----------------------------------- +def test_update_model_display_name(name_factory, update_model, list_models, caplog): + # PARAMS + list_model_body = list_models() + model_id = list_model_body["results"][0]["id"] + + caplog.set_level(logging.INFO, logger="gateway_tests") + + # Payload + display_name = name_factory(base="integration-test") + + payload = {"display_name": display_name} + + body, r = update_model(model_id, replace=True, **payload) + assert isinstance(body, dict) + assert r.status_code == 201 + assert display_name == body["display_name"] + + +def test_update_model_display_name_for_missing_model_id( + name_factory, update_model, caplog +): + # PARAMS + # Sample nonexistent model_id + model_id = "e436969b-24bf-46e5-ac6f-7d0653da0f14" + + caplog.set_level(logging.INFO, logger="gateway_tests") + + # Payload + display_name = name_factory(base="integration-test") + + payload = {"display_name": display_name} + + body, r = update_model(model_id, replace=True, **payload) + assert isinstance(body, dict) + assert r.status_code == 404 + assert "Model not found" == body["detail"] + + +# --------[get_model] Tests -------------------------------------- +def test_get_model_(get_model, list_models, caplog): + caplog.set_level(logging.INFO, logger="gateway_tests") + + # PARAMS + list_model_body = list_models() + model_id = list_model_body["results"][0]["id"] + + body, r = get_model(model_id) + assert isinstance(body, dict) + assert r.status_code == 200 + + +def test_get_model_with_nonexisting_model_id(get_model, caplog): + caplog.set_level(logging.INFO, logger="gateway_tests") + + # PARAMS + # Sample nonexistent model_id + model_id = "e436969b-24bf-46e5-ac6f-7d0653da0f14" + + body, r = get_model(model_id) + assert isinstance(body, dict) + assert r.status_code == 404 + assert "Model not found" == body["detail"] + + +# --------[delete_model] Tests ----------------------------------- +def test_delete_model_create_then_delete( + name_factory, create_model, delete_model, caplog +): + caplog.set_level(logging.INFO, logger="gateway_tests") + + create_model_body = create_model(payloads.SANDBOX_MODEL) + model_id = create_model_body["id"] + delete_model_r = delete_model(model_id) + assert delete_model_r.status_code == 204 + + +# --------[retrieve_amo_task](2) Tests ------------------------------ +def test_retrieve_amo_task_error( + amo_tests_activate, retrieve_amo_task, list_models, caplog +): + caplog.set_level(logging.INFO, logger="gateway_tests") + + # PARAMS + list_model_body = list_models() + model_id = list_model_body["results"][0]["id"] + + body, r = retrieve_amo_task(model_id) + assert r.status_code == 422 + assert "Model ID must not exceed 30 characters." == body["detail"] + + +def test_retrieve_amo_task_onboard_inference_model_then_retrieve_amo_task( + amo_tests_activate, name_factory, onboard_inference_model, retrieve_amo_task, caplog +): + caplog.set_level(logging.INFO, logger="gateway_tests") + + # SETUP + # ---payload + model_name = name_factory(base="integration-test") + model_id = name_factory(base="test") + model_id_amo_compatible = model_id.replace("_", "-")[:30] + payload_overrides = { + "model_name": model_name, + "model_id": model_id_amo_compatible, # confusing as model_id has 2 meanings + } + # --- fixture query + body, r = onboard_inference_model(**payload_overrides) + + # FIXTURE QUERY + body, r = retrieve_amo_task(model_id_amo_compatible) + + # TEST + assert r.status_code == 200 + assert f"amo-{model_id_amo_compatible}" == body["model_id"] + + +# --------[offboard_inference_model] Tests ----------------------- +def test_onboard_inference_model_onboard_then_offboard_inference_model( + amo_tests_activate, + name_factory, + onboard_inference_model, + offboard_inference_model, + caplog, +): + caplog.set_level(logging.INFO, logger="gateway_tests") + + # SETUP + # ---payload + model_name = name_factory(base="integration-test") + model_id = name_factory(base="test") + model_id_amo_compatible = model_id.replace("_", "-")[:30] + payload_overrides = { + "model_name": model_name, + "model_id": model_id_amo_compatible, # confusing as model_id has 2 meanings + } + # --- fixture query + body, r = onboard_inference_model(**payload_overrides) + + # FIXTURE QUERY + offboard_inference_model_body, offboard_inference_model_r = ( + offboard_inference_model(model_id_amo_compatible) + ) + + # TEST + assert offboard_inference_model_r.status_code == 200 + assert ( + "Model offboarding request submitted" + == offboard_inference_model_body["message"] + ) + + +# --------[onboard_inference_model] Tests ------------------------ +def test_onboard_inference_model_( + amo_tests_activate, + name_factory, + onboard_inference_model, + offboard_inference_model, + caplog, +): + caplog.set_level(logging.INFO, logger="gateway_tests") + + # PAYLOAD + model_name = name_factory(base="integration-test") + model_id = name_factory(base="test") + model_id_amo_compatible = model_id.replace("_", "-")[:30] + payload_overrides = { + "model_name": model_name, + "model_id": model_id_amo_compatible, # confusing as model_id has 2 meanings + } + + # FIXTURE QUERY + body, r = onboard_inference_model(**payload_overrides) + + # TEST + assert r.status_code == 200 + + # TEARDOWN + # FIXTURE QUERY + offboard_inference_model_body, offboard_inference_model_r = ( + offboard_inference_model(model_id_amo_compatible) + ) + + # TEST + assert offboard_inference_model_r.status_code == 200 + assert ( + "Model offboarding request submitted" + == offboard_inference_model_body["message"] + ) diff --git a/tests/integration/utils.py b/tests/integration/utils.py new file mode 100644 index 00000000..7ee403e8 --- /dev/null +++ b/tests/integration/utils.py @@ -0,0 +1,106 @@ +import json +import re +from datetime import datetime, timezone +from typing import Any, Dict + +__all__ = [ + "REDACT_KEYS", + "mask_secret_string", + "redact_obj", + "redacted_response_text", +] + +# redact these common secret fields if present +REDACT_KEYS = { + "value", + "token", + "access_token", + "api_token", + "apiKey", + "api_key", + "secret", + "password", +} + + +def mask_secret_string(s: str) -> str: + """Redact keys and replace api_keys with pak-****""" + if not isinstance(s, str) or not s: + return "" + # mask api_keys with pak-**** + if s.startswith("pak-"): + return "pak-****" + # mask other auth tokens (JWT-ish) + if re.fullmatch(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", s): + return "***TOKEN***" + return "****" + + +def redact_obj(obj: Any) -> Any: + """Redact dict/list/str recursively, preserving structure.""" + if isinstance(obj, dict): + out: Dict[str, Any] = {} + for k, v in obj.items(): + if k in REDACT_KEYS and isinstance(v, str): + out[k] = mask_secret_string(v) + else: + out[k] = redact_obj(v) + return out + if isinstance(obj, list): + return [redact_obj(x) for x in obj] + if isinstance(obj, str): + s = obj + # mask api_keys with pak-**** + s = re.sub(r"\bpak-[A-Za-z0-9]+\b", "pak-****", s) + # mask other auth tokens (JWT-ish) + s = re.sub( + r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", + "***TOKEN***", + s, + ) + return s + return obj + + +def redacted_response_text(r) -> str: + """Render a response with secrets redacted. + Tries JSON first; falls back to plaintext masking. + """ + try: + data = r.json() + return json.dumps(redact_obj(data), indent=2, ensure_ascii=False) + except Exception: + txt = getattr(r, "text", "") or "" + txt = re.sub(r"\bpak-[A-Za-z0-9]+\b", "pak-****", txt) + txt = re.sub( + r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", + "***TOKEN***", + txt, + ) + return txt + + +def make_timestamped_name( + base: str, + *, + prefix: str | None = None, + ext: str = "", + now: datetime | None = None, +) -> str: + """ + Build: --__ + """ + now = now or datetime.now(timezone.utc) + epoch6 = str(int(now.timestamp()))[:6] + yyyymmdd = now.strftime("%Y%m%d") + hhmmss = now.strftime("%H%M%S") + + def norm(s: str) -> str: + return s.strip().replace(" ", "_") + + parts = [] + if prefix: + parts.append(norm(prefix)) + parts.append(norm(base)) + stem = "-".join(parts) + f"-{epoch6}_{yyyymmdd}_{hhmmss}" + return stem + (ext or "") diff --git a/uninstall.sh b/uninstall.sh index 08dd27a1..8dd62290 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -104,6 +104,8 @@ echo "Waiting for PostgreSQL pods to terminate..." $KUBECTL_CMD wait --for=delete pod -l app.kubernetes.io/name=postgresql -n $OC_PROJECT --timeout=300s 2>/dev/null || echo "No PostgreSQL pods found or timeout reached" echo "----------------------------------------------------------------------" +<<<<<<< HEAD +<<<<<<< HEAD echo "-------------------- Deleting Deployments --------------------------" echo "----------------------------------------------------------------------" @@ -144,6 +146,69 @@ $KUBECTL_CMD delete jobs -l app.kubernetes.io/instance=studio -n $OC_PROJECT 2>/ # Delete jobs by name pattern (for jobs without the label) $KUBECTL_CMD get jobs -n $OC_PROJECT -o name | grep -E "(geofm-|gfm-|gateway-)" | xargs -r $KUBECTL_CMD delete -n $OC_PROJECT 2>/dev/null || echo "No additional Studio jobs found" +======= +echo "-------------------- Deleting PVCs ---------------------------------" +echo "----------------------------------------------------------------------" + +# Delete Redis PVCs +$KUBECTL_CMD delete pvc redis-data-geofm-redis-master-0 -n $OC_PROJECT 2>/dev/null || echo "Redis master PVC not found" +$KUBECTL_CMD delete pvc redis-data-geofm-redis-replicas-0 -n $OC_PROJECT 2>/dev/null || echo "Redis replica PVC not found" + +# Delete PostgreSQL PVC +$KUBECTL_CMD delete pvc data-postgresql-0 -n $OC_PROJECT 2>/dev/null || echo "PostgreSQL PVC not found" + +# Delete all Studio-related PVCs +echo "Deleting all Studio-related PVCs..." +$KUBECTL_CMD get pvc -n $OC_PROJECT -o name | grep -E "(gfm-|geofm-|inference-|generic-)" | xargs -r $KUBECTL_CMD delete -n $OC_PROJECT 2>/dev/null || echo "No additional Studio PVCs found" + +echo "----------------------------------------------------------------------" +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) +echo "-------------------- Deleting Deployments --------------------------" +echo "----------------------------------------------------------------------" + +# Delete Geoserver +if [ -f "workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml" ]; then + echo "Deleting Geoserver..." + $KUBECTL_CMD delete -f workspace/$DEPLOYMENT_ENV/initialisation/geoserver-deployment.yaml -n $OC_PROJECT 2>/dev/null || echo "Geoserver not found" +fi + +# Delete Keycloak +if [ -f "workspace/$DEPLOYMENT_ENV/initialisation/keycloak-deployment.yaml" ]; then + echo "Deleting Keycloak..." + $KUBECTL_CMD delete -f workspace/$DEPLOYMENT_ENV/initialisation/keycloak-deployment.yaml -n $OC_PROJECT 2>/dev/null || echo "Keycloak not found" +fi + +# Delete MinIO +if [ -f "workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml" ]; then + echo "Deleting MinIO..." + $KUBECTL_CMD delete -f workspace/$DEPLOYMENT_ENV/initialisation/minio-deployment.yaml -n $OC_PROJECT 2>/dev/null || echo "MinIO not found" +fi + +echo "----------------------------------------------------------------------" +echo "-------------------- Deleting Jobs and Pods ------------------------" +echo "----------------------------------------------------------------------" + +# Delete populate buckets jobs/pods +if [ -f "workspace/$DEPLOYMENT_ENV/initialisation/populate-buckets-with-initial-data.yaml" ]; then + $KUBECTL_CMD delete -f workspace/$DEPLOYMENT_ENV/initialisation/populate-buckets-with-initial-data.yaml -n $OC_PROJECT 2>/dev/null || echo "Populate buckets job not found" +fi + +if [ -f "workspace/$DEPLOYMENT_ENV/initialisation/populate-buckets-default-pvc.yaml" ]; then + $KUBECTL_CMD delete -f workspace/$DEPLOYMENT_ENV/initialisation/populate-buckets-default-pvc.yaml -n $OC_PROJECT 2>/dev/null || echo "Populate buckets PVC job not found" +fi + +# Delete any remaining jobs +<<<<<<< HEAD +$KUBECTL_CMD delete jobs -l app.kubernetes.io/instance=studio -n $OC_PROJECT 2>/dev/null || echo "No Studio jobs found" +>>>>>>> 1d5df52 (♻️ refactor(uninstall): enhance cleanup script for reliable resource removal) +======= +echo "Deleting all Studio-related jobs..." +$KUBECTL_CMD delete jobs -l app.kubernetes.io/instance=studio -n $OC_PROJECT 2>/dev/null || echo "No Studio jobs with label found" + +# Delete jobs by name pattern (for jobs without the label) +$KUBECTL_CMD get jobs -n $OC_PROJECT -o name | grep -E "(geofm-|gfm-|gateway-)" | xargs -r $KUBECTL_CMD delete -n $OC_PROJECT 2>/dev/null || echo "No additional Studio jobs found" +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) echo "----------------------------------------------------------------------" echo "-------------------- Deleting Secrets ------------------------------" @@ -217,6 +282,10 @@ if [ -f "workspace/$DEPLOYMENT_ENV/initialisation/ibm-object-csi-driver/cos-s3-c fi echo "----------------------------------------------------------------------" +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) echo "-------------------- Deleting PVCs ---------------------------------" echo "----------------------------------------------------------------------" @@ -232,6 +301,11 @@ echo "Deleting all Studio-related PVCs..." $KUBECTL_CMD get pvc -n $OC_PROJECT -o name | grep -E "(gfm-|geofm-|inference-|generic-)" | xargs -r $KUBECTL_CMD delete -n $OC_PROJECT 2>/dev/null || echo "No additional Studio PVCs found" echo "----------------------------------------------------------------------" +<<<<<<< HEAD +======= +>>>>>>> 1d5df52 (♻️ refactor(uninstall): enhance cleanup script for reliable resource removal) +======= +>>>>>>> b566d2e (✨ feat(pgbouncer): add isolated connection pools to prevent service starvation) echo "-------------------- Removing Node Labels --------------------------" echo "----------------------------------------------------------------------"