Fix for Load baln error #343
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: [push, pull_request] | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ['3.13'] | |
| env: | |
| REALM: test | |
| USER: oauth_user | |
| PASSWORD: password | |
| CLIENT_ID: vertica | |
| CLIENT_SECRET: P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Kubernetes (KinD) | |
| uses: helm/kind-action@v1.8.0 | |
| with: | |
| cluster_name: vertica-ci | |
| node_image: kindest/node:v1.29.0 | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: "3.11.3" | |
| - name: Add Helm repos | |
| run: | | |
| helm repo add vertica-charts https://vertica.github.io/charts || true | |
| helm repo add bitnami https://charts.bitnami.com/bitnami || true | |
| helm repo update | |
| - name: Install MinIO (namespace minio) | |
| run: | | |
| kubectl create ns minio || true | |
| cat <<'EOF' > minio.yaml | |
| apiVersion: apps/v1 | |
| kind: Deployment | |
| metadata: | |
| name: minio | |
| namespace: minio | |
| spec: | |
| replicas: 1 | |
| selector: | |
| matchLabels: | |
| app: minio | |
| template: | |
| metadata: | |
| labels: | |
| app: minio | |
| spec: | |
| containers: | |
| - name: minio | |
| image: minio/minio:RELEASE.2025-09-07T16-13-09Z-cpuv1 | |
| args: ["server", "/data"] | |
| env: | |
| - name: MINIO_ROOT_USER | |
| value: "minioadmin" | |
| - name: MINIO_ROOT_PASSWORD | |
| value: "minioadmin" | |
| ports: | |
| - containerPort: 9000 | |
| volumeMounts: | |
| - name: data | |
| mountPath: /data | |
| volumes: | |
| - name: data | |
| emptyDir: {} | |
| --- | |
| apiVersion: v1 | |
| kind: Service | |
| metadata: | |
| name: minio | |
| namespace: minio | |
| spec: | |
| selector: | |
| app: minio | |
| ports: | |
| - port: 9000 | |
| targetPort: 9000 | |
| EOF | |
| kubectl apply -f minio.yaml | |
| kubectl -n minio rollout status deployment/minio --timeout=2m || true | |
| kubectl get pods -n minio -o wide || true | |
| kubectl get svc -n minio || true | |
| - name: Ensure MinIO bucket exists | |
| run: | | |
| kubectl run mc-client --rm -i --restart=Never \ | |
| --image=minio/mc:latest \ | |
| -n minio \ | |
| --command -- bash -c " | |
| mc alias set localminio http://minio.minio.svc.cluster.local:9000 minioadmin minioadmin && \ | |
| mc mb --ignore-existing localminio/vertica-fleeting && \ | |
| mc ls localminio | |
| " | |
| - name: Create MinIO Secret | |
| run: | | |
| kubectl create ns my-verticadb-operator || true | |
| kubectl delete secret communal-creds -n my-verticadb-operator --ignore-not-found | |
| kubectl create secret generic communal-creds \ | |
| -n my-verticadb-operator \ | |
| --from-literal=accesskey="minioadmin" \ | |
| --from-literal=secretkey="minioadmin" | |
| kubectl get secret communal-creds -n my-verticadb-operator -o yaml || true | |
| - name: Install Vertica Operator | |
| run: | | |
| cat <<'EOF' > operator-values.yaml | |
| installCRDs: true | |
| controller: | |
| extraEnv: | |
| - name: AWS_REGION | |
| value: "us-east-1" | |
| - name: AWS_DEFAULT_REGION | |
| value: "us-east-1" | |
| EOF | |
| helm upgrade --install vdb-op vertica-charts/verticadb-operator \ | |
| -n my-verticadb-operator -f operator-values.yaml --wait --timeout 10m | |
| kubectl -n my-verticadb-operator get pods -o wide || true | |
| - name: Deploy VerticaDB | |
| run: | | |
| cat <<'EOF' | kubectl apply -f - | |
| apiVersion: vertica.com/v1 | |
| kind: VerticaDB | |
| metadata: | |
| name: verticadb-sample | |
| namespace: my-verticadb-operator | |
| spec: | |
| image: opentext/vertica-k8s:latest | |
| dbName: vdb | |
| initPolicy: Create | |
| communal: | |
| path: s3://vertica-fleeting/mkottakota/ | |
| credentialSecret: communal-creds | |
| endpoint: http://minio.minio.svc.cluster.local:9000 | |
| region: us-east-1 | |
| local: | |
| dataPath: /data | |
| depotPath: /depot | |
| subclusters: | |
| - name: defaultsubcluster | |
| size: 3 | |
| EOF | |
| kubectl annotate verticadb verticadb-sample -n my-verticadb-operator \ | |
| vertica.com/ci-reconcile="$(date -u +%s)" --overwrite || true | |
| - name: Wait for Vertica readiness | |
| run: | | |
| NS=my-verticadb-operator | |
| SS=verticadb-sample-defaultsubcluster | |
| POD=${SS}-0 | |
| for i in {1..30}; do | |
| kubectl get pod ${POD} -n ${NS} && break || sleep 10 | |
| done | |
| kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m || true | |
| - name: Deploy Keycloak | |
| run: | | |
| kubectl create ns keycloak || true | |
| cat <<'EOF' | kubectl apply -f - | |
| apiVersion: apps/v1 | |
| kind: Deployment | |
| metadata: | |
| name: keycloak | |
| namespace: keycloak | |
| spec: | |
| replicas: 1 | |
| selector: | |
| matchLabels: | |
| app: keycloak | |
| template: | |
| metadata: | |
| labels: | |
| app: keycloak | |
| spec: | |
| containers: | |
| - name: keycloak | |
| image: quay.io/keycloak/keycloak:23.0.4 | |
| args: ["start-dev"] | |
| env: | |
| - name: KEYCLOAK_ADMIN | |
| value: admin | |
| - name: KEYCLOAK_ADMIN_PASSWORD | |
| value: admin | |
| ports: | |
| - containerPort: 8080 | |
| readinessProbe: | |
| httpGet: | |
| path: / | |
| port: 8080 | |
| initialDelaySeconds: 20 | |
| periodSeconds: 5 | |
| failureThreshold: 6 | |
| --- | |
| apiVersion: v1 | |
| kind: Service | |
| metadata: | |
| name: keycloak | |
| namespace: keycloak | |
| spec: | |
| selector: | |
| app: keycloak | |
| ports: | |
| - port: 8080 | |
| targetPort: 8080 | |
| EOF | |
| - name: Wait for Keycloak readiness | |
| run: | | |
| kubectl -n keycloak rollout status deploy/keycloak --timeout=2m | |
| kubectl -n keycloak get pods -o wide | |
| - name: Configure Keycloak realm, client, and user | |
| run: | | |
| kubectl -n keycloak exec deploy/keycloak -- \ | |
| /opt/keycloak/bin/kcadm.sh config credentials \ | |
| --server http://localhost:8080 --realm master \ | |
| --user admin --password admin | |
| kubectl -n keycloak exec deploy/keycloak -- \ | |
| /opt/keycloak/bin/kcadm.sh create realms -s realm=test -s enabled=true | |
| kubectl -n keycloak exec deploy/keycloak -- \ | |
| /opt/keycloak/bin/kcadm.sh create clients -r test \ | |
| -s clientId=vertica -s enabled=true \ | |
| -s secret=P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs \ | |
| -s 'redirectUris=["*"]' \ | |
| -s directAccessGrantsEnabled=true | |
| kubectl -n keycloak exec deploy/keycloak -- \ | |
| /opt/keycloak/bin/kcadm.sh create users -r test \ | |
| -s username=oauth_user -s enabled=true | |
| kubectl -n keycloak exec deploy/keycloak -- \ | |
| /opt/keycloak/bin/kcadm.sh set-password -r test \ | |
| --username oauth_user --new-password password | |
| - name: Configure Vertica Authentication | |
| run: | | |
| NS=my-verticadb-operator | |
| POD=verticadb-sample-defaultsubcluster-0 | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -c \" | |
| CREATE AUTHENTICATION v_oauth METHOD 'oauth' HOST '0.0.0.0/0'; | |
| ALTER AUTHENTICATION v_oauth SET client_id = 'vertica'; | |
| ALTER AUTHENTICATION v_oauth SET client_secret = 'P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs'; | |
| ALTER AUTHENTICATION v_oauth SET discovery_url = 'http://keycloak.keycloak.svc.cluster.local:8080/realms/test/.well-known/openid-configuration'; | |
| ALTER AUTHENTICATION v_oauth SET introspect_url = 'http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect'; | |
| CREATE USER oauth_user; | |
| GRANT AUTHENTICATION v_oauth TO oauth_user; | |
| GRANT ALL ON SCHEMA PUBLIC TO oauth_user; | |
| CREATE AUTHENTICATION v_dbadmin_hash METHOD 'hash' HOST '0.0.0.0/0'; | |
| ALTER AUTHENTICATION v_dbadmin_hash PRIORITY 10000; | |
| GRANT AUTHENTICATION v_dbadmin_hash TO dbadmin; | |
| \" | |
| " | |
| - name: Configure Vertica Authentication | |
| run: | | |
| NS=my-verticadb-operator | |
| POD=verticadb-sample-defaultsubcluster-0 | |
| echo "🔧 Configuring Vertica load balancing policy..." | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -lc " | |
| /opt/vertica/bin/vsql -U dbadmin -c \" | |
| SELECT set_load_balance_policy('ROUNDROBIN'); | |
| \" | |
| " | |
| - name: Configure Vertica Load Balancing | |
| run: | | |
| NS=my-verticadb-operator | |
| POD=verticadb-sample-defaultsubcluster-0 | |
| echo "🔧 Setting load balance policy to ROUNDROBIN..." | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -c \" | |
| SELECT set_load_balance_policy('ROUNDROBIN'); | |
| \" | |
| " | |
| echo "🔍 Verifying load balance policy..." | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -c \" | |
| SELECT get_load_balance_policy(); | |
| \" | |
| " | |
| - name: Test load balancing is working | |
| run: | | |
| NS=my-verticadb-operator | |
| POD=verticadb-sample-defaultsubcluster-0 | |
| SVC=verticadb-sample-defaultsubcluster | |
| echo "📊 Testing load balancing distribution..." | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -c \" | |
| DROP TABLE IF EXISTS lb_test_verify CASCADE; | |
| CREATE TABLE lb_test_verify (node_name VARCHAR(256)); | |
| \" | |
| " | |
| # Make 9 test connections | |
| for i in {1..9}; do | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -h ${SVC}.${NS}.svc.cluster.local -c \" | |
| INSERT INTO lb_test_verify SELECT node_name FROM current_session; | |
| \" | |
| " >/dev/null 2>&1 | |
| done | |
| echo "📈 Connection distribution:" | |
| kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -c \" | |
| SELECT node_name, COUNT(*) as connections | |
| FROM lb_test_verify | |
| GROUP BY node_name | |
| ORDER BY node_name; | |
| \" | |
| " | |
| UNIQUE_NODES=$(kubectl -n ${NS} exec ${POD} -c server -- bash -c " | |
| /opt/vertica/bin/vsql -U dbadmin -At -c \" | |
| SELECT COUNT(DISTINCT node_name) FROM lb_test_verify; | |
| \" | |
| ") | |
| echo "Unique nodes used: ${UNIQUE_NODES}/3" | |
| if [ "${UNIQUE_NODES}" != "3" ]; then | |
| echo "⚠️ Warning: Load balancing may not be working correctly" | |
| echo "Expected 3 nodes, got ${UNIQUE_NODES}" | |
| else | |
| echo "✅ Load balancing verified: all 3 nodes in use" | |
| fi | |
| - name: Run Python tests in-cluster | |
| run: | | |
| set -euo pipefail | |
| NS=my-verticadb-operator | |
| SVC=verticadb-sample-defaultsubcluster | |
| POD=py-test-runner | |
| IMAGE=python:${{ matrix.python-version }}-slim | |
| echo "⏳ Waiting for Vertica service endpoints..." | |
| WAIT_TIMEOUT=300 | |
| deadline=$((SECONDS + WAIT_TIMEOUT)) | |
| while [ $SECONDS -lt $deadline ]; do | |
| addrs=$(kubectl -n ${NS} get endpoints ${SVC} -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true) | |
| [ -n "$addrs" ] && break || sleep 5 | |
| done | |
| if [ -z "$addrs" ]; then | |
| echo "❌ Vertica service endpoints not found" | |
| kubectl -n ${NS} get pods -o wide | |
| kubectl -n ${NS} get endpoints ${SVC} -o yaml | |
| exit 1 | |
| fi | |
| echo "✅ Service endpoints ready: $addrs" | |
| echo "🚀 Creating Python test pod..." | |
| kubectl -n ${NS} delete pod ${POD} --ignore-not-found --wait || true | |
| kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity | |
| kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=3m | |
| echo "📂 Copying repository into pod..." | |
| kubectl -n ${NS} exec -i pod/${POD} -- mkdir -p /workspace | |
| tar cf - . | kubectl -n ${NS} exec -i pod/${POD} -- tar xf - -C /workspace | |
| echo "🧰 Installing dependencies..." | |
| kubectl -n ${NS} exec pod/${POD} -- bash -c ' | |
| apt-get update -qq && \ | |
| apt-get install -y -qq build-essential libssl-dev libpq-dev netcat-traditional curl | |
| ' | |
| kubectl -n ${NS} exec pod/${POD} -- bash -c ' | |
| python -m pip install --upgrade pip >/dev/null 2>&1 && \ | |
| pip install tox pytest >/dev/null 2>&1 | |
| ' | |
| echo "🔑 Fetching OAuth token from Keycloak..." | |
| CT_POD="curl-token-$$" | |
| kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found --wait || true | |
| kubectl -n keycloak run ${CT_POD} --restart=Never \ | |
| --image=curlimages/curl:latest --command -- sleep 180 | |
| kubectl -n keycloak wait --for=condition=Ready pod/${CT_POD} --timeout=2m | |
| kubectl -n keycloak exec pod/${CT_POD} -- sh -c " | |
| curl -s -X POST 'http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token' \ | |
| -d 'client_id=${CLIENT_ID}' \ | |
| -d 'username=${USER}' \ | |
| -d 'password=${PASSWORD}' \ | |
| -d 'grant_type=password' \ | |
| -d 'client_secret=${CLIENT_SECRET}' > /tmp/token.json | |
| " | |
| kubectl -n keycloak cp ${CT_POD}:/tmp/token.json token.json || { | |
| echo "❌ Failed to copy token.json" | |
| kubectl -n keycloak logs ${CT_POD} | |
| exit 1 | |
| } | |
| kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found || true | |
| TOKEN=$(python3 -c 'import json; print(json.load(open("token.json")).get("access_token",""))') | |
| if [ -z "$TOKEN" ]; then | |
| echo "❌ No access_token found" | |
| cat token.json | |
| exit 1 | |
| fi | |
| echo "✅ Access token retrieved (length: ${#TOKEN})" | |
| echo "🏃 Running Python tests..." | |
| kubectl -n ${NS} exec -i pod/${POD} -- bash -c " | |
| set -euo pipefail | |
| cd /workspace | |
| export VP_TEST_OAUTH_ACCESS_TOKEN='${TOKEN}' | |
| export VP_TEST_HOST='${SVC}.${NS}.svc.cluster.local' | |
| export VP_TEST_PORT=5433 | |
| export VP_TEST_DATABASE='vdb' | |
| export VP_TEST_OAUTH_USER='oauth_user' | |
| export VP_TEST_USER='dbadmin' | |
| export VP_TEST_PASSWORD='' | |
| echo '🔍 Testing Vertica connectivity...' | |
| nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { | |
| echo '❌ Cannot reach Vertica' | |
| exit 1 | |
| } | |
| echo '🔍 Verifying OAuth token...' | |
| INTROSPECT=\$(curl -s -X POST \ | |
| http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect \ | |
| -d 'client_id=vertica' \ | |
| -d 'client_secret=P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs' \ | |
| -d \"token=\${VP_TEST_OAUTH_ACCESS_TOKEN}\") | |
| if echo \"\$INTROSPECT\" | grep -q '\"active\":true'; then | |
| echo '✅ Token is valid' | |
| else | |
| echo '❌ Token validation failed' | |
| echo \"\$INTROSPECT\" | |
| exit 1 | |
| fi | |
| echo '🧪 Running test suite...' | |
| tox -e py | |
| " | |
| echo "🧹 Cleaning up test pod..." | |
| kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true | |
| - name: Uninstall MinIO | |
| if: always() | |
| run: | | |
| kubectl delete pod minio -n minio --ignore-not-found || true | |
| kubectl delete svc minio -n minio --ignore-not-found || true | |
| kubectl delete ns minio || true | |
| echo "MinIO cleanup complete" |