diff --git a/.gitignore b/.gitignore index 34ad909..103a11f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,17 @@ logs pgdata projects .env +.idea +*.iml + +# macOS +.DS_Store +**/.DS_Store + +# Claude Code - Local only (don't track) +*.local.md +.claude/settings.local.json +.claude/scripts/ +.vscode +**/**/CLAUDE.md +tpch001 diff --git a/Dockerfile_airflow b/Dockerfile_airflow index 831aaf6..f1f52c0 100644 --- a/Dockerfile_airflow +++ b/Dockerfile_airflow @@ -39,10 +39,18 @@ USER airflow # Install aws cli RUN pip install --no-cache-dir -U awscli -# Configure aws cli -RUN mkdir -p /home/airflow/.aws -COPY conf/aws/credentials /home/airflow/.aws/credentials -COPY conf/aws/config /home/airflow/.aws/config +# NOTE: AWS credentials should NOT be baked into the image. +# For Kubernetes: Use IRSA (EKS), Workload Identity (GKE), or mount credentials via K8s Secret. +# For Docker Compose: Mount ~/.aws as a volume or use environment variables. +# Example K8s Secret mount: +# volumes: +# - name: aws-credentials +# secret: +# secretName: aws-credentials +# volumeMounts: +# - name: aws-credentials +# mountPath: /home/airflow/.aws +# readOnly: true # Install airflow amazon and google providers RUN pip install --no-cache-dir \ diff --git a/Dockerfile_airflow3 b/Dockerfile_airflow3 index 1814a2a..b985a67 100644 --- a/Dockerfile_airflow3 +++ b/Dockerfile_airflow3 @@ -41,10 +41,18 @@ USER airflow # Install aws cli RUN pip install --no-cache-dir -U awscli -# Configure aws cli -RUN mkdir -p /home/airflow/.aws -COPY conf/aws/credentials /home/airflow/.aws/credentials -COPY conf/aws/config /home/airflow/.aws/config +# NOTE: AWS credentials should NOT be baked into the image. +# For Kubernetes: Use IRSA (EKS), Workload Identity (GKE), or mount credentials via K8s Secret. +# For Docker Compose: Mount ~/.aws as a volume or use environment variables. +# Example K8s Secret mount: +# volumes: +# - name: aws-credentials +# secret: +# secretName: aws-credentials +# volumeMounts: +# - name: aws-credentials +# mountPath: /home/airflow/.aws +# readOnly: true # Install airflow amazon, google and fab providers RUN pip install --no-cache-dir \ diff --git a/Dockerfile_airflow_k8s b/Dockerfile_airflow_k8s new file mode 100644 index 0000000..f31b4f8 --- /dev/null +++ b/Dockerfile_airflow_k8s @@ -0,0 +1,102 @@ +# Dockerfile for Airflow on Kubernetes with K8s Job execution +# Use this instead of Dockerfile_airflow for Kubernetes deployments +# +# This image creates K8s Jobs for each starlake command, offloading heavy +# processing from the Airflow pod. Requires: +# - ServiceAccount with RBAC permissions to create/manage Jobs +# - Job template ConfigMap mounted at /etc/starlake/job-template.yaml +# - automountServiceAccountToken: true + +# Stage 1: Copy Starlake CLI from UI image +FROM starlakeai/starlake-1.5-ui:1.5 AS starlake-cli + +# Stage 2: Build Airflow image with Starlake CLI +FROM apache/airflow:2.11.0 + +# Switch to root user to install additional packages +USER root + +# Install NFS client utilities and kubectl for K8s Job creation +# kubectl is downloaded with SHA256 checksum verification for supply chain security +RUN apt-get update \ + && apt-get install -y nfs-common \ + mandoc \ + less \ + curl \ + && KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) \ + && ARCH=$(dpkg --print-architecture || echo "amd64") \ + && curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" \ + && curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl.sha256" \ + && echo "$(cat kubectl.sha256) kubectl" | sha256sum --check \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/kubectl \ + && rm kubectl.sha256 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ADD conf/airflow/webserver_config.py /opt/airflow/webserver_config.py + +# Required to mount NFS volumes +RUN echo "airflow ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow + +# Copy the actual Starlake CLI (Java) from UI image +COPY --from=starlake-cli /app/starlake /app/starlake + +# Install SL CLI wrapper for Kubernetes Job execution +# This wrapper creates K8s Jobs to offload starlake commands from the Airflow pod +# IMPORTANT: We rename the original starlake to starlake-original and replace it with our wrapper +# because Airflow DAGs call /app/starlake/starlake directly (not /usr/local/bin/starlake) +RUN mv /app/starlake/starlake /app/starlake/starlake-original 2>/dev/null || true +COPY scripts/kubernetes/starlake.sh /app/starlake/starlake +RUN chmod +x /app/starlake/starlake \ + && chmod +x /app/starlake/starlake.sh \ + && chmod +x /app/starlake/starlake-original 2>/dev/null || true \ + && ln -sf /app/starlake/starlake /usr/local/bin/starlake \ + && ln -sf /app/starlake/starlake /usr/local/bin/starlake-k8s + +# Copy Java runtime from UI image (required for starlake CLI) +COPY --from=starlake-cli /opt/java /opt/java +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# Make JAVA_HOME available in all subshells (for Airflow BashOperator/subprocess) +RUN echo "export JAVA_HOME=/opt/java/openjdk" >> /etc/bash.bashrc \ + && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/bash.bashrc \ + && echo "export JAVA_HOME=/opt/java/openjdk" >> /etc/profile.d/java.sh \ + && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile.d/java.sh \ + && chmod +x /etc/profile.d/java.sh + +# Install gcloud sdk +RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz \ + && mkdir -p /usr/local/gcloud \ + && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ + && ln -s /usr/local/gcloud/google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud \ + && rm /tmp/google-cloud-sdk.tar.gz + +# Switch back to the airflow user +USER airflow + +# Install aws cli +RUN pip install --no-cache-dir -U awscli + +# NOTE: AWS credentials should NOT be baked into the image. +# For Kubernetes: Use IRSA (EKS), Workload Identity (GKE), or mount credentials via K8s Secret. +# Example K8s Secret mount in Helm values.yaml: +# airflow: +# extraVolumes: +# - name: aws-credentials +# secret: +# secretName: aws-credentials +# extraVolumeMounts: +# - name: aws-credentials +# mountPath: /home/airflow/.aws +# readOnly: true + +# Install airflow amazon and google providers +RUN pip install --no-cache-dir \ + apache-airflow-providers-amazon \ + apache-airflow-providers-google + +# Install SL Python libraries for Airflow 2 (no docker package needed in K8s) +# IMPORTANT: Pin starlake-airflow~=0.4 for Airflow 2 compatibility (0.5+ requires Airflow 3) +RUN pip install --no-cache-dir "starlake-airflow>=0.4,<0.5" \ No newline at end of file diff --git a/Dockerfile_dagster b/Dockerfile_dagster index 4399f6c..d44ad6c 100644 --- a/Dockerfile_dagster +++ b/Dockerfile_dagster @@ -1,4 +1,5 @@ -FROM debian:11-slim +# Security fix: Debian 12 (bookworm) for latest security patches +FROM debian:12-slim # Add Docker's official GPG key: RUN apt update \ diff --git a/Dockerfile_projects b/Dockerfile_projects index e42e8e6..5577801 100644 --- a/Dockerfile_projects +++ b/Dockerfile_projects @@ -1,4 +1,5 @@ -FROM alpine:latest +# Security fix: Pin Alpine version for reproducible builds and security tracking +FROM alpine:3.21 RUN apk add --no-cache --no-progress \ nfs-utils \ diff --git a/README.md b/README.md index e960139..5ecf1ac 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Starlake uses Docker Compose **profiles** to manage different configurations (e. To start the Pragmatic Duck Data Stack with Airflow and Gizmo on local file system, use the following command: ```bash -COMPOSE_PROFILES=airflow,gizmo SL_API_APP_TYPE=ducklake docker compose up --build +COMPOSE_PROFILES=airflow,minio,gizmo SL_API_APP_TYPE=ducklake docker compose up --build ``` or simply @@ -100,12 +100,6 @@ or simply $ ./dags-stack.sh ``` -To start the Pragmatic Duck Data Stack with Airflow & Minio and Gizmo, use the following command: - -```bash -COMPOSE_PROFILES=airflow,minio,gizmo SL_API_APP_TYPE=ducklake docker compose up --build -``` - To start the stack with a specific profile (e.g., `airflow`) and address any Cloud Datawarehouses, use the following commands: ```bash @@ -163,43 +157,6 @@ Once up, the services are accessible at the following default URLs: > If you are affected by this [Docker issue](https://github.com/docker/for-mac/issues/7583), please upgrade your Docker install. -## Mounting external projects - -If you have any starlake container projects and want to mount it: - -- run `setup_mac_nfs.sh` if you are on mac in order to expose your folder via NFS. - Modify the root folder to share if necessary. By default it is set to /user. - This change is not specific to starlake and may be used in other container. -- comment `- external_projects_data:/external_projects` in the `volumes` section of the starlake-nas container -- uncomment `- starlake-prj-nfs-mount:/external_projects` right below the line above in the docker compose file -- go to the end of the file and comment uncomment the `starlake-prj-nfs-mount:` section as follows: - -``` - starlake-prj-nfs-mount: - driver: local - driver_opts: - type: nfs - o: addr=host.docker.internal,rw,nolock,hard,nointr,nfsvers=3 - device: ":/path_to_starlake_project_container" # absolute path to folder on your host where projects are located. -``` - -Starlake container folder should contain the starlake project folder: - -``` - /path_to_starlake_project_container - | - - my_first_starlake_project - | - - metadata - - ... - | - - my_second_starlake_project - | - - metadata - - ... -``` - -If you have many container projects, create as many volume as needed. ## Stopping Starlake UI @@ -207,4 +164,4 @@ To stop Starlake UI, run the following command in the same directory ```bash docker compose down -``` +``` \ No newline at end of file diff --git a/cloud-data-stack.sh b/cloud-data-stack.sh index 124d172..92d8fce 100755 --- a/cloud-data-stack.sh +++ b/cloud-data-stack.sh @@ -1 +1,4 @@ -docker compose --profile airflow up --build \ No newline at end of file +#!/usr/bin/env bash +set -euo pipefail + +docker compose --profile airflow up --build \ No newline at end of file diff --git a/dags-stack.sh b/dags-stack.sh index 9a9fdf9..9f93d55 100755 --- a/dags-stack.sh +++ b/dags-stack.sh @@ -1 +1,4 @@ -COMPOSE_PROFILES=airflow,gizmo SL_API_APP_TYPE=ducklake docker compose up --build +#!/usr/bin/env bash +set -euo pipefail + +COMPOSE_PROFILES=airflow,gizmo SL_API_APP_TYPE=ducklake docker compose up --build diff --git a/docker-compose-dagster.yml b/docker-compose-dagster.yml index b449de2..dc043ec 100644 --- a/docker-compose-dagster.yml +++ b/docker-compose-dagster.yml @@ -4,7 +4,8 @@ version: '3.8' services: starlake-db: - image: postgres:17 + # Security fix: Pin PostgreSQL to patch version for security tracking + image: postgres:17.2 restart: on-failure container_name: starlake-db ports: @@ -172,4 +173,4 @@ services: - SL_ASK_API_DOCS_PASSWORD=s3cret.Paw volumes: - pgdata: + pgdata: \ No newline at end of file diff --git a/docker-compose-snowflake.yml b/docker-compose-snowflake.yml index 6f173ba..4d912ef 100644 --- a/docker-compose-snowflake.yml +++ b/docker-compose-snowflake.yml @@ -4,7 +4,8 @@ version: '3.8' services: starlake-db: - image: postgres:17 + # Security fix: Pin PostgreSQL to patch version for security tracking + image: postgres:17.2 restart: on-failure container_name: starlake-db ports: @@ -137,4 +138,4 @@ services: - SL_ASK_API_DOCS_PASSWORD=s3cret.Paw volumes: - pgdata: + pgdata: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 32d2b33..496b087 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,6 @@ x-starlake-airflow: - ${PROJECTS_DATA_PATH:-./projects}:/projects - ${PROJECTS_DATA_PATH:-./projects}/dags:/opt/airflow/dags - ${AIRFLOW_LOGS:-./airflow/logs}:/opt/airflow/logs - - /Users/hayssams/.config:/home/airflow/.config x-starlake-ui-common: &starlake-ui-common @@ -51,7 +50,7 @@ x-starlake-ui-common: interval: "5s" retries: 60 ports: - - ${SL_PORT:-80}:9900 # starlake-ui default port + - ${SL_PORT:-80}:9900 # starlake-ui default port environment: &starlake-ui-common-env SL_API_GIZMO_ON_DEMAND_URL: http://starlake-gizmo:10900 @@ -135,7 +134,8 @@ services: - airflow3 - dagster - snowflake - image: postgres:17 + # Security fix: Pin PostgreSQL to patch version for security tracking + image: postgres:17.2 restart: on-failure container_name: starlake-db ports: @@ -409,7 +409,8 @@ services: minio: profiles: - minio - image: "quay.io/minio/minio:latest" + # Security fix: Chainguard MinIO - official quay.io/minio/minio is unmaintained since Oct 2025 + image: "cgr.dev/chainguard/minio:latest" ports: - "${SL_MINIO_PORT:-9000}:9000" - "${SL_MINIO_CONSOLE_PORT:-9001}:9001" diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 0000000..cb5a9a5 --- /dev/null +++ b/helm/README.md @@ -0,0 +1,36 @@ +# Starlake Helm Chart + +This directory contains the official Helm chart for deploying the Starlake Data Stack on Kubernetes. + +## Documentation + +- **[Chart Documentation](starlake/README.md)** - Complete chart documentation (installation, configuration, parameters) +- **[Quick Start Guide](docs/QUICKSTART.md)** - Step-by-step deployment guides +- **[Local Testing](docs/LOCAL_TESTING.md)** - Test locally with K3s/K3d + +## Quick Start + +```bash +# Automated test script (creates K3s cluster, installs chart, validates) +./test-helm-chart.sh + +# Manual installation +helm install starlake ./starlake \ + --namespace starlake \ + --create-namespace +``` + +## Directory Structure + +``` +helm/ +├── starlake/ # The Helm chart +│ ├── Chart.yaml # Chart metadata +│ ├── values.yaml # Default configuration +│ ├── templates/ # Kubernetes templates +│ └── README.md # Chart documentation +├── docs/ # Additional documentation +└── test-helm-chart.sh # Automated test script +``` + +For full documentation, see [starlake/README.md](starlake/README.md). diff --git a/helm/docs/LOCAL_TESTING.md b/helm/docs/LOCAL_TESTING.md new file mode 100644 index 0000000..197065f --- /dev/null +++ b/helm/docs/LOCAL_TESTING.md @@ -0,0 +1,609 @@ +# Tests Locaux du Helm Chart avec K3s + +Ce guide explique comment tester le Helm chart Starlake localement avec K3s (Kubernetes léger). + +## Pourquoi K3s ? + +- ✅ **Léger** : 50 Mo vs 500+ Mo pour Minikube +- ✅ **Rapide** : Démarre en quelques secondes +- ✅ **Complet** : Support Ingress, LoadBalancer (via Traefik), storage local +- ✅ **Production-like** : Architecture identique à un vrai cluster +- ✅ **Multi-plateforme** : macOS, Linux, Windows (WSL2) + +## Installation de K3s + +### macOS / Linux + +```bash +# Installer k3s via k3d (K3s in Docker - plus simple sur macOS) +brew install k3d + +# Ou télécharger directement +# curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash +``` + +### Windows (WSL2) + +```bash +curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash +``` + +### Vérifier l'installation + +```bash +k3d version +# Exemple de sortie: k3d version v5.6.0 +``` + +## Création du Cluster de Test + +### Option 1 : Cluster Basique + +```bash +# Créer un cluster K3s simple +k3d cluster create starlake-test \ + --agents 2 \ + --port "8080:80@loadbalancer" \ + --port "8443:443@loadbalancer" + +# Vérifier que kubectl est configuré +kubectl cluster-info +kubectl get nodes +``` + +### Option 2 : Cluster avec Configuration Avancée + +```bash +# Créer un cluster avec plus de ressources +k3d cluster create starlake-test \ + --agents 3 \ + --servers 1 \ + --port "8080:80@loadbalancer" \ + --port "8443:443@loadbalancer" \ + --k3s-arg "--disable=traefik@server:0" \ + --volume "$(pwd)/projects:/projects@all" + +# Note: On désactive Traefik si on veut utiliser NGINX Ingress +``` + +## Multi-Node Clusters and local-path Storage Limitations + +When using K3d with multiple nodes and the default `local-path` storage class, there are important limitations to understand. + +### The Problem + +The `local-path` storage provisioner in K3s has the following behavior: + +1. **Node Affinity**: PersistentVolumes are created with node affinity - the volume is physically stored on one specific node +2. **First Consumer Binding**: The PV binds to whichever node first creates a pod that uses the PVC +3. **No Cross-Node Access**: Pods on other nodes cannot access the volume + +This creates issues in multi-node clusters: + +``` +Example scenario: +- PVC is created and bound to agent-0 (first pod scheduled there) +- Gizmo pod with hostNetwork needs to run on server-0 (where ports are mapped) +- Gizmo cannot start because the PVC is only accessible on agent-0 +``` + +### K3d Port Mapping and hostNetwork + +K3d port mapping (e.g., `--port "11900:11900@server:0"`) forwards traffic from the host to a specific node: + +- `@server:0` - Forward to the first server node +- `@loadbalancer` - Forward to the built-in load balancer (for HTTP/HTTPS) +- `@agent:0` - Forward to the first agent node + +When a service uses `hostNetwork: true` (like Gizmo for Arrow Flight SQL), the pod must run on the node where the port is mapped. But if the PVC is bound to a different node, there's a conflict. + +### Solutions + +#### Solution 1: Single-Node Cluster (Recommended for Development) + +The simplest solution is to use a single-node cluster where there's no node affinity conflict: + +```bash +k3d cluster create starlake-test \ + --servers 1 \ + --agents 0 \ + --port "8080:80@loadbalancer" \ + --port "11900-11920:11900-11920@server:0" +``` + +This is the recommended approach for local development and testing. + +#### Solution 2: Use port-forward for Gizmo + +In multi-node clusters, use `kubectl port-forward` instead of hostNetwork port mapping: + +```bash +# Start the cluster without Gizmo-specific port mappings +k3d cluster create starlake-test --servers 1 --agents 2 --port "8080:80@loadbalancer" + +# After deployment, port-forward to Gizmo +kubectl port-forward deploy/starlake-gizmo 11900:11900 -n starlake +``` + +#### Solution 3: RWX Storage (Production Approach) + +For production or production-like testing, use a storage class that supports ReadWriteMany: + +- **NFS Provisioner**: Works in any environment +- **AWS EFS**: For EKS clusters +- **GCP Filestore**: For GKE clusters +- **Azure Files**: For AKS clusters + +With RWX storage, any pod on any node can access the volume. + +### Gizmo Connection Details + +When using port-forward or hostNetwork, connect to Gizmo using: + +``` +JDBC URL: jdbc:arrow-flight-sql://localhost:11900?useEncryption=true&disableCertificateVerification=true +Username: gizmosql_user +Password: gizmosql_password +``` + +For DBeaver or other SQL clients: +1. Install the Arrow Flight SQL JDBC driver +2. Use the connection URL above +3. Enable SSL but disable certificate verification (for development) + +### Summary Table + +| Cluster Type | Storage | Gizmo Access Method | Complexity | +|-------------|---------|---------------------|------------| +| Single-node K3d | local-path | hostNetwork (direct) | Simple | +| Multi-node K3d | local-path | port-forward | Medium | +| Multi-node K3d | NFS | hostNetwork (direct) | Medium | +| Production (EKS/GKE/AKS) | EFS/Filestore/Azure Files | Ingress or LoadBalancer | Production-ready | + +## Installation des Prérequis + +### 1. Installer Helm (si pas déjà fait) + +```bash +# macOS +brew install helm + +# Linux +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + +# Vérifier +helm version +``` + +### 2. Installer un Storage Provisioner (pour ReadWriteMany) + +K3s inclut local-path-provisioner par défaut (ReadWriteOnce uniquement). +Pour ReadWriteMany, on installe NFS provisioner : + +```bash +# Ajouter le repo Helm +helm repo add nfs-subdir-external-provisioner \ + https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ + +# Pour K3s, on utilise un NFS server local +# Option A : Installer NFS server sur l'hôte (recommandé) + +# macOS (NFS est déjà intégré, juste besoin de le configurer) +sudo mkdir -p /System/Volumes/Data/nfs/starlake-projects +# Ajouter à /etc/exports: +echo "/System/Volumes/Data/nfs/starlake-projects -alldirs -mapall=$(id -u):$(id -g) localhost" | sudo tee -a /etc/exports +# Redémarrer NFS +sudo nfsd restart + +# Linux +sudo apt-get install nfs-kernel-server +sudo mkdir -p /srv/nfs/starlake-projects +sudo chown nobody:nogroup /srv/nfs/starlake-projects +echo "/srv/nfs/starlake-projects *(rw,sync,no_subtree_check,no_root_squash)" | sudo tee -a /etc/exports +sudo systemctl restart nfs-kernel-server + +# Option B : Utiliser le provisioner local de K3s avec un workaround +# (moins idéal mais fonctionne pour les tests) +kubectl apply -f - < /tmp/starlake-manifests.yaml + +# Vérifier le fichier généré +less /tmp/starlake-manifests.yaml +``` + +### Étape 2 : Déployer avec PostgreSQL Interne + +```bash +# Créer le namespace +kubectl create namespace starlake + +# Installer avec configuration de test +helm install starlake ./helm/starlake \ + --namespace starlake \ + --set postgresql.internal.persistence.size=2Gi \ + --set persistence.projects.size=5Gi \ + --set persistence.projects.storageClass=nfs-client \ + --set ui.resources.requests.memory=256Mi \ + --set ui.resources.limits.memory=1Gi \ + --set airflow.webserver.resources.requests.memory=256Mi \ + --set airflow.webserver.resources.limits.memory=1Gi \ + --set agent.resources.requests.memory=128Mi \ + --set agent.resources.limits.memory=512Mi + +# Alternative : Utiliser le fichier values-development.yaml +helm install starlake ./helm/starlake \ + --namespace starlake \ + --values ./helm/starlake/values-development.yaml \ + --set persistence.projects.storageClass=nfs-client +``` + +### Étape 3 : Suivre le Déploiement + +```bash +# Voir les pods en cours de création +kubectl get pods -n starlake -w + +# Voir les logs d'un pod spécifique +kubectl logs -n starlake -l app.kubernetes.io/component=postgresql -f + +# Voir tous les événements +kubectl get events -n starlake --sort-by='.lastTimestamp' + +# Vérifier les PVCs +kubectl get pvc -n starlake +``` + +### Étape 4 : Tester l'Accès + +#### Option A : Port-Forward (Simple et Recommandé) + +```bash +# Port-forward vers l'UI (point d'entrée principal) +# L'UI proxie automatiquement /airflow vers le service Airflow interne +kubectl port-forward svc/starlake-ui 8080:80 -n starlake + +# Ouvrir dans le navigateur +open http://localhost:8080 # UI Starlake +open http://localhost:8080/airflow # Airflow (via proxy UI) + +# Credentials Airflow par défaut : airflow / airflow +``` + +> **Note** : L'UI agit comme reverse proxy pour Airflow. Un seul port-forward suffit pour accéder aux deux services sur le même port. + +#### Option B : LoadBalancer (K3s inclut un LoadBalancer) + +```bash +# Obtenir l'IP externe (sera localhost ou 127.0.0.1) +kubectl get svc starlake-ui -n starlake + +# Accéder via le port mappé lors de la création du cluster +open http://localhost:8080 +``` + +### Étape 5 : Tests Fonctionnels + +```bash +# 1. Tester la connexion PostgreSQL +kubectl exec -it starlake-postgresql-0 -n starlake -- \ + psql -U dbuser -d starlake -c "SELECT version();" + +# 2. Vérifier les bases de données +kubectl exec -it starlake-postgresql-0 -n starlake -- \ + psql -U dbuser -c "\l" + +# 3. Tester la connectivité UI → PostgreSQL +kubectl exec -it deployment/starlake-ui -n starlake -- \ + nc -zv starlake-postgresql 5432 + +# 4. Vérifier les health checks +kubectl get pods -n starlake -o wide +kubectl describe pod starlake-ui-xxxxx -n starlake | grep -A 10 "Liveness\|Readiness" + +# 5. Tester les API (avec port-forward sur 8080) +# Health check UI +curl http://localhost:8080/api/v1/health + +# Health check Airflow (via proxy UI) +curl http://localhost:8080/airflow/health +``` + +### Étape 6 : Test avec PostgreSQL Externe (Simulation) + +```bash +# Créer un PostgreSQL externe dans le cluster (pour simuler RDS) +kubectl run postgres-external \ + --image=postgres:17 \ + --env="POSTGRES_PASSWORD=external123" \ + --env="POSTGRES_USER=externaluser" \ + --env="POSTGRES_DB=starlake" \ + -n starlake + +# Exposer comme service +kubectl expose pod postgres-external \ + --port=5432 \ + --name=postgres-external \ + -n starlake + +# Attendre que le pod soit prêt +kubectl wait --for=condition=ready pod/postgres-external -n starlake --timeout=60s + +# Réinstaller Starlake avec PostgreSQL externe +helm uninstall starlake -n starlake + +helm install starlake ./helm/starlake \ + --namespace starlake \ + --set postgresql.external.enabled=true \ + --set postgresql.external.host=postgres-external \ + --set postgresql.internal.enabled=false \ + --set postgresql.credentials.username=externaluser \ + --set postgresql.credentials.password=external123 \ + --set persistence.projects.storageClass=nfs-client +``` + +## Tests de Mise à Jour (Upgrade) + +```bash +# Modifier une valeur (ex: changer le nombre de replicas UI) +helm upgrade starlake ./helm/starlake \ + --namespace starlake \ + --reuse-values \ + --set ui.replicas=2 + +# Voir l'historique +helm history starlake -n starlake + +# Rollback si nécessaire +helm rollback starlake -n starlake +``` + +## Tests de Performance (Optionnel) + +```bash +# Stress test simple sur l'UI +kubectl run -it --rm load-test \ + --image=busybox \ + --restart=Never \ + -- sh -c 'while true; do wget -q -O- http://starlake-ui.starlake.svc/api/v1/health; done' + +# Voir l'utilisation des ressources +kubectl top pods -n starlake +kubectl top nodes +``` + +## Checklist de Tests + +- [ ] PostgreSQL démarre et est accessible +- [ ] UI démarre et se connecte à PostgreSQL +- [ ] Airflow démarre (webserver + scheduler) +- [ ] Agent démarre +- [ ] Health checks passent pour tous les pods +- [ ] PVC projects est créé avec ReadWriteMany +- [ ] Logs sont accessibles via `kubectl logs` +- [ ] Port-forward fonctionne +- [ ] LoadBalancer fonctionne (si configuré) +- [ ] Upgrade/rollback fonctionnent +- [ ] PostgreSQL externe fonctionne (test de simulation) + +## Nettoyage + +```bash +# Supprimer le release Helm +helm uninstall starlake -n starlake + +# Supprimer les PVCs (optionnel) +kubectl delete pvc -l app.kubernetes.io/instance=starlake -n starlake + +# Supprimer le namespace +kubectl delete namespace starlake + +# Supprimer le cluster K3s +k3d cluster delete starlake-test +``` + +## Dépannage + +### Pods en CrashLoopBackOff + +```bash +# Voir les logs du pod +kubectl logs -n starlake --previous + +# Décrire le pod pour voir les events +kubectl describe pod -n starlake +``` + +### PVC en Pending + +```bash +# Vérifier le PVC +kubectl describe pvc starlake-projects -n starlake + +# Vérifier les storage classes +kubectl get storageclass + +# Si NFS ne fonctionne pas, utiliser local-path pour les tests +helm upgrade starlake ./helm/starlake \ + --namespace starlake \ + --set persistence.projects.storageClass=local-path \ + --set persistence.projects.size=2Gi +``` + +### PostgreSQL ne démarre pas + +```bash +# Vérifier les logs +kubectl logs starlake-postgresql-0 -n starlake + +# Vérifier le PVC +kubectl get pvc -n starlake | grep postgresql + +# Supprimer et recréer +helm uninstall starlake -n starlake +kubectl delete pvc data-starlake-postgresql-0 -n starlake +helm install starlake ./helm/starlake --namespace starlake +``` + +## Automatisation des Tests + +Créer un script de test automatisé : + +```bash +#!/bin/bash +# test-helm-chart.sh + +set -e + +echo "🧪 Test du Helm Chart Starlake" + +# 1. Créer le cluster +echo "📦 Création du cluster K3s..." +k3d cluster create starlake-test --agents 2 --port "8080:80@loadbalancer" + +# 2. Installer le chart +echo "🚀 Installation du chart..." +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --values ./helm/starlake/values-development.yaml \ + --wait --timeout 10m + +# 3. Vérifier les pods +echo "✅ Vérification des pods..." +kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=starlake -n starlake --timeout=5m + +# 4. Tests fonctionnels +echo "🔍 Tests fonctionnels..." + +# Test PostgreSQL +kubectl exec starlake-postgresql-0 -n starlake -- psql -U dbuser -d starlake -c "SELECT 1" > /dev/null +echo " ✓ PostgreSQL OK" + +# Test UI health +kubectl port-forward svc/starlake-ui 8080:80 -n starlake & +sleep 5 +curl -f http://localhost:8080/api/v1/health > /dev/null +echo " ✓ UI Health OK" +kill %1 + +# 5. Nettoyage +echo "🧹 Nettoyage..." +helm uninstall starlake -n starlake +k3d cluster delete starlake-test + +echo "✅ Tous les tests ont réussi !" +``` + +Rendre le script exécutable : +```bash +chmod +x helm/test-helm-chart.sh +./helm/test-helm-chart.sh +``` + +## Intégration Continue (CI) + +Exemple de GitHub Actions workflow : + +```yaml +# .github/workflows/test-helm.yml +name: Test Helm Chart + +on: + pull_request: + paths: + - 'helm/**' + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Install Helm + uses: azure/setup-helm@v3 + + - name: Create K3s cluster + run: k3d cluster create test --agents 2 + + - name: Lint Helm chart + run: helm lint ./helm/starlake + + - name: Install chart + run: | + helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --values ./helm/starlake/values-development.yaml \ + --wait --timeout 10m + + - name: Test pods are running + run: | + kubectl wait --for=condition=ready pod \ + -l app.kubernetes.io/instance=starlake \ + -n starlake --timeout=5m + + - name: Cleanup + if: always() + run: k3d cluster delete test +``` + +## Prochaines Étapes + +Après validation locale avec K3s : + +1. **Tester sur un vrai cluster** (EKS, GKE, AKS) +2. **Configurer monitoring** (Prometheus, Grafana) +3. **Mettre en place CI/CD** (ArgoCD, Flux) +4. **Documenter les cas d'usage production** +5. **Publier le chart** (sur un Helm repository) diff --git a/helm/docs/QUICKSTART.md b/helm/docs/QUICKSTART.md new file mode 100644 index 0000000..f4fa206 --- /dev/null +++ b/helm/docs/QUICKSTART.md @@ -0,0 +1,481 @@ +# Guide de Démarrage Rapide - Helm Chart Starlake + +Ce guide vous aide à déployer rapidement Starlake sur Kubernetes. + +## 🎯 Choix du Scénario + +Choisissez le scénario qui correspond à votre situation : + +### 1️⃣ Tests Locaux (K3d) - 15 min +Pour tester rapidement Starlake sur votre machine. + +### 2️⃣ PostgreSQL Externe (AWS RDS, GCP CloudSQL, etc.) - 30 min +Pour utiliser une base de données managée existante. + +### 3️⃣ Déploiement Complet sur Cloud - 1h +Déploiement production-ready sur AWS, GCP ou Azure. + +--- + +## 1️⃣ Tests Locaux avec K3d + +Le projet utilise **K3d** (K3s in Docker) pour les tests locaux. Un script automatisé gère tout le cycle de test. + +### Prérequis +```bash +# Installer K3d +brew install k3d # macOS +# ou curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + +# Docker doit être installé et démarré +docker info +``` + +### Test Rapide (Recommandé) + +```bash +cd helm + +# Test développement (single-node, credentials par défaut) +./test-helm-chart.sh + +# Test production (credentials sécurisés) +./test-helm-chart.sh --production + +# Test multi-node avec SeaweedFS (S3 storage) +./test-helm-chart.sh --multi-node --seaweedfs + +# Validation sécurité uniquement (rapide, sans cluster) +./test-helm-chart.sh --security-only +``` + +Le script gère automatiquement : +- Création du cluster K3d avec ports mappés +- Build et import des images locales +- Déploiement Helm avec attente de readiness +- Port-forward pour accès local +- Cleanup à la fin + +### Accéder à Starlake + +Après `./test-helm-chart.sh`, les URLs sont affichées : +``` + UI: http://localhost:8080 + Airflow: http://localhost:8080/airflow + Gizmo: http://localhost:10900 + + Credentials Airflow: airflow / airflow +``` + +> **Note** : L'UI agit comme reverse proxy pour Airflow. Un seul port suffit pour accéder aux deux services. + +### Options du Script de Test + +| Option | Description | +|--------|-------------| +| `--production` | Credentials sécurisés, validation activée | +| `--multi-node` | Cluster 1 server + N agents | +| `--seaweedfs` | Stockage S3 (SeaweedFS) | +| `--security-only` | Validation sécurité uniquement | +| `--agents N` | Nombre d'agents (défaut: 3) | + +### Important : Cluster Multi-Noeud et local-path Storage + +Avec K3d multi-node, `local-path` storage crée des volumes liés à un nœud spécifique : + +- **Single-node cluster recommended**: For local testing, use `--servers 1 --agents 0` +- **Gizmo access in multi-node**: Use port-forward instead of hostNetwork: + ```bash + kubectl port-forward deploy/starlake-gizmo 11900:11900 -n starlake + ``` +- **Gizmo JDBC connection**: + ``` + jdbc:arrow-flight-sql://localhost:11900?useEncryption=true&disableCertificateVerification=true + User: gizmosql_user / Password: gizmosql_password + ``` + +For production environments, use RWX storage (EFS, Filestore, Azure Files, NFS). + +--- + +## 2️⃣ Avec PostgreSQL Externe (RDS, CloudSQL, etc.) + +### Prérequis + +1. **Base de données PostgreSQL** existante et accessible depuis le cluster +2. **Credentials** de connexion +3. **Storage class** supportant ReadWriteMany (EFS, Filestore, Azure Files) + +### Exemple avec AWS RDS + EFS + +#### Étape 1 : Préparer EFS + +```bash +# Installer EFS CSI Driver +kubectl apply -k "github.com/kubernetes-sigs/aws-efs-csi-driver/deploy/kubernetes/overlays/stable/?ref=master" + +# Créer un EFS file system (via AWS Console ou CLI) +# Note l'ID: fs-abc12345 + +# Créer le StorageClass +cat < k8s-starlake-xxxxx.us-east-1.elb.amazonaws.com +``` + +Accéder à : `https://starlake.mycompany.com` + +--- + +## 🔍 Vérifications Post-Installation + +### Vérifier l'état des pods + +```bash +kubectl get pods -n starlake + +# Tous les pods doivent être en Running +# Example output: +# NAME READY STATUS RESTARTS AGE +# starlake-postgresql-0 1/1 Running 0 5m +# starlake-ui-xxxxx 1/1 Running 0 4m +# starlake-airflow-xxxxx 1/1 Running 0 4m +# starlake-agent-xxxxx 1/1 Running 0 4m +``` + +### Vérifier les logs + +```bash +# UI +kubectl logs -n starlake -l app.kubernetes.io/component=ui -f + +# Airflow +kubectl logs -n starlake -l app.kubernetes.io/component=airflow -f + +# PostgreSQL (si interne) +kubectl logs -n starlake -l app.kubernetes.io/component=postgresql -f +``` + +### Tester la connexion PostgreSQL + +```bash +# Si PostgreSQL interne +kubectl exec -it starlake-postgresql-0 -n starlake -- \ + psql -U dbuser -d starlake -c "SELECT version();" + +# Lister les bases +kubectl exec -it starlake-postgresql-0 -n starlake -- \ + psql -U dbuser -c "\l" +``` + +--- + +## 🛠️ Dépannage Rapide + +### Pods en CrashLoopBackOff + +```bash +# Voir les logs du pod en erreur +kubectl logs -n starlake --previous + +# Décrire le pod pour voir les events +kubectl describe pod -n starlake +``` + +### PVC en Pending + +```bash +# Vérifier le PVC +kubectl describe pvc starlake-projects -n starlake + +# Vérifier si le storage class existe +kubectl get storageclass + +# Solutions: +# - Vérifier que le provisioner est installé +# - Vérifier que le storage class supporte ReadWriteMany +``` + +### Impossible de se connecter à PostgreSQL + +```bash +# Vérifier la connectivité réseau +kubectl exec -it deployment/starlake-ui -n starlake -- \ + nc -zv starlake-postgresql 5432 + +# Vérifier les secrets +kubectl get secret starlake-postgresql -n starlake -o yaml + +# Vérifier les variables d'environnement +kubectl exec deployment/starlake-ui -n starlake -- env | grep POSTGRES +``` + +--- + +## 📚 Prochaines Étapes + +1. **Configurer les projets Starlake** : Copier vos projets dans le PVC +2. **Paramétrer Airflow** : Configurer les connexions et variables +3. **Monitoring** : Installer Prometheus + Grafana +4. **Backups** : Configurer Velero pour les backups K8s +5. **CI/CD** : Intégrer avec ArgoCD ou FluxCD + +--- + +## 🆘 Aide + +- Documentation complète : [README.md](starlake/README.md) +- Issues GitHub : https://github.com/starlake-ai/starlake-data-stack/issues +- Slack : https://starlake.slack.com diff --git a/helm/docs/SEAWEEDFS_ISSUES.md b/helm/docs/SEAWEEDFS_ISSUES.md new file mode 100644 index 0000000..041eb53 --- /dev/null +++ b/helm/docs/SEAWEEDFS_ISSUES.md @@ -0,0 +1,203 @@ +# SeaweedFS Issues with Hadoop S3A + +Known issues discovered during Starlake Helm chart development when using SeaweedFS as the S3-compatible object storage backend with Hadoop S3A connector. + +## 1. 86-Byte Directory Marker Bug + +### Symptom + +Empty S3 directories appear as 86-byte files in Starlake UI file listings. For example, creating a new domain with subdirectories results in entries showing 86 bytes instead of 0 bytes. This causes incorrect file counts and confusing directory displays. + +### Root Cause + +When Hadoop S3A creates 0-byte directory markers via `PUT` over HTTP (not HTTPS), the AWS SDK 1.x uses chunked transfer encoding with AWS V4 signature. The chunk terminator has the format: + +``` +0;chunk-signature=<64-hex-chars>\r\n\r\n +``` + +This terminator is exactly **86 bytes** (`2 + 1 + 16 + 1 + 64 + 2 + 2 = 88` characters, but 86 bytes of meaningful payload). SeaweedFS stores this terminator as actual file content instead of recognizing and discarding it as a transfer encoding artifact. + +The obvious fix would be to disable chunked transfer encoding via `fs.s3a.payload.signing.enabled=false`. However, this property was only introduced in **Hadoop 3.3.5** ([HADOOP-17936](https://issues.apache.org/jira/browse/HADOOP-17936)). Starlake ships Hadoop 3.3.4 (`hadoop-aws-3.3.4.jar`), where this property does not exist. + +### Fix + +Force S3 V2 signing (the legacy signing algorithm), which does not use chunked transfer encoding at all: + +``` +fs.s3a.signing-algorithm=S3SignerType +``` + +This is safe for SeaweedFS since it supports both V2 and V4 signing. The V2 signer sends the full payload in a single request body, avoiding the chunked encoding problem entirely. + +### Configuration + +In `SL_STORAGE_CONF` (environment variable passed to Starlake API): + +``` +fs.s3a.signing-algorithm=S3SignerType,fs.s3a.path.style.access=true,fs.s3a.connection.ssl.enabled=false,... +``` + +In `core-site.xml` (used by Spark/Hadoop jobs): + +```xml + + fs.s3a.signing-algorithm + S3SignerType + +``` + +In `spark-defaults.conf`: + +``` +spark.hadoop.fs.s3a.signing-algorithm S3SignerType +``` + +### Why DuckLake Is Not Affected + +DuckLake (DuckDB) uses its own S3 client (`httpfs` extension), not Hadoop S3A. Key differences: + +| | Hadoop S3A | DuckLake (DuckDB httpfs) | +|---|---|---| +| Directory markers | Explicit 0-byte objects with trailing `/` | None -- directories are implicit S3 prefixes | +| HTTP client | AWS SDK 1.x with chunked V4 signing | Native HTTP with `Content-Length: 0` | +| Bug exposure | Yes (before fix) | Never | + +DuckLake writes parquet files directly (e.g., `test_s3/v2test/orders/ducklake-UUID.parquet`). The "directory" `orders/` is just a key prefix -- no `PUT` of a 0-byte marker object, no chunked encoding, no bug. + +### Verification + +After applying the fix, all newly created directory markers are 0 bytes. New domains show correct file counts in Starlake UI. Existing 86-byte markers from before the fix remain and must be cleaned up manually if needed (`aws s3 rm` and recreate). + +## 2. Filer 404 on S3-Created Directories + +### Symptom + +Parent directory listing in SeaweedFS Filer UI (`http://localhost:8888`) shows directories exist, but navigating into them returns HTTP 404. + +### Root Cause + +Known SeaweedFS bugs related to S3/Filer directory synchronization: + +- **[Issue #5193](https://github.com/seaweedfs/seaweedfs/issues/5193)**: Unable to list objects with prefix ending with `/`. The Filer does not always materialize implicit S3 directories as browsable Filer entries. +- **[Issue #6113](https://github.com/seaweedfs/seaweedfs/issues/6113)**: Directories disappear in Filer Store due to race condition (42% reproduction rate in reported tests). Concurrent Filer operations can delete parent directory entries even when children still exist. +- **[PR #7826](https://github.com/seaweedfs/seaweedfs/pull/7826)**: Changes to implicit directory handling for S3 client compatibility. This PR adjusts how directories created implicitly via S3 API are surfaced in the Filer. + +### Impact + +**Cosmetic only.** Starlake uses the S3 API for all data operations, which works correctly regardless of Filer state. The Filer UI is only an admin browsing tool. S3 API commands (`aws s3 ls`, `aws s3 cp`, etc.) always return correct results. + +### Workaround + +Use S3 API tools instead of the Filer web UI for directory inspection: + +```bash +# List bucket contents +aws s3 ls s3://starlake/ --endpoint-url http://localhost:8333 --recursive + +# List specific prefix +aws s3 ls s3://starlake/my-domain/ --endpoint-url http://localhost:8333 +``` + +### Status + +Unresolved upstream bugs. Not blocking for Starlake operations. + +## 3. Recommended S3A Configuration for SeaweedFS + +Full configuration with rationale for each property. All properties are set in both `core-site.xml` and `spark-defaults.conf` (with `spark.hadoop.` prefix) in the Helm chart. + +### Properties + +| Property | Value | Rationale | +|----------|-------|-----------| +| `fs.s3a.signing-algorithm` | `S3SignerType` | Use S3 V2 signing. Avoids V4 chunked encoding that produces 86-byte directory markers (see section 1). | +| `fs.s3a.path.style.access` | `true` | Required for non-AWS S3 backends. Uses `http://host/bucket/key` instead of `http://bucket.host/key`. | +| `fs.s3a.connection.ssl.enabled` | `false` | SeaweedFS uses HTTP internally in Kubernetes. No TLS termination at the S3 API level. | +| `fs.s3a.directory.marker.retention` | `keep` | Do not delete directory markers after file creation. SeaweedFS manages directories natively via its Filer layer. Deleting markers can cause empty directories to disappear. [SeaweedFS wiki recommendation](https://github.com/seaweedfs/seaweedfs/wiki/HDFS-via-S3-connector). | +| `fs.s3a.multiobjectdelete.enable` | `false` | Disable multi-object delete API. Behavior is unreliable on non-AWS S3 backends and can cause partial deletions. [SeaweedFS wiki recommendation](https://github.com/seaweedfs/seaweedfs/wiki/HDFS-via-S3-connector). | +| `fs.s3a.change.detection.mode` | `warn` | Log a warning instead of throwing an exception when file modification is detected during read. SeaweedFS does not support ETags the same way as AWS S3. | +| `fs.s3a.change.detection.version.required` | `false` | Do not require version IDs for change detection. SeaweedFS does not support S3 object versioning by default. | +| `fs.s3a.bucket.probe` | `0` | Skip `HEAD` bucket check on startup. Faster initialization; avoids 403/404 errors if credentials or bucket don't exist yet. | + +### Example core-site.xml + +```xml + + + + fs.s3a.endpoint + http://starlake-seaweedfs:8333 + + + fs.s3a.path.style.access + true + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.access.key + seaweedfs + + + fs.s3a.secret.key + seaweedfs123 + + + fs.s3a.signing-algorithm + S3SignerType + + + fs.s3a.bucket.probe + 0 + + + fs.s3a.directory.marker.retention + keep + + + fs.s3a.multiobjectdelete.enable + false + + + fs.s3a.change.detection.mode + warn + + + fs.s3a.change.detection.version.required + false + + +``` + +### Example spark-defaults.conf + +``` +spark.hadoop.fs.s3a.endpoint http://starlake-seaweedfs:8333 +spark.hadoop.fs.s3a.path.style.access true +spark.hadoop.fs.s3a.connection.ssl.enabled false +spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem +spark.hadoop.fs.s3a.access.key seaweedfs +spark.hadoop.fs.s3a.secret.key seaweedfs123 +spark.hadoop.fs.s3a.signing-algorithm S3SignerType +spark.hadoop.fs.s3a.bucket.probe 0 +spark.hadoop.fs.s3a.directory.marker.retention keep +spark.hadoop.fs.s3a.multiobjectdelete.enable false +spark.hadoop.fs.s3a.change.detection.mode warn +spark.hadoop.fs.s3a.change.detection.version.required false +``` + +## References + +- [SeaweedFS Wiki: HDFS via S3 connector](https://github.com/seaweedfs/seaweedfs/wiki/HDFS-via-S3-connector) -- official configuration guidance for Hadoop S3A with SeaweedFS +- [HADOOP-17936](https://issues.apache.org/jira/browse/HADOOP-17936) -- `fs.s3a.payload.signing.enabled` introduced in Hadoop 3.3.5 +- [SeaweedFS #5193](https://github.com/seaweedfs/seaweedfs/issues/5193) -- unable to list objects with prefix ending with "/" +- [SeaweedFS #6113](https://github.com/seaweedfs/seaweedfs/issues/6113) -- directories disappear in Filer Store (race condition) +- [SeaweedFS #7826](https://github.com/seaweedfs/seaweedfs/pull/7826) -- implicit directory handling for S3 compatibility +- Helm chart implementation: `helm/starlake/templates/seaweedfs/hadoop-config.yaml` diff --git a/helm/starlake/.helmignore b/helm/starlake/.helmignore new file mode 100644 index 0000000..33df904 --- /dev/null +++ b/helm/starlake/.helmignore @@ -0,0 +1,6 @@ +# Patterns to ignore when building packages. +*.md +*.txt +.git* +.DS_Store +examples/ diff --git a/helm/starlake/Chart.yaml b/helm/starlake/Chart.yaml new file mode 100644 index 0000000..fc79469 --- /dev/null +++ b/helm/starlake/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: starlake +description: Starlake Data Stack - A Helm chart for deploying Starlake with Airflow orchestrator on Kubernetes +type: application +version: 1.0.0 +appVersion: "1.5" +icon: https://docs.starlake.ai/img/starlake-logo.png +keywords: + - starlake + - airflow + - data + - etl + - orchestration +home: https://github.com/starlake-ai/starlake-data-stack +sources: + - https://github.com/starlake-ai/starlake-data-stack +maintainers: + - name: Starlake Team + email: contact@starlake.ai diff --git a/helm/starlake/README.md b/helm/starlake/README.md new file mode 100644 index 0000000..243cecf --- /dev/null +++ b/helm/starlake/README.md @@ -0,0 +1,618 @@ +# Starlake Helm Chart + +[![Helm](https://img.shields.io/badge/Helm-3.0%2B-blue?logo=helm)](https://helm.sh) +[![Kubernetes](https://img.shields.io/badge/Kubernetes-1.19%2B-blue?logo=kubernetes)](https://kubernetes.io) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +Deploy the **Starlake Data Stack** on Kubernetes with Airflow as orchestrator. + +## Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Ingress / LB │ +└─────────────────────────────┬───────────────────────────────┘ + │ + ┌─────────▼──────────┐ + │ Starlake UI │ Port 80 + │ (Main Entry Point)│ Handles /airflow proxy + └─────────┬──────────┘ + │ + ┌─────────────────────┼─────────────────────┐ + │ │ │ + ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐ + │ Airflow │ │ Agent │ │ Gizmo │ + │ :8080 │ │ :8000 │ │ :10900 │ + └─────┬─────┘ └─────┬─────┘ └───────────┘ + │ │ + └────────┬───────────┘ + ┌─────▼─────┐ + │ PostgreSQL│ + │ :5432 │ + └───────────┘ + + ┌─────────────────────────────────────────────────────────┐ + │ Shared Storage (PVC - RWX) │ + │ /projects │ + │ ┌──────────────────────────────────────────────────┐ │ + │ │ Projects │ DAGs │ DuckDB files │ Configurations │ │ + │ └──────────────────────────────────────────────────┘ │ + │ ▲ ▲ │ + │ │ │ │ + │ Starlake UI Airflow │ + │ (read/write) (read DAGs) │ + └─────────────────────────────────────────────────────────┘ +``` + +## Features + +- **Flexible PostgreSQL** - Internal StatefulSet or external managed database (RDS, CloudSQL, Azure Database) +- **Shared Storage** - PVC with ReadWriteMany for projects shared between UI and Airflow +- **Integrated Services** - Starlake UI (with reverse proxy), Airflow, AI Agent, Gizmo +- **Health Probes** - Startup, liveness, and readiness probes for all services +- **Secrets Management** - Support for existing Kubernetes secrets or inline credentials +- **Demo Mode** - Pre-configured demo projects for quick evaluation + +## Prerequisites + +| Requirement | Version | Notes | +|------------|---------|-------| +| Kubernetes | 1.19+ | EKS, GKE, AKS, or on-premise | +| Helm | 3.0+ | [Installation guide](https://helm.sh/docs/intro/install/) | +| Storage Class | RWX | NFS, EFS, Filestore, Azure Files | +| Ingress Controller | Optional | NGINX, Traefik, ALB, GCE | + +## Quick Start + +### Option 1: Development (Internal PostgreSQL) + +```bash +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace +``` + +### Option 2: Production (External PostgreSQL) + +```bash +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --set postgresql.external.enabled=true \ + --set postgresql.external.host=my-postgres.example.com \ + --set postgresql.internal.enabled=false \ + --set postgresql.credentials.existingSecret=my-postgres-secret +``` + +### Option 3: With Ingress + +```bash +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --set ingress.enabled=true \ + --set ingress.host=starlake.mycompany.com \ + --set ingress.className=nginx \ + --set ui.service.type=ClusterIP +``` + +### Access the Application + +```bash +# Port-forward (UI proxies /airflow automatically) +kubectl port-forward svc/starlake-ui 8080:80 -n starlake + +# Open in browser +# UI: http://localhost:8080 +# Airflow: http://localhost:8080/airflow (credentials: airflow/airflow) +``` + +> **Note**: The UI acts as a reverse proxy for Airflow. A single port-forward provides access to both services. + +## Configuration + +### PostgreSQL Options + +
+Internal PostgreSQL (Default) + +Deploys a PostgreSQL StatefulSet within the cluster: + +```yaml +postgresql: + external: + enabled: false + internal: + enabled: true + persistence: + size: 50Gi + storageClass: "standard" + credentials: + username: dbuser + password: dbuser123 # Change in production! +``` + +**Pros**: Simple, all-in-one deployment +**Cons**: Requires backup management, no native HA + +
+ +
+External PostgreSQL (Recommended for Production) + +Uses a managed database service: + +```yaml +postgresql: + external: + enabled: true + host: "my-rds.abc123.us-east-1.rds.amazonaws.com" + port: 5432 + starlakeDatabase: starlake + airflowDatabase: airflow + internal: + enabled: false + credentials: + existingSecret: my-postgres-secret + usernameKey: postgres-user + passwordKey: postgres-password +``` + +**Pros**: HA, automatic backups, better performance +**Cons**: Additional cost + +
+ +### Storage Configuration + +The `projects` PVC must support **ReadWriteMany** access mode. + +| Cloud Provider | Storage Class | Notes | +|---------------|---------------|-------| +| AWS | `efs-sc` | Requires EFS CSI driver | +| GCP | `filestore-csi` | Minimum 1TB | +| Azure | `azurefile` | Premium recommended | +| On-premise | `nfs-client` | Requires NFS provisioner | + +```yaml +persistence: + projects: + enabled: true + storageClass: "efs-sc" + size: 100Gi +``` + +### Secrets Management + +
+Using Existing Kubernetes Secret (Recommended) + +```bash +# Create secret +kubectl create secret generic my-postgres-secret \ + --from-literal=postgres-user=dbuser \ + --from-literal=postgres-password=SecurePassword123 \ + -n starlake +``` + +```yaml +postgresql: + credentials: + existingSecret: my-postgres-secret + usernameKey: postgres-user + passwordKey: postgres-password +``` + +
+ +
+Inline Credentials (Development Only) + +```yaml +postgresql: + credentials: + username: dbuser + password: my-password +``` + +⚠️ **Warning**: Not recommended for production environments. + +
+ +### Demo Mode + +Enable demo projects for quick evaluation: + +```bash +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --set demo.enabled=true +``` + +This automatically initializes: +- Demo projects (tpch001, starbake, etc.) +- DuckLake databases with sample data +- Pre-configured Airflow DAGs + +Access with: `admin@localhost.local` + +## Deployment Examples + +
+AWS (EKS + RDS + EFS) + +> **Note**: This configuration is provided as a reference. Ingress and multi-replica setups require validation in your environment. + +```yaml +# values-aws.yaml +postgresql: + external: + enabled: true + host: my-rds.abc123.us-east-1.rds.amazonaws.com + internal: + enabled: false + credentials: + existingSecret: starlake-postgres-secret + +persistence: + projects: + storageClass: efs-sc + size: 200Gi + +ui: + service: + type: ClusterIP + +airflow: + admin: + password: "ChangeThisPassword!" +``` + +```bash +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --values values-aws.yaml +``` + +
+ +
+GCP (GKE + CloudSQL + Filestore) + +> **Note**: This configuration is provided as a reference. Ingress and multi-replica setups require validation in your environment. + +```yaml +# values-gcp.yaml +postgresql: + external: + enabled: true + host: 10.1.2.3 # CloudSQL private IP + internal: + enabled: false + +persistence: + projects: + storageClass: filestore-csi + size: 1Ti # Filestore minimum + +serviceAccount: + create: true + annotations: + iam.gke.io/gcp-service-account: starlake-sa@PROJECT.iam.gserviceaccount.com +``` + +
+ +
+Local Testing (K3s/K3d) + +```bash +# Create cluster +k3d cluster create starlake-test --servers 1 --agents 0 --port "8080:80@loadbalancer" + +# Install chart +helm install starlake ./helm/starlake \ + --namespace starlake \ + --create-namespace \ + --set postgresql.internal.persistence.storageClass=local-path \ + --set persistence.projects.storageClass=local-path \ + --set ui.service.type=ClusterIP \ + --set ui.frontendUrl=http://localhost:8080 \ + --set airflow.baseUrl=http://localhost:8080/airflow + +# Access +kubectl port-forward svc/starlake-ui 8080:80 -n starlake +``` + +
+ +## Parameters Reference + +### Global + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `nameOverride` | Override chart name | `""` | +| `fullnameOverride` | Override full name | `""` | + +### PostgreSQL + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `postgresql.external.enabled` | Use external PostgreSQL | `false` | +| `postgresql.external.host` | External PostgreSQL host | `""` | +| `postgresql.external.port` | External PostgreSQL port | `5432` | +| `postgresql.internal.enabled` | Deploy internal PostgreSQL | `true` | +| `postgresql.internal.persistence.size` | PostgreSQL PVC size | `50Gi` | +| `postgresql.credentials.username` | PostgreSQL username | `dbuser` | +| `postgresql.credentials.password` | PostgreSQL password | `dbuser123` | +| `postgresql.credentials.existingSecret` | Use existing secret | `""` | + +### Starlake UI + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `ui.enabled` | Enable UI | `true` | +| `ui.replicas` | Number of replicas | `1` | +| `ui.appType` | App type (`ducklake` or `web`) | `ducklake` | +| `ui.frontendUrl` | Frontend URL override (for port-forward) | `""` | +| `ui.service.type` | Service type | `LoadBalancer` | +| `ui.service.port` | Service port | `80` | +| `ui.resources.requests.memory` | Memory request | `1Gi` | +| `ui.resources.limits.memory` | Memory limit | `4Gi` | + +### Airflow + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `airflow.enabled` | Enable Airflow | `true` | +| `airflow.version` | Airflow version (2 or 3) | `2` | +| `airflow.baseUrl` | Base URL override (for redirects) | `""` | +| `airflow.admin.username` | Admin username | `airflow` | +| `airflow.admin.password` | Admin password | `airflow` | +| `airflow.webserver.replicas` | Webserver replicas | `1` | +| `airflow.webserver.resources.requests.memory` | Memory request | `4Gi` | +| `airflow.webserver.resources.limits.memory` | Memory limit | `16Gi` | +| `airflow.secretKey` | Webserver session secret key | `starlake-airflow-...` | + +### Agent & Gizmo + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `agent.enabled` | Enable AI Agent | `true` | +| `agent.replicas` | Number of replicas | `1` | +| `gizmo.enabled` | Enable Gizmo SQL service | `false` | +| `gizmo.replicas` | Number of replicas | `1` | + +### Ingress + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `ingress.enabled` | Enable Ingress | `false` | +| `ingress.className` | Ingress class | `nginx` | +| `ingress.host` | Ingress hostname | `starlake.example.com` | +| `ingress.tls.enabled` | Enable TLS | `false` | +| `ingress.tls.secretName` | TLS secret name | `starlake-tls` | + +### Demo + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `demo.enabled` | Enable demo projects | `false` | + +For the complete list, see [values.yaml](values.yaml). + +## Upgrading + +```bash +# Upgrade with new values file (recommended) +helm upgrade starlake ./helm/starlake \ + --namespace starlake \ + --values values-production.yaml + +# Upgrade with specific value overrides +helm upgrade starlake ./helm/starlake \ + --namespace starlake \ + --set airflow.webserver.resources.limits.memory=16Gi + +# View history +helm history starlake -n starlake + +# Rollback if needed +helm rollback starlake 1 -n starlake +``` + +> **Warning**: Avoid using `--reuse-values` when you want to pick up new defaults from `values.yaml`. This flag preserves previously set values, which may override updated defaults. + +## Uninstalling + +```bash +# Remove the release +helm uninstall starlake -n starlake + +# Remove PVCs (optional - this deletes data!) +kubectl delete pvc -l app.kubernetes.io/instance=starlake -n starlake + +# Remove namespace +kubectl delete namespace starlake +``` + +## Troubleshooting + +
+Pods stuck in Pending state + +Check PVC status: +```bash +kubectl get pvc -n starlake +kubectl describe pvc starlake-projects -n starlake +``` + +Verify storage class supports ReadWriteMany: +```bash +kubectl get storageclass +``` + +
+ +
+PostgreSQL connection errors + +Test connectivity: +```bash +kubectl exec -it deployment/starlake-ui -n starlake -- nc -zv starlake-postgresql 5432 +``` + +Check credentials: +```bash +kubectl get secret starlake-postgresql -n starlake -o yaml +``` + +
+ +
+Airflow not starting + +Check init container logs: +```bash +kubectl logs -n starlake -l app.kubernetes.io/component=airflow -c init-airflow-db +``` + +Restart deployment: +```bash +kubectl rollout restart deployment/starlake-airflow -n starlake +``` + +
+ +
+Airflow OOMKilled errors + +If the Airflow pod shows `OOMKilled` status, the memory limit is too low. The pod runs webserver + scheduler + Starlake CLI (Java), requiring significant memory. + +Check current memory limits: +```bash +kubectl get pod -n starlake -l app.kubernetes.io/component=airflow \ + -o jsonpath='{.items[0].spec.containers[0].resources}' +``` + +Increase memory via helm upgrade (do NOT use `--reuse-values` to pick up new defaults): +```bash +helm upgrade starlake ./helm/starlake \ + --namespace starlake \ + --set airflow.webserver.resources.requests.memory=4Gi \ + --set airflow.webserver.resources.limits.memory=16Gi +``` + +Default values: `4Gi` request / `16Gi` limit. + +
+ +
+Airflow 403 Forbidden when reading logs + +If you see "Please make sure that all your Airflow components have the same 'secret_key' configured", the `AIRFLOW__WEBSERVER__SECRET_KEY` is not consistent. + +This is automatically handled by the chart via `airflow.secretKey`. For production, generate a new key: +```bash +python -c "import secrets; print(secrets.token_hex(32))" +``` + +Then set it: +```bash +helm upgrade starlake ./helm/starlake \ + --namespace starlake \ + --reuse-values \ + --set airflow.secretKey="your-generated-secret-key" +``` + +
+ +
+View component logs + +```bash +# UI logs +kubectl logs -n starlake -l app.kubernetes.io/component=ui -f + +# Airflow logs +kubectl logs -n starlake -l app.kubernetes.io/component=airflow -f + +# PostgreSQL logs +kubectl logs -n starlake -l app.kubernetes.io/component=postgresql -f +``` + +
+ +## Known Limitations + +### Multi-Node K3d Clusters with local-path Storage + +When testing with K3d in a multi-node configuration, the `local-path` storage class has significant limitations: + +**The Problem:** +- `local-path` creates PersistentVolumes with node affinity (the PV is bound to the node where it was first used) +- If a PVC is created on `agent-0`, only pods scheduled on that node can access the volume +- K3d port mapping (e.g., `--port "11900:11900@server:0"`) only forwards traffic to the server node +- Services using `hostNetwork: true` (like Gizmo) must run on the node where ports are mapped +- **Conflict**: Gizmo cannot use `nodeSelector` to run on the server node because the PVC is bound to a different agent node + +**Solutions:** + +1. **Single-node cluster (recommended for local testing)**: + ```bash + k3d cluster create starlake-test --servers 1 --agents 0 --port "8080:80@loadbalancer" + ``` + +2. **Use port-forward for Gizmo in multi-node clusters**: + ```bash + kubectl port-forward deploy/starlake-gizmo 11900:11900 -n starlake + ``` + +3. **Use RWX storage for production**: EFS (AWS), Filestore (GCP), Azure Files, or NFS provisioner + +**Gizmo JDBC Connection:** +``` +jdbc:arrow-flight-sql://localhost:11900?useEncryption=true&disableCertificateVerification=true +``` +- User: `gizmosql_user` +- Password: `gizmosql_password` + +## Roadmap + +The following features are planned but not yet tested in production: + +- [ ] **Ingress Support** - Test with NGINX, ALB, GCE, Traefik ingress controllers +- [ ] **Multiple Replicas** - Validate HA setup with RWX storage (EFS, Filestore, Azure Files) +- [ ] **Security Contexts** - Apply pod security standards and network policies +- [ ] **Monitoring** - Add Prometheus metrics and Grafana dashboards +- [ ] **Backup/Restore** - Document backup procedures for PostgreSQL and projects PVC + +Contributions to validate and document these features are welcome! + +## Contributing + +Contributions are welcome! Please read our [Contributing Guide](../../CONTRIBUTING.md) for details. + +1. Fork the repository +2. Create your feature branch (`git checkout -b feature/amazing-feature`) +3. Commit your changes (`git commit -m 'Add amazing feature'`) +4. Push to the branch (`git push origin feature/amazing-feature`) +5. Open a Pull Request + +## License + +This Helm chart is distributed under the [Apache License 2.0](../../LICENSE). + +## Support + +- 📖 **Documentation**: [starlake.ai/docs](https://starlake.ai/docs) +- 🐛 **Issues**: [GitHub Issues](https://github.com/starlake-ai/starlake-data-stack/issues) +- 💬 **Community**: [Slack](https://starlake.slack.com) +- 📧 **Email**: support@starlake.ai + +--- + +Made with ❤️ by the [Starlake](https://starlake.ai) team diff --git a/helm/starlake/scripts/init-airflow-database.sh b/helm/starlake/scripts/init-airflow-database.sh new file mode 100755 index 0000000..cd2af77 --- /dev/null +++ b/helm/starlake/scripts/init-airflow-database.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE ${AIRFLOW_DB:-airflow}; + GRANT ALL PRIVILEGES ON DATABASE ${AIRFLOW_DB:-airflow} TO "$POSTGRES_USER"; +EOSQL \ No newline at end of file diff --git a/helm/starlake/scripts/starlake.sh b/helm/starlake/scripts/starlake.sh new file mode 100755 index 0000000..61cee68 --- /dev/null +++ b/helm/starlake/scripts/starlake.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +old_ifs="$IFS" + +# Check if at least one argument is passed +if [ "$#" -eq 0 ]; then + echo "No arguments provided. Usage: starlake [args...]" + exit 1 +fi + +options="" +command="$1" +shift + +arguments=() +while [ $# -gt 0 ]; do + case "$1" in + -o|--options) options="$2"; shift 2 ;; + *) arguments+=("$1"); shift ;; + esac +done + +export JAVA_HOME=/opt/java/openjdk +export PATH=$JAVA_HOME/bin:$PATH + +# Export environment variables from --options, if provided +if [ -n "$options" ]; then + IFS=',' read -ra env_array <<< "$options" + for env in "${env_array[@]}"; do + name="${env%%=*}" + value="${env#*=}" + + # Remove surrounding quotes (both single and double) from value + value="${value%\"}" + value="${value#\"}" + value="${value%\'}" + value="${value#\'}" + + # Export the variable + export "$name=$value" + done + IFS="$old_ifs" +fi + +/app/starlake/starlake.sh $command ${arguments[@]} 2>&1 diff --git a/helm/starlake/templates/NOTES.txt b/helm/starlake/templates/NOTES.txt new file mode 100644 index 0000000..09c6554 --- /dev/null +++ b/helm/starlake/templates/NOTES.txt @@ -0,0 +1,122 @@ +🚀 Starlake Data Stack has been deployed! + +{{ if .Values.postgresql.internal.enabled -}} +✅ PostgreSQL: Internal (StatefulSet) +{{- else -}} +✅ PostgreSQL: External ({{ .Values.postgresql.external.host }}) +{{- end }} + +✅ Starlake UI: Enabled +{{ if .Values.airflow.enabled -}} +✅ Airflow: Enabled (version {{ .Values.airflow.version }}) +{{- end }} +{{ if .Values.agent.enabled -}} +✅ Starlake Agent: Enabled +{{- end }} +{{ if .Values.gizmo.enabled -}} +✅ Gizmo: Enabled +{{- end }} +{{ if .Values.seaweedfs.enabled -}} +✅ SeaweedFS: Enabled (S3 API on port {{ .Values.seaweedfs.service.s3Port }}) +{{- end }} + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +📋 Access Information: + +{{ if .Values.ingress.enabled -}} +🌐 Ingress URL: {{ if .Values.ingress.tls.enabled }}https{{ else }}http{{ end }}://{{ .Values.ingress.host }} + + - Starlake UI: {{ if .Values.ingress.tls.enabled }}https{{ else }}http{{ end }}://{{ .Values.ingress.host }}{{ .Values.ingress.paths.ui }} + {{- if .Values.airflow.enabled }} + - Airflow: {{ if .Values.ingress.tls.enabled }}https{{ else }}http{{ end }}://{{ .Values.ingress.host }}{{ .Values.ingress.paths.airflow }} + {{- end }} + {{- if .Values.agent.enabled }} + - Agent: {{ if .Values.ingress.tls.enabled }}https{{ else }}http{{ end }}://{{ .Values.ingress.host }}{{ .Values.ingress.paths.agent }} + {{- end }} +{{- else if eq .Values.ui.service.type "LoadBalancer" }} +⏳ Waiting for LoadBalancer IP... + +Run this command to get the external IP: + kubectl get svc {{ include "starlake.fullname" . }}-ui -n {{ .Release.Namespace }} + +Once ready, access Starlake at: + http://:{{ .Values.ui.service.port }} +{{- else if eq .Values.ui.service.type "NodePort" }} +🔗 NodePort Service: + +Get the NodePort: + export NODE_PORT=$(kubectl get svc {{ include "starlake.fullname" . }}-ui -n {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}') + export NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="ExternalIP")].address}') + echo "http://$NODE_IP:$NODE_PORT" +{{- else }} +🔗 Port-forward to access locally: + + kubectl port-forward svc/{{ include "starlake.fullname" . }}-ui 8080:{{ .Values.ui.service.port }} -n {{ .Release.Namespace }} + +Then open: + - Starlake UI: http://localhost:8080 + {{- if .Values.airflow.enabled }} + - Airflow: http://localhost:8080/airflow (via UI proxy) + {{- end }} + +Note: UI acts as reverse proxy for Airflow. Single port-forward for both services. +{{- end }} + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +{{ if .Values.airflow.enabled -}} +🔐 Airflow Credentials: + Username: {{ .Values.airflow.admin.username }} + Password: {{ .Values.airflow.admin.password }} +{{- end }} + +{{ if .Values.postgresql.internal.enabled -}} +📊 PostgreSQL Credentials: + Username: {{ .Values.postgresql.credentials.username }} + Password: {{ .Values.postgresql.credentials.password }} + + Connect from within cluster: + Host: {{ include "starlake.fullname" . }}-postgresql + Port: 5432 + Database: {{ .Values.postgresql.external.starlakeDatabase }} +{{- end }} + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +📚 Useful Commands: + +# View all resources +kubectl get all -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} + +# View logs for specific components +kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=ui -f +kubectl logs -n {{ .Release.Namespace }} -l app.kubernetes.io/component=airflow -f + +# Check PostgreSQL status +{{ if .Values.postgresql.internal.enabled -}} +kubectl get statefulset {{ include "starlake.fullname" . }}-postgresql -n {{ .Release.Namespace }} +{{- end }} + +# Access PostgreSQL shell +{{ if .Values.postgresql.internal.enabled -}} +kubectl exec -it {{ include "starlake.fullname" . }}-postgresql-0 -n {{ .Release.Namespace }} -- psql -U {{ .Values.postgresql.credentials.username }} -d {{ .Values.postgresql.external.starlakeDatabase }} +{{- end }} + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +⚠️ Important Notes: + +{{ if not .Values.postgresql.external.enabled -}} +1. PostgreSQL data is persisted in a PVC. To reset, delete the PVC: + kubectl delete pvc data-{{ include "starlake.fullname" . }}-postgresql-0 -n {{ .Release.Namespace }} +{{- end }} + +2. Projects are stored in PVC: {{ include "starlake.fullname" . }}-projects + Ensure your cluster supports ReadWriteMany access mode (e.g., NFS, EFS) + +3. For production deployments, update default passwords in values.yaml or use existing secrets + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +For more information, visit: https://github.com/starlake-ai/starlake-data-stack diff --git a/helm/starlake/templates/_helpers.tpl b/helm/starlake/templates/_helpers.tpl new file mode 100644 index 0000000..f3d0d98 --- /dev/null +++ b/helm/starlake/templates/_helpers.tpl @@ -0,0 +1,332 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "starlake.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "starlake.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "starlake.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "starlake.labels" -}} +helm.sh/chart: {{ include "starlake.chart" . }} +{{ include "starlake.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "starlake.selectorLabels" -}} +app.kubernetes.io/name: {{ include "starlake.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Component-specific labels +*/}} +{{- define "starlake.componentLabels" -}} +{{- $component := .component }} +{{- with .context }} +{{ include "starlake.labels" . }} +app.kubernetes.io/component: {{ $component }} +{{- end }} +{{- end }} + +{{/* +Component-specific selector labels +*/}} +{{- define "starlake.componentSelectorLabels" -}} +{{- $component := .component }} +{{- with .context }} +{{ include "starlake.selectorLabels" . }} +app.kubernetes.io/component: {{ $component }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "starlake.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "starlake.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +PostgreSQL host +*/}} +{{- define "starlake.postgresql.host" -}} +{{- if .Values.postgresql.external.enabled -}} +{{- .Values.postgresql.external.host -}} +{{- else -}} +{{- printf "%s-postgresql" (include "starlake.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +PostgreSQL port +*/}} +{{- define "starlake.postgresql.port" -}} +{{- if .Values.postgresql.external.enabled -}} +{{- .Values.postgresql.external.port -}} +{{- else -}} +5432 +{{- end -}} +{{- end -}} + +{{/* +PostgreSQL Starlake database name +*/}} +{{- define "starlake.postgresql.starlakeDatabase" -}} +{{- .Values.postgresql.external.starlakeDatabase -}} +{{- end -}} + +{{/* +PostgreSQL Airflow database name +*/}} +{{- define "starlake.postgresql.airflowDatabase" -}} +{{- .Values.postgresql.external.airflowDatabase -}} +{{- end -}} + +{{/* +PostgreSQL username +*/}} +{{- define "starlake.postgresql.username" -}} +{{- .Values.postgresql.credentials.username -}} +{{- end -}} + +{{/* +PostgreSQL password secret name +*/}} +{{- define "starlake.postgresql.secretName" -}} +{{- if .Values.postgresql.credentials.existingSecret -}} +{{- .Values.postgresql.credentials.existingSecret -}} +{{- else -}} +{{- printf "%s-postgresql" (include "starlake.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +PostgreSQL username secret key +*/}} +{{- define "starlake.postgresql.usernameKey" -}} +{{- if .Values.postgresql.credentials.existingSecret -}} +{{- .Values.postgresql.credentials.usernameKey | default "postgres-user" -}} +{{- else -}} +postgres-user +{{- end -}} +{{- end -}} + +{{/* +PostgreSQL password secret key +*/}} +{{- define "starlake.postgresql.passwordKey" -}} +{{- if .Values.postgresql.credentials.existingSecret -}} +{{- .Values.postgresql.credentials.passwordKey | default "postgres-password" -}} +{{- else -}} +postgres-password +{{- end -}} +{{- end -}} + +{{/* +PostgreSQL JDBC URL for Starlake database +*/}} +{{- define "starlake.postgresql.jdbcUrl" -}} +{{- $host := include "starlake.postgresql.host" . }} +{{- $port := include "starlake.postgresql.port" . }} +{{- $database := include "starlake.postgresql.starlakeDatabase" . }} +{{- $username := include "starlake.postgresql.username" . }} +{{- printf "jdbc:postgresql://%s:%s/%s?user=%s" $host $port $database $username }} +{{- end }} + +{{/* +PostgreSQL connection string for Airflow +*/}} +{{- define "starlake.postgresql.airflowConnectionString" -}} +{{- $host := include "starlake.postgresql.host" . }} +{{- $port := include "starlake.postgresql.port" . }} +{{- $database := include "starlake.postgresql.airflowDatabase" . }} +{{- $username := include "starlake.postgresql.username" . }} +{{- printf "postgresql+psycopg2://%s:$(POSTGRES_PASSWORD)@%s:%s/%s" $username $host $port $database }} +{{- end }} + +{{/* +Storage class for PVCs +*/}} +{{- define "starlake.storageClass" -}} +{{- if .storageClass }} +{{- .storageClass }} +{{- else if .context.Values.global.storageClass }} +{{- .context.Values.global.storageClass }} +{{- else }} +{{- "" }} +{{- end }} +{{- end }} + +{{/* +Frontend URL +*/}} +{{- define "starlake.frontendUrl" -}} +{{- if .Values.ingress.enabled -}} +{{- if .Values.ingress.tls.enabled -}} +{{- printf "https://%s" .Values.ingress.host -}} +{{- else -}} +{{- printf "http://%s" .Values.ingress.host -}} +{{- end -}} +{{- else -}} +{{- printf "http://localhost:%d" (int .Values.ui.service.port) -}} +{{- end -}} +{{- end -}} + +{{/* +Starlake domain +*/}} +{{- define "starlake.domain" -}} +{{- if .Values.ingress.enabled -}} +{{- .Values.ingress.host -}} +{{- else -}} +localhost +{{- end -}} +{{- end -}} + +{{/* +Wait for PostgreSQL init container +*/}} +{{- define "starlake.waitForPostgresql" -}} +- name: wait-for-postgresql + image: {{ .Values.initImages.busybox.repository }}:{{ .Values.initImages.busybox.tag }} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + until nc -z {{ include "starlake.postgresql.host" . }} {{ include "starlake.postgresql.port" . }}; do + echo "Waiting for PostgreSQL..." + sleep 2 + done + echo "PostgreSQL is ready!" +{{- end -}} + +{{/* +Wait for SeaweedFS S3 bucket init container +Only included when seaweedfs.enabled is true +Uses aws-cli to verify the bucket exists and is accessible +*/}} +{{- define "starlake.waitForSeaweedfs" -}} +{{- if .Values.seaweedfs.enabled }} +- name: wait-for-seaweedfs-bucket + image: {{ .Values.initImages.awsCli.repository }}:{{ .Values.initImages.awsCli.tag }} + imagePullPolicy: IfNotPresent + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: BUCKET_NAME + value: {{ .Values.seaweedfs.s3.bucket | quote }} + command: + - /bin/sh + - -c + - | + echo "Waiting for SeaweedFS S3 bucket '$BUCKET_NAME' to be ready..." + MAX_RETRIES=60 + RETRY_INTERVAL=5 + RETRIES=0 + + while [ $RETRIES -lt $MAX_RETRIES ]; do + # Try to list the bucket (will fail if bucket doesn't exist or SeaweedFS not ready) + if aws --endpoint-url "$S3_ENDPOINT" s3 ls "s3://$BUCKET_NAME" 2>/dev/null; then + echo "Bucket s3://$BUCKET_NAME is ready!" + exit 0 + fi + + RETRIES=$((RETRIES + 1)) + echo "Bucket not ready yet (attempt $RETRIES/$MAX_RETRIES), waiting ${RETRY_INTERVAL}s..." + sleep $RETRY_INTERVAL + done + + echo "ERROR: Bucket s3://$BUCKET_NAME not available after $MAX_RETRIES attempts" + exit 1 +{{- end }} +{{- end -}} + +{{/* +Validate credentials - fails deployment if insecure defaults are used in production +Enable this validation with: security.validateCredentials: true +Recommended for production deployments to enforce secure credentials +*/}} +{{- define "starlake.validateCredentials" -}} +{{- if .Values.security.validateCredentials }} + {{- /* PostgreSQL password validation */ -}} + {{- if not .Values.postgresql.credentials.existingSecret }} + {{- if eq .Values.postgresql.credentials.password "dbuser123" }} + {{- fail "SECURITY ERROR: postgresql.credentials.password is set to default value 'dbuser123'. For production, set a secure password or use existingSecret." }} + {{- end }} + {{- if not .Values.postgresql.credentials.password }} + {{- fail "SECURITY ERROR: postgresql.credentials.password is required. Set a secure password or use existingSecret." }} + {{- end }} + {{- end }} + {{- /* Airflow admin password validation */ -}} + {{- if .Values.airflow.enabled }} + {{- if eq .Values.airflow.admin.password "airflow" }} + {{- fail "SECURITY ERROR: airflow.admin.password is set to default value 'airflow'. For production, set a secure password." }} + {{- end }} + {{- /* Airflow secretKey validation */ -}} + {{- if eq .Values.airflow.secretKey "starlake-airflow-secret-key-change-in-production" }} + {{- fail "SECURITY ERROR: airflow.secretKey is set to default value. For production, generate a new key: python -c \"import secrets; print(secrets.token_hex(32))\"" }} + {{- end }} + {{- end }} + {{- /* Gizmo credentials validation */ -}} + {{- if .Values.gizmo.enabled }} + {{- if eq .Values.gizmo.apiKey "a_secret_api_key" }} + {{- fail "SECURITY ERROR: gizmo.apiKey is set to default value. For production, set a secure API key." }} + {{- end }} + {{- end }} + {{- /* Agent applicationKey validation */ -}} + {{- if .Values.agent.enabled }} + {{- if or (eq .Values.agent.applicationKey "change-me-in-production") (eq .Values.agent.applicationKey "Starlake7157") }} + {{- fail "SECURITY ERROR: agent.applicationKey is set to default value. For production, set a secure application key." }} + {{- end }} + {{- end }} + {{- /* SeaweedFS S3 credentials validation */ -}} + {{- if .Values.seaweedfs.enabled }} + {{- if eq .Values.seaweedfs.s3.accessKey "seaweedfs" }} + {{- fail "SECURITY ERROR: seaweedfs.s3.accessKey is set to default value 'seaweedfs'. For production, set a secure access key." }} + {{- end }} + {{- if eq .Values.seaweedfs.s3.secretKey "seaweedfs123" }} + {{- fail "SECURITY ERROR: seaweedfs.s3.secretKey is set to default value 'seaweedfs123'. For production, set a secure secret key." }} + {{- end }} + {{- end }} +{{- end }} +{{- end -}} diff --git a/helm/starlake/templates/agent/deployment.yaml b/helm/starlake/templates/agent/deployment.yaml new file mode 100644 index 0000000..9afba69 --- /dev/null +++ b/helm/starlake/templates/agent/deployment.yaml @@ -0,0 +1,140 @@ +{{- if .Values.agent.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "starlake.fullname" . }}-agent + labels: + {{- include "starlake.componentLabels" (dict "component" "agent" "context" .) | nindent 4 }} +spec: + replicas: {{ .Values.agent.replicas }} + selector: + matchLabels: + {{- include "starlake.componentSelectorLabels" (dict "component" "agent" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "starlake.componentSelectorLabels" (dict "component" "agent" "context" .) | nindent 8 }} + spec: + serviceAccountName: {{ include "starlake.serviceAccountName" . }} + # fsGroup ensures shared volume files are accessible by all pods + securityContext: + fsGroup: {{ .Values.podSecurityContext.fsGroup }} + initContainers: + {{- include "starlake.waitForPostgresql" . | nindent 8 }} + {{- include "starlake.waitForSeaweedfs" . | nindent 8 }} + containers: + - name: agent + image: "{{ .Values.agent.image.repository }}:{{ .Values.agent.image.tag }}" + imagePullPolicy: {{ .Values.agent.image.pullPolicy }} + # Note: Agent image requires root, disabling securityContext + ports: + - name: http + containerPort: 8000 + protocol: TCP + env: + - name: HOME + value: /tmp + - name: UV_CACHE_DIR + value: /tmp/.cache/uv + - name: SL_ASK_APPLICATION_KEY + value: {{ .Values.agent.applicationKey | quote }} + - name: SL_ASK_STARLAKE_JSON_FOLDER + value: /app/starlake_ask/resources + # Storage Mode - ALWAYS local filesystem, even with SeaweedFS enabled + # SeaweedFS is provisioned as infrastructure but not used by default + # Users create S3 projects manually via UI when needed + - name: SL_ROOT + value: "/projects" + {{- if .Values.seaweedfs.enabled }} + # S3 credentials available for manual project creation in UI + # These are NOT used by default - only when user creates S3-backed projects + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: AWS_S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: AWS_REGION + value: "us-east-1" + - name: HADOOP_CONF_DIR + value: "/etc/hadoop/conf" + {{- end }} + - name: SL_ASK_PG_USER + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.usernameKey" . }} + - name: SL_ASK_PG_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: SL_ASK_PG_DB + value: {{ include "starlake.postgresql.starlakeDatabase" . | quote }} + - name: SL_ASK_PG_SERVER + value: {{ include "starlake.postgresql.host" . | quote }} + - name: SL_ASK_PG_PORT + value: {{ include "starlake.postgresql.port" . | quote }} + - name: SL_ASK_API_SERVER_URL + value: http://{{ include "starlake.fullname" . }}-ui:{{ .Values.ui.service.port }} + - name: SL_ASK_HOST + value: 0.0.0.0 + - name: SL_ASK_DB_TYPE + value: postgres + - name: SL_ASK_API_DOCS_USER + value: starlake + - name: SL_ASK_API_DOCS_PASSWORD + value: s3cret.Paw + # Startup probe - allows slow startup + # Agent Python app with uvicorn needs significant time to initialize + # initialDelaySeconds=30 gives time for Python deps to load + startupProbe: + httpGet: + path: /ask/health + port: http + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + # Liveness probe + livenessProbe: + httpGet: + path: /ask/health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness probe + readinessProbe: + httpGet: + path: /ask/health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + {{- toYaml .Values.agent.resources | nindent 10 }} + {{- if .Values.seaweedfs.enabled }} + volumeMounts: + - name: hadoop-config + mountPath: /etc/hadoop/conf + {{- end }} + {{- if .Values.seaweedfs.enabled }} + volumes: + - name: hadoop-config + configMap: + name: {{ include "starlake.fullname" . }}-hadoop-config + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/agent/service.yaml b/helm/starlake/templates/agent/service.yaml new file mode 100644 index 0000000..abd1612 --- /dev/null +++ b/helm/starlake/templates/agent/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.agent.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "starlake.fullname" . }}-agent + labels: + {{- include "starlake.componentLabels" (dict "component" "agent" "context" .) | nindent 4 }} +spec: + type: {{ .Values.agent.service.type }} + ports: + - name: http + port: {{ .Values.agent.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "starlake.componentSelectorLabels" (dict "component" "agent" "context" .) | nindent 4 }} +{{- end }} diff --git a/helm/starlake/templates/airflow/deployment.yaml b/helm/starlake/templates/airflow/deployment.yaml new file mode 100644 index 0000000..a899cf5 --- /dev/null +++ b/helm/starlake/templates/airflow/deployment.yaml @@ -0,0 +1,475 @@ +{{- if .Values.airflow.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "starlake.fullname" . }}-airflow + labels: + {{- include "starlake.componentLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +spec: + replicas: {{ .Values.airflow.webserver.replicas }} + selector: + matchLabels: + {{- include "starlake.componentSelectorLabels" (dict "component" "airflow" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "starlake.componentSelectorLabels" (dict "component" "airflow" "context" .) | nindent 8 }} + spec: + serviceAccountName: {{ include "starlake.serviceAccountName" . }} + {{- if .Values.airflow.jobRunner.enabled }} + # Required for Job Runner: ensures SA token is mounted for kubectl auth + automountServiceAccountToken: true + {{- end }} + # fsGroup ensures shared volume files are accessible by all pods + # Airflow runs as user 50000, but fsGroup adds group 1000 for shared access + securityContext: + fsGroup: {{ .Values.podSecurityContext.fsGroup }} + initContainers: + {{- include "starlake.waitForPostgresql" . | nindent 8 }} + {{- include "starlake.waitForSeaweedfs" . | nindent 8 }} + {{- if not .Values.seaweedfs.enabled }} + # Fix permissions on existing files - differentiated for security + # Most files: group read/write (g+rwX) for shared access + # DuckDB stored_secrets: group read only (g-w) to protect credentials + # Skip when SeaweedFS is enabled (data stored in S3, no local PVC) + - name: fix-permissions + image: {{ .Values.initImages.busybox.repository }}:{{ .Values.initImages.busybox.tag }} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Fixing permissions on /projects for group {{ .Values.podSecurityContext.fsGroup }}..." + + # Set group ownership on all files + chgrp -R {{ .Values.podSecurityContext.fsGroup }} /projects 2>/dev/null || true + + # Default: group read/write for all files + chmod -R g+rwX /projects 2>/dev/null || true + + # DuckDB stored_secrets directories: remove group write (contains unencrypted credentials) + find /projects -type d -name "stored_secrets" -exec chmod -R g-w {} \; 2>/dev/null || true + + # DuckDB secret files: owner-only (DuckDB requires 0600 for security) + find /projects -name "*.duckdb_secret" -exec chmod 600 {} \; 2>/dev/null || true + + echo "Permissions fixed (stored_secrets: g-w, .duckdb_secret: 600, rest: g+rwX)" + ls -la /projects/ + securityContext: + runAsUser: 0 + volumeMounts: + - name: projects + mountPath: /projects + {{- end }} + {{- if .Values.airflow.installPythonPackages }} + # Copy Starlake CLI from UI image (only needed when using official Airflow image) + # When using custom image (Dockerfile_airflow), starlake is already installed + - name: setup-starlake-tools + image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag }}" + imagePullPolicy: {{ .Values.ui.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + echo "=== Setting up Starlake tools for Airflow ===" + + # Copy entire Starlake directory (contains starlake.sh and lib/) + echo "Copying Starlake CLI directory..." + mkdir -p /shared-tools/starlake + if [ -d "/app/starlake" ]; then + cp -r /app/starlake/* /shared-tools/starlake/ 2>/dev/null || true + echo "Copied from /app/starlake" + ls -la /shared-tools/starlake/ + fi + + # Copy gcloud SDK if available in UI image + if [ -d "/usr/local/gcloud" ]; then + echo "Copying Google Cloud SDK..." + cp -r /usr/local/gcloud /shared-tools/ + fi + + # Create wrapper scripts in bin directory + mkdir -p /shared-tools/bin + + # Create a shell-mode starlake wrapper that executes locally (not via docker exec) + # This wrapper handles --options and calls the actual starlake.sh + cat > /shared-tools/bin/starlake << 'EOFSL' + #!/usr/bin/env bash + old_ifs="$IFS" + + if [ "$#" -eq 0 ]; then + echo "No arguments provided. Usage: starlake [args...]" + exit 1 + fi + + options="" + command="$1" + shift + + arguments=() + while [ $# -gt 0 ]; do + case "$1" in + -o|--options) options="$2"; shift 2 ;; + *) arguments+=("$1"); shift ;; + esac + done + + # Set Java home if available + if [ -d "/opt/java/openjdk" ]; then + export JAVA_HOME=/opt/java/openjdk + export PATH=$JAVA_HOME/bin:$PATH + fi + + # Export environment variables from --options, if provided + if [ -n "$options" ]; then + IFS=',' read -ra env_array <<< "$options" + for env in "${env_array[@]}"; do + name="${env%%=*}" + value="${env#*=}" + value="${value%\"}" + value="${value#\"}" + value="${value%\'}" + value="${value#\'}" + export "$name=$value" + done + IFS="$old_ifs" + fi + + # Execute starlake directly (shell mode, not docker mode) + if [ -x "/shared-tools/starlake/starlake.sh" ]; then + exec /shared-tools/starlake/starlake.sh "$command" "${arguments[@]}" 2>&1 + elif [ -x "/shared-tools/starlake/starlake" ]; then + exec /shared-tools/starlake/starlake "$command" "${arguments[@]}" 2>&1 + elif [ -x "/usr/local/bin/starlake" ]; then + exec /usr/local/bin/starlake "$command" "${arguments[@]}" 2>&1 + else + echo "ERROR: starlake executable not found" + exit 1 + fi + EOFSL + chmod +x /shared-tools/bin/starlake + + # gcloud wrapper (if SDK was copied) + if [ -d "/shared-tools/gcloud" ]; then + ln -sf /shared-tools/gcloud/google-cloud-sdk/bin/gcloud /shared-tools/bin/gcloud + ln -sf /shared-tools/gcloud/google-cloud-sdk/bin/gsutil /shared-tools/bin/gsutil + fi + + {{- if .Values.airflow.jobRunner.enabled }} + # Install kubectl for Job Runner (must be done in initContainer as root) + echo "Installing kubectl for Job Runner..." + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + mv kubectl /shared-tools/bin/kubectl + echo "kubectl installed: $(/shared-tools/bin/kubectl version --client 2>/dev/null | head -1)" + {{- end }} + + echo "Tools setup complete!" + echo "Contents of /shared-tools:" + ls -la /shared-tools/ + echo "Contents of /shared-tools/bin:" + ls -la /shared-tools/bin/ + echo "Contents of /shared-tools/starlake:" + ls -la /shared-tools/starlake/ || echo "No starlake dir" + volumeMounts: + - name: shared-tools + mountPath: /shared-tools + {{- end }} + # Initialize Airflow DB directly in init container (more reliable than waiting for Job) + - name: init-airflow-db + image: "{{ .Values.airflow.image.repository }}:{{ .Values.airflow.image.tag }}" + imagePullPolicy: {{ .Values.airflow.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + echo "Waiting for database to be ready..." + sleep 5 + echo "Initializing Airflow database..." + airflow db init || airflow db migrate + echo "Creating Airflow admin user..." + airflow users create \ + --username {{ .Values.airflow.admin.username }} \ + --firstname {{ .Values.airflow.admin.firstname }} \ + --lastname {{ .Values.airflow.admin.lastname }} \ + --role Admin \ + --email {{ .Values.airflow.admin.email }} \ + --password "$AIRFLOW_ADMIN_PASSWORD" || echo "User already exists" + echo "Airflow database initialization complete!" + env: + # Admin password from Secret (not visible in kubectl describe) + - name: AIRFLOW_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.fullname" . }}-airflow + key: admin-password + - name: AIRFLOW__CORE__EXECUTOR + value: {{ .Values.airflow.executor | quote }} + - name: AIRFLOW__WEBSERVER__SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "starlake.fullname" . }}-airflow + key: secret-key + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN + value: {{ include "starlake.postgresql.airflowConnectionString" . | quote }} + - name: AIRFLOW__CORE__LOAD_EXAMPLES + value: "false" + containers: + {{- if not .Values.seaweedfs.enabled }} + # Sidecar container that continuously fixes permissions on /projects + # This is needed because Airflow subprocesses don't inherit umask from the main shell + # Skip when SeaweedFS is enabled (data stored in S3, no local PVC) + - name: permission-fixer + image: {{ .Values.initImages.busybox.repository }}:{{ .Values.initImages.busybox.tag }} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + echo "Starting permission fixer sidecar..." + while true; do + # Fix group ownership and permissions on all files + chgrp -R {{ .Values.podSecurityContext.fsGroup }} /projects 2>/dev/null || true + chmod -R g+rwX /projects 2>/dev/null || true + + # DuckDB stored_secrets directories: remove group write (contains unencrypted credentials) + find /projects -type d -name "stored_secrets" -exec chmod -R g-w {} \; 2>/dev/null || true + + # DuckDB secret files: owner-only (DuckDB requires 0600 for security) + find /projects -name "*.duckdb_secret" -exec chmod 600 {} \; 2>/dev/null || true + + # Run every 5 seconds + sleep 5 + done + securityContext: + runAsUser: 0 + resources: + limits: + cpu: 50m + memory: 32Mi + requests: + cpu: 10m + memory: 16Mi + volumeMounts: + - name: projects + mountPath: /projects + {{- end }} + - name: airflow + image: "{{ .Values.airflow.image.repository }}:{{ .Values.airflow.image.tag }}" + imagePullPolicy: {{ .Values.airflow.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + env: + # Java environment for Starlake CLI + - name: JAVA_HOME + value: /opt/java/openjdk + - name: AIRFLOW__CORE__EXECUTOR + value: {{ .Values.airflow.executor | quote }} + # Secret key must be the same across all Airflow components + - name: AIRFLOW__WEBSERVER__SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "starlake.fullname" . }}-airflow + key: secret-key + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN + value: {{ include "starlake.postgresql.airflowConnectionString" . | quote }} + - name: AIRFLOW__CORE__LOAD_EXAMPLES + value: "false" + - name: AIRFLOW__API__AUTH_BACKENDS + value: "airflow.api.auth.backend.basic_auth" + - name: SL_HOME + value: /app/starlake + # Storage Mode - ALWAYS local filesystem, even with SeaweedFS enabled + # SeaweedFS is provisioned as infrastructure but not used by default + # Users create S3 projects manually via UI when needed + - name: SL_FS + value: "file://" + - name: SL_ROOT + value: "/projects" + {{- if .Values.seaweedfs.enabled }} + # S3 credentials available for manual project creation in UI + # These are NOT used by default - only when user creates S3-backed projects + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: AWS_S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: AWS_REGION + value: "us-east-1" + - name: HADOOP_CONF_DIR + value: "/etc/hadoop/conf" + {{- end }} + - name: AIRFLOW__WEBSERVER__BASE_URL + {{- if .Values.airflow.baseUrl }} + value: {{ .Values.airflow.baseUrl | quote }} + {{- else }} + value: {{ include "starlake.frontendUrl" . }}/airflow + {{- end }} + - name: AIRFLOW__CORE__LOGGING_LEVEL + value: {{ .Values.airflow.logLevel | quote }} + - name: AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL + value: {{ .Values.airflow.dagDirListInterval | quote }} + - name: AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL + value: {{ .Values.airflow.dagMinFileProcessInterval | quote }} + {{- if .Values.airflow.jobRunner.enabled }} + # Use starlake-k8s wrapper to execute tasks as Kubernetes Jobs + - name: SL_STARLAKE_PATH + value: "starlake-k8s" + - name: STARLAKE_NAMESPACE + value: {{ .Release.Namespace | quote }} + # Add shared-tools/bin to PATH for kubectl + - name: PATH + value: "/shared-tools/bin:/opt/java/openjdk/bin:/home/airflow/.local/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" + {{- end }} + command: + - /bin/bash + - -c + - | + # Set umask to allow group write on new files/directories + umask 002 + + echo "Waiting before starting Airflow..." + sleep 10 + + {{- if .Values.airflow.installPythonPackages }} + echo "Installing Python packages (starlake-airflow for Airflow 2)..." + # IMPORTANT: Pin starlake-airflow~=0.4 for Airflow 2 (0.5+ requires Airflow 3) + pip install --no-cache-dir \ + "starlake-airflow>=0.4,<0.5" \ + docker \ + apache-airflow-providers-amazon \ + apache-airflow-providers-google + {{- else }} + echo "Skipping pip install (packages pre-installed in image)" + {{- end }} + + # Note: kubectl is installed in initContainer (setup-starlake-tools) when jobRunner.enabled + echo "Setting up Starlake tools in PATH..." + # Add shared-tools/bin to PATH for starlake, gcloud, gsutil + export PATH="/shared-tools/bin:$PATH" + # If using custom image, starlake is already in /usr/local/bin + if [ ! -f /usr/local/bin/starlake ]; then + ln -sf /shared-tools/bin/starlake /usr/local/bin/starlake 2>/dev/null || true + chmod +x /usr/local/bin/starlake 2>/dev/null || true + fi + + echo "Starting Airflow webserver in background..." + airflow webserver & + + echo "Starting Airflow scheduler..." + exec airflow scheduler + # Startup probe - allows slow startup (pip install + db init can take 2-3 min) + # Matches docker-compose: interval=5s, retries=60 = 5 min max + startupProbe: + httpGet: + path: /airflow/health + port: http + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + # Liveness probe - after startup succeeds + livenessProbe: + httpGet: + path: /airflow/health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness probe + readinessProbe: + httpGet: + path: /airflow/health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + {{- toYaml .Values.airflow.webserver.resources | nindent 12 }} + volumeMounts: + - name: shared-tools + mountPath: /shared-tools + - name: projects + mountPath: /projects + - name: projects + mountPath: /opt/airflow/dags + subPath: dags + {{- if .Values.airflow.logs.persistence.enabled }} + - name: logs + mountPath: /opt/airflow/logs + {{- end }} + {{- if .Values.airflow.jobRunner.enabled }} + - name: job-template + mountPath: /etc/starlake + {{- if not .Values.airflow.jobRunner.useBuiltinWrapper }} + - name: starlake-wrapper + mountPath: /usr/local/bin/starlake-k8s + subPath: starlake-k8s.sh + {{- end }} + {{- end }} + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + mountPath: /etc/hadoop/conf + {{- end }} + volumes: + - name: shared-tools + emptyDir: {} + - name: projects + # Always use PVC for local filesystem storage + # SeaweedFS provides S3 infrastructure but doesn't change deployment mode + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-projects + {{- if .Values.airflow.jobRunner.enabled }} + - name: job-template + configMap: + name: {{ include "starlake.fullname" . }}-job-template + {{- if not .Values.airflow.jobRunner.useBuiltinWrapper }} + - name: starlake-wrapper + configMap: + name: {{ include "starlake.fullname" . }}-starlake-wrapper + defaultMode: 0755 + {{- end }} + {{- end }} + {{- if .Values.airflow.logs.persistence.enabled }} + - name: logs + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-airflow-logs + {{- else }} + - name: logs + emptyDir: {} + {{- end }} + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + configMap: + name: {{ include "starlake.fullname" . }}-hadoop-config + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/airflow/job-template-configmap.yaml b/helm/starlake/templates/airflow/job-template-configmap.yaml new file mode 100644 index 0000000..641b0dd --- /dev/null +++ b/helm/starlake/templates/airflow/job-template-configmap.yaml @@ -0,0 +1,106 @@ +{{- if and .Values.airflow.enabled .Values.airflow.jobRunner.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "starlake.fullname" . }}-job-template + labels: + {{- include "starlake.componentLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +data: + # Job template used by the starlake wrapper to create Kubernetes Jobs + # Placeholders are replaced at runtime: + # __JOB_NAME__: unique job name (e.g., sl-load-customer-20240129-123456) + # __STARLAKE_COMMAND__: starlake command (e.g., load) + # __STARLAKE_ARGS__: command arguments as JSON array + # __ENV_VARS__: environment variables as JSON array + job-template.yaml: | + apiVersion: batch/v1 + kind: Job + metadata: + name: __JOB_NAME__ + labels: + app.kubernetes.io/name: starlake + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: starlake-job + app.kubernetes.io/managed-by: airflow + spec: + ttlSecondsAfterFinished: {{ .Values.airflow.jobRunner.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.airflow.jobRunner.backoffLimit }} + template: + metadata: + labels: + app.kubernetes.io/name: starlake + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: starlake-job + spec: + restartPolicy: Never + securityContext: + fsGroup: {{ .Values.podSecurityContext.fsGroup }} + containers: + - name: starlake + image: "{{ .Values.airflow.jobRunner.image.repository }}:{{ .Values.airflow.jobRunner.image.tag }}" + imagePullPolicy: {{ .Values.airflow.jobRunner.image.pullPolicy }} + # Use bash to create /incoming symlink before running starlake + # NOTE: Must use /bin/bash because /app/starlake.sh has #!/usr/bin/env bash + command: + - /bin/bash + - -c + - | + # Create /incoming symlink pointing to $SL_ROOT/incoming + # Required because starlake projects use absolute /incoming path + if [ -n "$SL_ROOT" ]; then + mkdir -p "$SL_ROOT/incoming" 2>/dev/null || true + ln -sf "$SL_ROOT/incoming" /incoming 2>/dev/null || true + echo "- /incoming -> $SL_ROOT/incoming" + fi + # Execute starlake with the provided args + # /app/starlake.sh is the CLI location in the starlakeai/starlake image + exec /app/starlake.sh "$@" + - "--" + args: __STARLAKE_ARGS__ + env: + # Storage Mode - ALWAYS local filesystem, even with SeaweedFS enabled + # SL_ROOT is dynamically replaced by Airflow DAG with project path + # SeaweedFS is provisioned but not used by default + - name: SL_ROOT + value: __SL_ROOT__ + {{- if .Values.seaweedfs.enabled }} + # S3 credentials available for manual project creation in UI + # These are NOT used by default - only when user creates S3-backed projects + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: AWS_S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: AWS_REGION + value: "us-east-1" + - name: HADOOP_CONF_DIR + value: "/etc/hadoop/conf" + {{- end }} + __ENV_VARS__ + resources: + requests: + memory: "{{ .Values.airflow.jobRunner.resources.requests.memory }}" + cpu: "{{ .Values.airflow.jobRunner.resources.requests.cpu }}" + limits: + memory: "{{ .Values.airflow.jobRunner.resources.limits.memory }}" + cpu: "{{ .Values.airflow.jobRunner.resources.limits.cpu }}" + volumeMounts: + - name: projects + mountPath: /projects + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + mountPath: /etc/hadoop/conf + {{- end }} + volumes: + # Always use PVC for /projects (local filesystem) + # SeaweedFS is provisioned separately but not used by default + - name: projects + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-projects + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + configMap: + name: {{ include "starlake.fullname" . }}-hadoop-config + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/airflow/rbac.yaml b/helm/starlake/templates/airflow/rbac.yaml new file mode 100644 index 0000000..9a07ddb --- /dev/null +++ b/helm/starlake/templates/airflow/rbac.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.airflow.enabled .Values.airflow.jobRunner.enabled }} +--- +# Role to allow Airflow to create and manage Kubernetes Jobs for Starlake tasks +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "starlake.fullname" . }}-job-runner + labels: + {{- include "starlake.componentLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +rules: + # Jobs management - create, monitor, delete Starlake task jobs + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] + # Pods management - needed to stream logs and check status + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + # Pod logs - needed to stream task output back to Airflow + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list", "watch"] +--- +# Bind the role to the Starlake ServiceAccount used by Airflow +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "starlake.fullname" . }}-job-runner + labels: + {{- include "starlake.componentLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "starlake.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "starlake.fullname" . }}-job-runner + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/helm/starlake/templates/airflow/service.yaml b/helm/starlake/templates/airflow/service.yaml new file mode 100644 index 0000000..245154c --- /dev/null +++ b/helm/starlake/templates/airflow/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.airflow.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "starlake.fullname" . }}-airflow + labels: + {{- include "starlake.componentLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +spec: + type: {{ .Values.airflow.webserver.service.type }} + ports: + - name: http + port: {{ .Values.airflow.webserver.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "starlake.componentSelectorLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +{{- end }} diff --git a/helm/starlake/templates/airflow/starlake-wrapper-configmap.yaml b/helm/starlake/templates/airflow/starlake-wrapper-configmap.yaml new file mode 100644 index 0000000..1798a1c --- /dev/null +++ b/helm/starlake/templates/airflow/starlake-wrapper-configmap.yaml @@ -0,0 +1,157 @@ +{{- if and .Values.airflow.enabled .Values.airflow.jobRunner.enabled (not .Values.airflow.jobRunner.useBuiltinWrapper) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "starlake.fullname" . }}-starlake-wrapper + labels: + {{- include "starlake.componentLabels" (dict "component" "airflow" "context" .) | nindent 4 }} +data: + # Wrapper script that executes starlake commands as Kubernetes Jobs + # Usage: starlake-k8s [args...] + # Example: starlake-k8s load --domains=customer --tables=orders + starlake-k8s.sh: | + #!/bin/bash + set -e + + # Configuration from environment + JOB_TEMPLATE="/etc/starlake/job-template.yaml" + NAMESPACE="${STARLAKE_NAMESPACE:-{{ .Release.Namespace }}}" + + # Configure kubectl for in-cluster authentication + # Use kubernetes.default.svc as fallback when env vars not available (e.g., in Airflow subprocess) + if [ -n "$KUBERNETES_SERVICE_HOST" ]; then + KUBE_API_SERVER="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" + else + KUBE_API_SERVER="https://kubernetes.default.svc:443" + fi + KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + KUBE_CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + # Set kubectl alias with in-cluster config + kubectl() { + /shared-tools/bin/kubectl --server="$KUBE_API_SERVER" --token="$KUBE_TOKEN" --certificate-authority="$KUBE_CA_CERT" "$@" + } + + # Generate unique job name based on command and timestamp + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + RANDOM_SUFFIX=$(printf '%04x' $RANDOM) + COMMAND="${1:-unknown}" + JOB_NAME="sl-${COMMAND}-${TIMESTAMP}-${RANDOM_SUFFIX}" + # Kubernetes job names must be lowercase and max 63 chars + JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]' | cut -c1-63) + + # Build args array for YAML (JSON array format) + # Proper JSON escaping to prevent command injection + ARGS_JSON="[" + FIRST=true + for arg in "$@"; do + if [ "$FIRST" = true ]; then + FIRST=false + else + ARGS_JSON="${ARGS_JSON}, " + fi + # Properly escape for JSON: backslashes first, then quotes, tabs, newlines + ESCAPED_ARG=$(printf '%s' "$arg" | sed 's/\\/\\\\/g; s/"/\\"/g; s/ /\\t/g' | tr '\n' ' ') + ARGS_JSON="${ARGS_JSON}\"${ESCAPED_ARG}\"" + done + ARGS_JSON="${ARGS_JSON}]" + + # Create temporary job manifest + TEMP_JOB=$(mktemp /tmp/starlake-job-XXXXXX.yaml) + trap "rm -f $TEMP_JOB" EXIT + + # Build env vars YAML (12 spaces indent to match template) + ENV_FILE=$(mktemp /tmp/starlake-env-XXXXXX.yaml) + trap "rm -f $TEMP_JOB $ENV_FILE" EXIT + + for var in $(env | grep -E "^SL_" | grep -v "^SL_ROOT=" | cut -d= -f1); do + value="${!var}" + # Escape for YAML string: backslashes first, then double quotes + escaped_value=$(printf '%s' "$value" | sed 's/\\/\\\\/g; s/"/\\"/g') + # 12 spaces for - name:, 14 spaces for value: (matching template) + echo " - name: ${var}" >> "$ENV_FILE" + echo " value: \"${escaped_value}\"" >> "$ENV_FILE" + done + + # Read template and create job manifest + # First pass: replace simple placeholders + sed -e "s|__JOB_NAME__|${JOB_NAME}|g" \ + -e "s|__STARLAKE_ARGS__|${ARGS_JSON}|g" \ + -e "s|__SL_ROOT__|${SL_ROOT:-/projects}|g" \ + "$JOB_TEMPLATE" > "$TEMP_JOB" + + # Second pass: replace __ENV_VARS__ with actual env vars from file + if [ -s "$ENV_FILE" ]; then + # Use awk to replace __ENV_VARS__ with file contents + awk -v envfile="$ENV_FILE" ' + /__ENV_VARS__/ { + while ((getline line < envfile) > 0) print line + close(envfile) + next + } + {print} + ' "$TEMP_JOB" > "${TEMP_JOB}.tmp" && mv "${TEMP_JOB}.tmp" "$TEMP_JOB" + else + # No env vars, just remove the placeholder + sed -i '/__ENV_VARS__/d' "$TEMP_JOB" + fi + + echo "=== Creating Kubernetes Job: ${JOB_NAME} ===" + echo "Command: starlake $@" + echo "Namespace: ${NAMESPACE}" + echo "SL_ROOT: ${SL_ROOT:-/projects}" + + # Create the job + kubectl apply -f "$TEMP_JOB" -n "$NAMESPACE" + + # Wait for pod to be created and get its name + echo "=== Waiting for pod to start ===" + POD_NAME="" + for i in $(seq 1 60); do + POD_NAME=$(kubectl get pods -n "$NAMESPACE" -l "job-name=${JOB_NAME}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [ -n "$POD_NAME" ]; then + POD_STATUS=$(kubectl get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Pending") + if [ "$POD_STATUS" != "Pending" ]; then + break + fi + fi + sleep 2 + done + + if [ -z "$POD_NAME" ]; then + echo "ERROR: Pod not created after 120 seconds" + kubectl get jobs -n "$NAMESPACE" -l "job-name=${JOB_NAME}" -o yaml + exit 1 + fi + + echo "=== Pod ${POD_NAME} started, streaming logs ===" + + # Stream logs (follow until completion) + kubectl logs -f "$POD_NAME" -n "$NAMESPACE" -c starlake 2>/dev/null || true + + # Wait for job to complete and get exit status + echo "=== Waiting for job completion ===" + kubectl wait --for=condition=complete --timeout=3600s "job/${JOB_NAME}" -n "$NAMESPACE" 2>/dev/null && { + echo "=== Job completed successfully ===" + exit 0 + } + + # Check if job failed + kubectl wait --for=condition=failed --timeout=10s "job/${JOB_NAME}" -n "$NAMESPACE" 2>/dev/null && { + echo "=== Job failed ===" + # Get exit code from pod + EXIT_CODE=$(kubectl get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || echo "1") + exit "${EXIT_CODE:-1}" + } + + # Fallback - job might still be running or in unknown state + JOB_STATUS=$(kubectl get job "${JOB_NAME}" -n "$NAMESPACE" -o jsonpath='{.status.conditions[*].type}' 2>/dev/null || echo "Unknown") + echo "=== Job status: ${JOB_STATUS} ===" + + # Return appropriate exit code + if echo "$JOB_STATUS" | grep -q "Complete"; then + exit 0 + else + exit 1 + fi +{{- end }} diff --git a/helm/starlake/templates/configmap.yaml b/helm/starlake/templates/configmap.yaml new file mode 100644 index 0000000..acffe25 --- /dev/null +++ b/helm/starlake/templates/configmap.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "starlake.fullname" . }}-scripts + labels: + {{- include "starlake.labels" . | nindent 4 }} +data: + init-airflow-database.sh: | +{{ .Files.Get "scripts/init-airflow-database.sh" | indent 4 }} + + starlake.sh: | +{{ .Files.Get "scripts/starlake.sh" | indent 4 }} + +{{- if .Values.postgresql.internal.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "starlake.fullname" . }}-postgresql-config + labels: + {{- include "starlake.labels" . | nindent 4 }} +data: + postgresql.conf: | + max_connections = {{ .Values.postgresql.internal.config.maxConnections }} + shared_buffers = {{ .Values.postgresql.internal.config.sharedBuffers }} + # Additional PostgreSQL configuration + listen_addresses = '*' + log_timezone = 'UTC' + datestyle = 'iso, mdy' + timezone = 'UTC' + lc_messages = 'en_US.utf8' + lc_monetary = 'en_US.utf8' + lc_numeric = 'en_US.utf8' + lc_time = 'en_US.utf8' + default_text_search_config = 'pg_catalog.english' +{{- end }} diff --git a/helm/starlake/templates/database/service-alias.yaml b/helm/starlake/templates/database/service-alias.yaml new file mode 100644 index 0000000..43f3d2f --- /dev/null +++ b/helm/starlake/templates/database/service-alias.yaml @@ -0,0 +1,16 @@ +{{- if .Values.postgresql.internal.enabled }} +# Alias service for Docker Compose compatibility +# Projects created in Docker Compose use "starlake-db" as the PostgreSQL hostname +# This ExternalName service redirects "starlake-db" to the actual PostgreSQL service +apiVersion: v1 +kind: Service +metadata: + name: starlake-db + namespace: {{ .Release.Namespace }} + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: database-alias +spec: + type: ExternalName + externalName: {{ include "starlake.postgresql.host" . }}.{{ .Release.Namespace }}.svc.cluster.local +{{- end }} diff --git a/helm/starlake/templates/database/service.yaml b/helm/starlake/templates/database/service.yaml new file mode 100644 index 0000000..0362b25 --- /dev/null +++ b/helm/starlake/templates/database/service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.postgresql.internal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "starlake.fullname" . }}-postgresql + labels: + {{- include "starlake.componentLabels" (dict "component" "postgresql" "context" .) | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None # Headless service for StatefulSet + ports: + - name: postgresql + port: 5432 + targetPort: postgresql + protocol: TCP + selector: + {{- include "starlake.componentSelectorLabels" (dict "component" "postgresql" "context" .) | nindent 4 }} +{{- end }} diff --git a/helm/starlake/templates/database/statefulset.yaml b/helm/starlake/templates/database/statefulset.yaml new file mode 100644 index 0000000..66e6b65 --- /dev/null +++ b/helm/starlake/templates/database/statefulset.yaml @@ -0,0 +1,139 @@ +{{- if .Values.postgresql.internal.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "starlake.fullname" . }}-postgresql + labels: + {{- include "starlake.componentLabels" (dict "component" "postgresql" "context" .) | nindent 4 }} +spec: + serviceName: {{ include "starlake.fullname" . }}-postgresql + replicas: 1 + selector: + matchLabels: + {{- include "starlake.componentSelectorLabels" (dict "component" "postgresql" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "starlake.componentSelectorLabels" (dict "component" "postgresql" "context" .) | nindent 8 }} + spec: + serviceAccountName: {{ include "starlake.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: postgresql + image: "{{ .Values.postgresql.internal.image.repository }}:{{ .Values.postgresql.internal.image.tag }}" + imagePullPolicy: {{ .Values.postgresql.internal.image.pullPolicy }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + ports: + - name: postgresql + containerPort: 5432 + protocol: TCP + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.usernameKey" . }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: POSTGRES_DB + value: {{ include "starlake.postgresql.starlakeDatabase" . | quote }} + - name: AIRFLOW_DB + value: {{ include "starlake.postgresql.airflowDatabase" . | quote }} + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + args: + - -c + - config_file=/etc/postgresql/postgresql.conf + # Startup probe - allows slow startup without affecting liveness + startupProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -q -U "{{ .Values.postgresql.credentials.username }}" -d "{{ .Values.postgresql.external.starlakeDatabase }}" -h 127.0.0.1 -p 5432 + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 30 # 30 * 2s = 60s max startup time + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -q -U "{{ .Values.postgresql.credentials.username }}" -d "{{ .Values.postgresql.external.starlakeDatabase }}" -h 127.0.0.1 -p 5432 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -q -U "{{ .Values.postgresql.credentials.username }}" -d "{{ .Values.postgresql.external.starlakeDatabase }}" -h 127.0.0.1 -p 5432 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + resources: + {{- toYaml .Values.postgresql.internal.resources | nindent 10 }} + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: config + mountPath: /etc/postgresql + - name: init-scripts + mountPath: /docker-entrypoint-initdb.d + volumes: + - name: config + configMap: + name: {{ include "starlake.fullname" . }}-postgresql-config + - name: init-scripts + configMap: + name: {{ include "starlake.fullname" . }}-scripts + items: + - key: init-airflow-database.sh + path: init-airflow-database.sh + mode: 0755 + {{- if not .Values.postgresql.internal.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.postgresql.internal.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "starlake.componentLabels" (dict "component" "postgresql" "context" .) | nindent 8 }} + spec: + accessModes: + - ReadWriteOnce + {{- with (include "starlake.storageClass" (dict "storageClass" .Values.postgresql.internal.persistence.storageClass "context" .)) }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.postgresql.internal.persistence.size }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/gizmo/deployment.yaml b/helm/starlake/templates/gizmo/deployment.yaml new file mode 100644 index 0000000..743f163 --- /dev/null +++ b/helm/starlake/templates/gizmo/deployment.yaml @@ -0,0 +1,156 @@ +{{- if .Values.gizmo.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "starlake.fullname" . }}-gizmo + labels: + {{- include "starlake.componentLabels" (dict "component" "gizmo" "context" .) | nindent 4 }} +spec: + replicas: {{ .Values.gizmo.replicas }} + selector: + matchLabels: + {{- include "starlake.componentSelectorLabels" (dict "component" "gizmo" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "starlake.componentSelectorLabels" (dict "component" "gizmo" "context" .) | nindent 8 }} + spec: + serviceAccountName: {{ include "starlake.serviceAccountName" . }} + {{- if .Values.gizmo.hostNetwork }} + # hostNetwork exposes SQL ports 11900-12000 directly on node IP + # Required for users to connect to Gizmo SQL instances + # See: helm/docs/GIZMO_NETWORK_EXPOSURE.md + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + # fsGroup ensures shared volume files are accessible by all pods + securityContext: + fsGroup: {{ .Values.podSecurityContext.fsGroup }} + initContainers: + {{- include "starlake.waitForPostgresql" . | nindent 8 }} + {{- include "starlake.waitForSeaweedfs" . | nindent 8 }} + containers: + - name: gizmo + image: "{{ .Values.gizmo.image.repository }}:{{ .Values.gizmo.image.tag }}" + imagePullPolicy: {{ .Values.gizmo.image.pullPolicy }} + ports: + - name: http + containerPort: 10900 + protocol: TCP + env: + - name: SL_GIZMO_ON_DEMAND_HOST + value: "0.0.0.0" + - name: SL_GIZMO_ON_DEMAND_PORT + value: "10900" + - name: SL_GIZMO_MIN_PORT + value: {{ .Values.gizmo.minPort | quote }} + - name: SL_GIZMO_MAX_PORT + value: {{ .Values.gizmo.maxPort | quote }} + - name: SL_GIZMO_MAX_PROCESSES + value: {{ .Values.gizmo.maxProcesses | quote }} + - name: SL_GIZMO_API_KEY + value: {{ .Values.gizmo.apiKey | quote }} + # Storage Mode - ALWAYS local filesystem, even with SeaweedFS enabled + # SeaweedFS is provisioned as infrastructure but not used by default + # Users create S3 projects manually via UI when needed + - name: SL_ROOT + value: "/projects" + {{- if .Values.seaweedfs.enabled }} + # S3 credentials available for manual project creation in UI + # These are NOT used by default - only when user creates S3-backed projects + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: AWS_S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: AWS_REGION + value: "us-east-1" + - name: HADOOP_CONF_DIR + value: "/etc/hadoop/conf" + {{- end }} + - name: PG_HOST + value: {{ include "starlake.postgresql.host" . | quote }} + - name: PG_PORT + value: {{ include "starlake.postgresql.port" . | quote }} + - name: PG_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.usernameKey" . }} + - name: PG_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: GIZMOSQL_USERNAME + value: {{ .Values.gizmo.sql.username | quote }} + - name: GIZMOSQL_PASSWORD + value: {{ .Values.gizmo.sql.password | quote }} + - name: JWT_SECRET_KEY + value: {{ .Values.gizmo.jwt.secretKey | quote }} + - name: TLS_ENABLED + value: "0" + # Startup probe - allows slow startup (up to 60s) + startupProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 30 + # Liveness probe - matches docker-compose: interval=5s, retries=60 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness probe + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + {{- toYaml .Values.gizmo.resources | nindent 10 }} + volumeMounts: + - name: projects + mountPath: /projects + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + mountPath: /etc/hadoop/conf + {{- end }} + volumes: + - name: projects + # Always use PVC for local filesystem storage + # SeaweedFS provides S3 infrastructure but doesn't change deployment mode + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-projects + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + configMap: + name: {{ include "starlake.fullname" . }}-hadoop-config + {{- end }} + {{- if .Values.gizmo.nodeSelector }} + nodeSelector: + {{- toYaml .Values.gizmo.nodeSelector | nindent 8 }} + {{- else }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/gizmo/service.yaml b/helm/starlake/templates/gizmo/service.yaml new file mode 100644 index 0000000..4d44ee3 --- /dev/null +++ b/helm/starlake/templates/gizmo/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.gizmo.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "starlake.fullname" . }}-gizmo + labels: + {{- include "starlake.componentLabels" (dict "component" "gizmo" "context" .) | nindent 4 }} +spec: + type: {{ .Values.gizmo.service.type }} + ports: + - port: {{ .Values.gizmo.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "starlake.componentSelectorLabels" (dict "component" "gizmo" "context" .) | nindent 4 }} +{{- end }} diff --git a/helm/starlake/templates/ingress.yaml b/helm/starlake/templates/ingress.yaml new file mode 100644 index 0000000..0c9c275 --- /dev/null +++ b/helm/starlake/templates/ingress.yaml @@ -0,0 +1,36 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "starlake.fullname" . }} + labels: + {{- include "starlake.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end }} + rules: + - {{- if .Values.ingress.host }} + host: {{ .Values.ingress.host }} + {{- end }} + http: + paths: + # Starlake UI - Root path (UI now handles all routing, no separate proxy) + - path: {{ .Values.ingress.paths.ui }} + pathType: Prefix + backend: + service: + name: {{ include "starlake.fullname" . }}-ui + port: + number: {{ .Values.ui.service.port }} +{{- end }} diff --git a/helm/starlake/templates/pvc.yaml b/helm/starlake/templates/pvc.yaml new file mode 100644 index 0000000..ec9ddc0 --- /dev/null +++ b/helm/starlake/templates/pvc.yaml @@ -0,0 +1,65 @@ +{{- if .Values.persistence.projects.enabled }} +{{- /* PVC needed even with SeaweedFS: stores .db files (DuckDB JDBC requires local access) */}} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "starlake.fullname" . }}-projects + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: storage +spec: + accessModes: + {{- $storageClass := include "starlake.storageClass" (dict "storageClass" .Values.persistence.projects.storageClass "context" .) }} + {{- if eq $storageClass "local-path" }} + - ReadWriteOnce # local-path only supports RWO (for K3s testing) + {{- else }} + - ReadWriteMany # Required for sharing between UI, Airflow, etc. + {{- end }} + {{- with $storageClass }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.projects.size }} +{{- end }} + +{{- if .Values.persistence.externalProjects.enabled }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "starlake.fullname" . }}-external-projects + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: storage +spec: + accessModes: + - ReadWriteMany + {{- with (include "starlake.storageClass" (dict "storageClass" .Values.persistence.externalProjects.storageClass "context" .)) }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.externalProjects.size }} +{{- end }} + +{{- if and .Values.airflow.enabled .Values.airflow.logs.persistence.enabled }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "starlake.fullname" . }}-airflow-logs + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: airflow +spec: + accessModes: + - ReadWriteOnce + {{- with (include "starlake.storageClass" (dict "storageClass" .Values.airflow.logs.persistence.storageClass "context" .)) }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.airflow.logs.persistence.size }} +{{- end }} diff --git a/helm/starlake/templates/seaweedfs/CLAUDE.md b/helm/starlake/templates/seaweedfs/CLAUDE.md new file mode 100644 index 0000000..adfdcb1 --- /dev/null +++ b/helm/starlake/templates/seaweedfs/CLAUDE.md @@ -0,0 +1,7 @@ + +# Recent Activity + + + +*No recent activity* + \ No newline at end of file diff --git a/helm/starlake/templates/seaweedfs/deployment.yaml b/helm/starlake/templates/seaweedfs/deployment.yaml new file mode 100644 index 0000000..1153a81 --- /dev/null +++ b/helm/starlake/templates/seaweedfs/deployment.yaml @@ -0,0 +1,107 @@ +{{- if .Values.seaweedfs.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "starlake.fullname" . }}-seaweedfs + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs +spec: + replicas: 1 + selector: + matchLabels: + {{- include "starlake.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: seaweedfs + template: + metadata: + labels: + {{- include "starlake.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: seaweedfs + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + fsGroup: 1000 + containers: + # Master + Volume + Filer in single container (dev/test mode) + - name: seaweedfs + image: "{{ .Values.seaweedfs.image.repository }}:{{ .Values.seaweedfs.image.tag }}" + imagePullPolicy: {{ .Values.seaweedfs.image.pullPolicy }} + args: + - "server" + - "-dir=/data" + - "-s3" + - "-s3.port=8333" + - "-s3.config=/etc/seaweedfs/s3.json" + - "-master.volumeSizeLimitMB={{ .Values.seaweedfs.volumeSizeLimitMB }}" + - "-ip.bind=0.0.0.0" + ports: + # HTTP ports + - name: master + containerPort: 9333 + protocol: TCP + - name: volume + containerPort: 8080 + protocol: TCP + - name: filer + containerPort: 8888 + protocol: TCP + - name: s3 + containerPort: 8333 + protocol: TCP + # gRPC ports (HTTP port + 10000) + - name: master-grpc + containerPort: 19333 + protocol: TCP + - name: volume-grpc + containerPort: 18080 + protocol: TCP + - name: filer-grpc + containerPort: 18888 + protocol: TCP + env: + - name: WEED_MASTER_VOLUME_SIZE_LIMIT_MB + value: "{{ .Values.seaweedfs.volumeSizeLimitMB }}" + volumeMounts: + - name: data + mountPath: /data + - name: s3-config + mountPath: /etc/seaweedfs + livenessProbe: + httpGet: + path: /cluster/status + port: master + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /cluster/status + port: master + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.seaweedfs.resources | nindent 12 }} + volumes: + - name: data + {{- if .Values.seaweedfs.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-seaweedfs + {{- else }} + emptyDir: {} + {{- end }} + - name: s3-config + secret: + secretName: {{ include "starlake.fullname" . }}-seaweedfs + {{- with .Values.seaweedfs.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seaweedfs.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/seaweedfs/hadoop-config.yaml b/helm/starlake/templates/seaweedfs/hadoop-config.yaml new file mode 100644 index 0000000..a169296 --- /dev/null +++ b/helm/starlake/templates/seaweedfs/hadoop-config.yaml @@ -0,0 +1,105 @@ +{{- if .Values.seaweedfs.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "starlake.fullname" . }}-hadoop-config + labels: + {{- include "starlake.componentLabels" (dict "component" "seaweedfs" "context" .) | nindent 4 }} +data: + # Spark defaults configuration with Hadoop S3A settings + # Spark reads Hadoop config via spark.hadoop.* prefix + spark-defaults.conf: | + # S3A FileSystem endpoint - SeaweedFS S3 API + spark.hadoop.fs.s3a.endpoint http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }} + spark.hadoop.fs.s3a.path.style.access true + spark.hadoop.fs.s3a.connection.ssl.enabled false + spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem + spark.hadoop.fs.s3a.access.key {{ .Values.seaweedfs.s3.accessKey }} + spark.hadoop.fs.s3a.secret.key {{ .Values.seaweedfs.s3.secretKey }} + spark.hadoop.fs.s3a.bucket.probe 0 + # Use S3 V2 signing to avoid AWS V4 chunked encoding (Hadoop 3.3.4 doesn't + # support payload.signing.enabled; V4 chunked encoding causes SeaweedFS to + # store 86-byte chunk terminators as directory marker content) + spark.hadoop.fs.s3a.signing-algorithm S3SignerType + # SeaweedFS-recommended settings (https://github.com/seaweedfs/seaweedfs/wiki/HDFS-via-S3-connector) + spark.hadoop.fs.s3a.directory.marker.retention keep + spark.hadoop.fs.s3a.multiobjectdelete.enable false + spark.hadoop.fs.s3a.change.detection.mode warn + spark.hadoop.fs.s3a.change.detection.version.required false + + # Hadoop core-site.xml configuration for S3A connector to use SeaweedFS + core-site.xml: | + + + + + + fs.s3a.endpoint + http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }} + + + + + fs.s3a.path.style.access + true + + + + + fs.s3a.connection.ssl.enabled + false + + + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + + + fs.s3a.access.key + {{ .Values.seaweedfs.s3.accessKey }} + + + fs.s3a.secret.key + {{ .Values.seaweedfs.s3.secretKey }} + + + + + fs.s3a.signing-algorithm + S3SignerType + + + + + fs.s3a.bucket.probe + 0 + + + + + + fs.s3a.directory.marker.retention + keep + + + + fs.s3a.multiobjectdelete.enable + false + + + + fs.s3a.change.detection.mode + warn + + + fs.s3a.change.detection.version.required + false + + +{{- end }} diff --git a/helm/starlake/templates/seaweedfs/job-init-bucket.yaml b/helm/starlake/templates/seaweedfs/job-init-bucket.yaml new file mode 100644 index 0000000..64912bd --- /dev/null +++ b/helm/starlake/templates/seaweedfs/job-init-bucket.yaml @@ -0,0 +1,90 @@ +{{- if .Values.seaweedfs.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "starlake.fullname" . }}-seaweedfs-init + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs-init + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + ttlSecondsAfterFinished: 60 + template: + metadata: + labels: + {{- include "starlake.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: seaweedfs-init + spec: + restartPolicy: OnFailure + # NOTE: Demo projects are NO LONGER uploaded to S3 + # Demos are handled by UI application and stored entirely on local PVC + # SeaweedFS is for user-created production projects only + containers: + - name: create-bucket + image: {{ .Values.initImages.awsCli.repository }}:{{ .Values.initImages.awsCli.tag }} + imagePullPolicy: IfNotPresent + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: BUCKET_NAME + value: {{ .Values.seaweedfs.s3.bucket | quote }} + command: + - /bin/sh + - -c + - | + set -e + echo "=== SeaweedFS Bucket Initialization ===" + echo "Endpoint: $S3_ENDPOINT" + echo "Bucket: $BUCKET_NAME" + + # Wait for SeaweedFS to be available (with retries) + echo "Waiting for SeaweedFS S3 API to be ready..." + MAX_WAIT=120 + WAIT_COUNT=0 + until aws --endpoint-url "$S3_ENDPOINT" s3 ls 2>/dev/null; do + WAIT_COUNT=$((WAIT_COUNT + 1)) + if [ $WAIT_COUNT -ge $MAX_WAIT ]; then + echo "ERROR: SeaweedFS S3 API not available after ${MAX_WAIT} attempts" + exit 1 + fi + echo "SeaweedFS not ready yet (attempt $WAIT_COUNT/$MAX_WAIT), waiting 2s..." + sleep 2 + done + echo "SeaweedFS S3 API is ready!" + + # Create bucket (ignore error if already exists) + echo "Creating bucket s3://$BUCKET_NAME..." + aws --endpoint-url "$S3_ENDPOINT" s3 mb "s3://$BUCKET_NAME" 2>/dev/null || { + if aws --endpoint-url "$S3_ENDPOINT" s3 ls "s3://$BUCKET_NAME" 2>/dev/null; then + echo "Bucket already exists, skipping creation" + else + echo "ERROR: Failed to create bucket" + exit 1 + fi + } + + # Verify bucket exists + echo "Verifying bucket..." + if aws --endpoint-url "$S3_ENDPOINT" s3 ls "s3://$BUCKET_NAME" 2>/dev/null; then + echo "Bucket s3://$BUCKET_NAME is ready!" + else + echo "ERROR: Bucket verification failed" + exit 1 + fi + + # List buckets for debugging + echo "Available buckets:" + aws --endpoint-url "$S3_ENDPOINT" s3 ls + + echo "" + echo "=== Bucket Initialization Complete ===" + echo "Note: Demo projects are managed by UI and stored on local PVC" + echo "SeaweedFS bucket is ready for user-created production projects" +{{- end }} diff --git a/helm/starlake/templates/seaweedfs/pvc.yaml b/helm/starlake/templates/seaweedfs/pvc.yaml new file mode 100644 index 0000000..3459b12 --- /dev/null +++ b/helm/starlake/templates/seaweedfs/pvc.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.seaweedfs.enabled .Values.seaweedfs.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "starlake.fullname" . }}-seaweedfs + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs +spec: + accessModes: + - {{ .Values.seaweedfs.persistence.accessMode }} + {{- if .Values.seaweedfs.persistence.storageClass }} + storageClassName: {{ .Values.seaweedfs.persistence.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.seaweedfs.persistence.size }} +{{- end }} diff --git a/helm/starlake/templates/seaweedfs/secret.yaml b/helm/starlake/templates/seaweedfs/secret.yaml new file mode 100644 index 0000000..857ddd2 --- /dev/null +++ b/helm/starlake/templates/seaweedfs/secret.yaml @@ -0,0 +1,27 @@ +{{- if .Values.seaweedfs.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "starlake.fullname" . }}-seaweedfs + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs +type: Opaque +stringData: + # S3 API configuration for SeaweedFS + s3.json: | + { + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "{{ .Values.seaweedfs.s3.accessKey }}", + "secretKey": "{{ .Values.seaweedfs.s3.secretKey }}" + } + ], + "actions": ["Admin", "Read", "Write", "List", "Tagging"] + } + ] + } +{{- end }} diff --git a/helm/starlake/templates/seaweedfs/service.yaml b/helm/starlake/templates/seaweedfs/service.yaml new file mode 100644 index 0000000..ae01549 --- /dev/null +++ b/helm/starlake/templates/seaweedfs/service.yaml @@ -0,0 +1,37 @@ +{{- if .Values.seaweedfs.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "starlake.fullname" . }}-seaweedfs + labels: + {{- include "starlake.labels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs +spec: + type: {{ .Values.seaweedfs.service.type }} + ports: + # HTTP ports + - port: {{ .Values.seaweedfs.service.s3Port }} + targetPort: s3 + protocol: TCP + name: s3 + - port: {{ .Values.seaweedfs.service.masterPort }} + targetPort: master + protocol: TCP + name: master + - port: {{ .Values.seaweedfs.service.filerPort }} + targetPort: filer + protocol: TCP + name: filer + # gRPC ports (HTTP port + 10000) + - port: 19333 + targetPort: master-grpc + protocol: TCP + name: master-grpc + - port: 18888 + targetPort: filer-grpc + protocol: TCP + name: filer-grpc + selector: + {{- include "starlake.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: seaweedfs +{{- end }} diff --git a/helm/starlake/templates/secrets.yaml b/helm/starlake/templates/secrets.yaml new file mode 100644 index 0000000..a0b758d --- /dev/null +++ b/helm/starlake/templates/secrets.yaml @@ -0,0 +1,64 @@ +{{- /* Validate credentials - fails if insecure defaults are used with security.validateCredentials: true */ -}} +{{- include "starlake.validateCredentials" . }} + +{{- if not .Values.postgresql.credentials.existingSecret }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "starlake.postgresql.secretName" . }} + labels: + {{- include "starlake.labels" . | nindent 4 }} +type: Opaque +stringData: + {{ include "starlake.postgresql.usernameKey" . }}: {{ .Values.postgresql.credentials.username | quote }} + {{ include "starlake.postgresql.passwordKey" . }}: {{ .Values.postgresql.credentials.password | quote }} +{{- end }} + +{{- if .Values.airflow.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "starlake.fullname" . }}-airflow + labels: + {{- include "starlake.labels" . | nindent 4 }} +type: Opaque +stringData: + admin-password: {{ .Values.airflow.admin.password | quote }} + secret-key: {{ .Values.airflow.secretKey | quote }} +{{- end }} + +{{- if .Values.ui.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "starlake.fullname" . }}-ui + labels: + {{- include "starlake.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- if .Values.ui.mail.enabled }} + mail-password: {{ .Values.ui.mail.password | quote }} + {{- end }} + docs-password: {{ .Values.ui.docs.password | default "s3cret.Paw" | quote }} +{{- end }} + +{{- if .Values.gizmo.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "starlake.fullname" . }}-gizmo + labels: + {{- include "starlake.labels" . | nindent 4 }} +type: Opaque +stringData: + api-key: {{ .Values.gizmo.apiKey | quote }} + sql-username: {{ .Values.gizmo.sql.username | quote }} + sql-password: {{ .Values.gizmo.sql.password | quote }} + jwt-secret: {{ .Values.gizmo.jwt.secretKey | quote }} +{{- end }} + +{{- /* MinIO section removed - SeaweedFS uses its own secret in seaweedfs/secret.yaml */}} diff --git a/helm/starlake/templates/serviceaccount.yaml b/helm/starlake/templates/serviceaccount.yaml new file mode 100644 index 0000000..caa088f --- /dev/null +++ b/helm/starlake/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "starlake.serviceAccountName" . }} + labels: + {{- include "starlake.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/ui/deployment.yaml b/helm/starlake/templates/ui/deployment.yaml new file mode 100644 index 0000000..e9186c9 --- /dev/null +++ b/helm/starlake/templates/ui/deployment.yaml @@ -0,0 +1,397 @@ +{{- if .Values.ui.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "starlake.fullname" . }}-ui + labels: + {{- include "starlake.componentLabels" (dict "component" "ui" "context" .) | nindent 4 }} +spec: + replicas: {{ .Values.ui.replicas }} + selector: + matchLabels: + {{- include "starlake.componentSelectorLabels" (dict "component" "ui" "context" .) | nindent 6 }} + template: + metadata: + labels: + {{- include "starlake.componentSelectorLabels" (dict "component" "ui" "context" .) | nindent 8 }} + spec: + serviceAccountName: {{ include "starlake.serviceAccountName" . }} + # fsGroup ensures shared volume files are accessible by all pods + securityContext: + fsGroup: {{ .Values.podSecurityContext.fsGroup }} + initContainers: + {{- include "starlake.waitForPostgresql" . | nindent 6 }} + {{- include "starlake.waitForSeaweedfs" . | nindent 6 }} + {{- if .Values.airflow.enabled }} + - name: wait-for-airflow + image: {{ .Values.initImages.busybox.repository }}:{{ .Values.initImages.busybox.tag }} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + until nc -z {{ include "starlake.fullname" . }}-airflow 8080; do + echo "Waiting for Airflow..." + sleep 2 + done + echo "Airflow is ready!" + {{- end }} + - name: install-starlake-airflow + image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag }}" + imagePullPolicy: {{ .Values.ui.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + echo "Installing starlake-airflow package (0.4.x for Airflow 2)..." + # IMPORTANT: Pin starlake-airflow~=0.4 for Airflow 2 (0.5+ requires Airflow 3) + python3 -m pip install --break-system-packages --no-cache-dir "starlake-airflow>=0.4,<0.5" docker + airflow db init || true + airflow db migrate || true + echo "Installation complete!" + volumeMounts: + - name: tmp + mountPath: /tmp + {{- if not .Values.seaweedfs.enabled }} + # Fix permissions on shared volumes - runs as root before main container + # Skip when SeaweedFS is enabled (data stored in S3, no local PVC) + - name: fix-permissions + image: {{ .Values.initImages.busybox.repository }}:{{ .Values.initImages.busybox.tag }} + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + runAsNonRoot: false + command: + - sh + - -c + - | + echo "Fixing permissions on /projects for group {{ .Values.podSecurityContext.fsGroup }}..." + # Change group ownership to fsGroup + chgrp -R {{ .Values.podSecurityContext.fsGroup }} /projects 2>/dev/null || true + # Set group read/write for all files + chmod -R g+rwX /projects 2>/dev/null || true + # DuckDB stored_secrets: remove group write (contains unencrypted credentials) + find /projects -type d -name "stored_secrets" -exec chmod -R g-w {} \; 2>/dev/null || true + echo "Permissions fixed!" + volumeMounts: + - name: projects + mountPath: /projects + {{- end }} + containers: + - name: ui + image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag }}" + imagePullPolicy: {{ .Values.ui.image.pullPolicy }} + # Note: UI image requires root (runs chmod in /app/run-api.sh) + # TODO: Update upstream image to support non-root execution + command: + - /bin/bash + - -c + - | + # Set umask to allow group write on new files + umask 002 + {{- if .Values.seaweedfs.enabled }} + echo "Configuring DuckDB S3 secret for SeaweedFS..." + # Install DuckDB CLI to create persistent secret + if ! command -v duckdb &> /dev/null; then + echo "Installing DuckDB CLI..." + ARCH=$(uname -m) + case "$ARCH" in + aarch64|arm64) + DUCKDB_URL="https://github.com/duckdb/duckdb/releases/download/v{{ .Values.seaweedfs.duckdb.version }}/duckdb_cli-linux-aarch64.zip" + ;; + x86_64|amd64) + DUCKDB_URL="https://github.com/duckdb/duckdb/releases/download/v{{ .Values.seaweedfs.duckdb.version }}/duckdb_cli-linux-amd64.zip" + ;; + *) + echo "ERROR: Unsupported architecture: $ARCH" + exit 1 + ;; + esac + + # Try wget first, then curl + if command -v wget &> /dev/null; then + wget -q "$DUCKDB_URL" -O /tmp/duckdb.zip + elif command -v curl &> /dev/null; then + curl -sL "$DUCKDB_URL" -o /tmp/duckdb.zip + else + echo "ERROR: Neither wget nor curl available" + exit 1 + fi + + # Extract and install (try unzip, fallback to python3 zipfile, fallback to jar) + if command -v unzip &> /dev/null; then + unzip -q -o /tmp/duckdb.zip -d /tmp + elif command -v python3 &> /dev/null; then + python3 -c "import zipfile; zipfile.ZipFile('/tmp/duckdb.zip').extractall('/tmp')" + elif command -v jar &> /dev/null; then + cd /tmp && jar xf duckdb.zip + else + echo "WARNING: No zip extraction tool available, skipping DuckDB CLI install" + rm -f /tmp/duckdb.zip + fi + chmod +x /tmp/duckdb + # Try to install in /usr/local/bin, fallback to ~/bin if no permission + if mv /tmp/duckdb /usr/local/bin/duckdb 2>/dev/null; then + echo "DuckDB CLI installed at /usr/local/bin/duckdb" + else + mkdir -p ~/bin + mv /tmp/duckdb ~/bin/duckdb + export PATH="$HOME/bin:$PATH" + echo "DuckDB CLI installed at ~/bin/duckdb" + fi + rm -f /tmp/duckdb.zip + fi + + # Create persistent S3 secret using DuckDB CLI + echo "CREATE OR REPLACE PERSISTENT SECRET (TYPE S3, KEY_ID '{{ .Values.seaweedfs.s3.accessKey }}', SECRET '{{ .Values.seaweedfs.s3.secretKey }}', ENDPOINT '{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}', URL_STYLE 'path', USE_SSL false, REGION 'us-east-1'); SELECT 'DuckDB S3 secret configured' AS status;" | duckdb :memory: || echo "WARNING: Failed to create DuckDB S3 secret" + {{- end }} + echo "Starting Starlake UI..." + exec /app/run-api.sh + ports: + - name: http + containerPort: 9900 + protocol: TCP + env: + - name: SL_HOME + value: /app/starlake + # Storage Mode - ALWAYS local filesystem, even with SeaweedFS enabled + # SeaweedFS is provisioned as infrastructure but not used by default + # Users create S3 projects manually via UI when needed + - name: SL_FS + value: "file://" + - name: SL_ROOT + value: "/projects" + {{- if .Values.seaweedfs.enabled }} + # S3 credentials available for manual project creation in UI + # These are NOT used by default - only when user creates S3-backed projects + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.seaweedfs.s3.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.seaweedfs.s3.secretKey | quote }} + - name: AWS_S3_ENDPOINT + value: "http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }}" + - name: AWS_REGION + value: "us-east-1" + - name: HADOOP_CONF_DIR + value: "/etc/hadoop/conf" + # SL_STORAGE_CONF passes Hadoop S3A config directly to HdfsStorageHandler + # Required because the API server classpath does not include HADOOP_CONF_DIR, + # so core-site.xml is not loaded by Hadoop Configuration() + - name: SL_STORAGE_CONF + # S3A config for SeaweedFS compatibility: + # - signing-algorithm=S3SignerType: Use S3 V2 signing to avoid AWS V4 chunked encoding + # (Hadoop 3.3.4 doesn't support payload.signing.enabled; V4 chunked encoding causes + # SeaweedFS to store 86-byte chunk terminators as directory marker content) + # - directory.marker.retention=keep: SeaweedFS manages directories natively + # - multiobjectdelete.enable=false: Unreliable on non-AWS S3 backends + value: "fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem,fs.s3a.endpoint=http://{{ include "starlake.fullname" . }}-seaweedfs:{{ .Values.seaweedfs.service.s3Port }},fs.s3a.access.key={{ .Values.seaweedfs.s3.accessKey }},fs.s3a.secret.key={{ .Values.seaweedfs.s3.secretKey }},fs.s3a.path.style.access=true,fs.s3a.connection.ssl.enabled=false,fs.s3a.signing-algorithm=S3SignerType,fs.s3a.directory.marker.retention=keep,fs.s3a.multiobjectdelete.enable=false,fs.s3a.change.detection.mode=warn,fs.s3a.change.detection.version.required=false" + {{- end }} + - name: SL_ENV + value: "" + - name: SL_USE_LOCAL_FILE_SYSTEM + value: "false" + - name: SL_API_GIT_COMMAND_ROOT + value: /git + - name: SL_API_SECURE + value: "false" + - name: SL_API + value: "true" + - name: SL_API_SESSION_AS_HEADER + value: "true" + - name: SL_API_HTTP_FRONT_URL + {{- if .Values.ui.frontendUrl }} + value: {{ .Values.ui.frontendUrl | quote }} + {{- else }} + value: {{ include "starlake.frontendUrl" . | quote }} + {{- end }} + - name: SL_API_HTTP_INTERFACE + value: 0.0.0.0 + - name: SL_API_HTTP_PORT + value: "9900" + - name: SL_LOG_LEVEL + value: {{ .Values.ui.logLevel | quote }} + - name: SL_API_JDBC_DRIVER + value: org.postgresql.Driver + - name: SL_API_JDBC_USER + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.usernameKey" . }} + - name: SL_API_JDBC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: SL_API_JDBC_URL + value: {{ include "starlake.postgresql.jdbcUrl" . }}&password=$(SL_API_JDBC_PASSWORD) + - name: SL_API_DOMAIN + value: {{ include "starlake.domain" . | quote }} + # Project root - ALWAYS local filesystem + # Users can manually configure S3 storage per-project via UI + - name: SL_API_PROJECT_ROOT + value: "/projects" + {{- if .Values.airflow.enabled }} + - name: SL_API_ORCHESTRATOR_PRIVATE_URL + value: http://{{ include "starlake.fullname" . }}-airflow:8080/airflow/ + - name: LOAD_DAG_REF + value: airflow_load_shell + - name: TRANSFORM_DAG_REF + value: airflow_transform_shell + - name: SL_API_AIRFLOW_VERSION + value: {{ .Values.airflow.version | quote }} + - name: SL_API_AIRFLOW_USERNAME + value: {{ .Values.airflow.admin.username | quote }} + - name: SL_API_AIRFLOW_PASSWORD + value: {{ .Values.airflow.admin.password | quote }} + {{- if .Values.airflow.jobRunner.enabled }} + # When Job Runner is enabled, dag-generate should produce DAGs that use starlake-k8s + - name: SL_STARLAKE_PATH + value: "starlake-k8s" + {{- end }} + {{- end }} + - name: SL_API_AI_URL + value: http://{{ include "starlake.fullname" . }}-agent:8000 + - name: SL_AI_APPLICATION_KEY + value: {{ .Values.agent.applicationKey | quote }} + - name: SL_API_GIZMO_ON_DEMAND_URL + value: http://{{ include "starlake.fullname" . }}-gizmo:10900 + - name: SL_GIZMO_API_KEY + value: {{ .Values.gizmo.apiKey | quote }} + - name: ENVIRONMENT + value: local + - name: FILESTORE_MNT_DIR + value: /projects + - name: EXTERNAL_PROJECTS_MNT_DIR + value: /external_projects + - name: POSTGRES_HOST + value: {{ include "starlake.postgresql.host" . | quote }} + - name: POSTGRES_DB + value: {{ include "starlake.postgresql.starlakeDatabase" . | quote }} + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.usernameKey" . }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.postgresql.secretName" . }} + key: {{ include "starlake.postgresql.passwordKey" . }} + - name: SL_UI_DEMO + # Enable demo mode if either ui.demo or demo.enabled is true + # When true, UI will auto-initialize demo projects with DuckLake setup + value: {{ or .Values.ui.demo .Values.demo.enabled | quote }} + - name: SL_API_APP_TYPE + value: {{ .Values.ui.appType | quote }} + {{- if .Values.ui.mail.enabled }} + - name: SL_API_MAIL_HOST + value: {{ .Values.ui.mail.host | quote }} + - name: SL_API_MAIL_PORT + value: {{ .Values.ui.mail.port | quote }} + - name: SL_API_MAIL_USER + value: {{ .Values.ui.mail.user | quote }} + - name: SL_API_MAIL_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.fullname" . }}-ui + key: mail-password + - name: SL_API_MAIL_FROM + value: {{ .Values.ui.mail.from | quote }} + {{- end }} + - name: SL_API_MAX_USER_SPACE_MB + value: {{ .Values.ui.fileUpload.maxUserSpaceMB | quote }} + - name: SL_API_FILE_UPLOAD_MAX_CONTENT_LENGTH + value: {{ .Values.ui.fileUpload.maxContentLength | quote }} + - name: SL_API_UI_FOLDER + value: /app/ui + - name: SL_API_DOCS_USERS + value: {{ .Values.ui.docs.user | default "starlake" | quote }} + - name: SL_API_DOCS_USER + value: {{ .Values.ui.docs.user | default "starlake" | quote }} + - name: SL_API_DOCS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "starlake.fullname" . }}-ui + key: docs-password + {{- with .Values.ui.env }} + {{- toYaml . | nindent 8 }} + {{- end }} + # Startup probe - allows slow startup (JVM + init can take 1-2 min) + # Matches docker-compose: interval=5s, retries=60 = 5 min max + startupProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + # Liveness probe - after startup succeeds + livenessProbe: + httpGet: + path: /api/v1/health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + # Readiness probe + readinessProbe: + httpGet: + path: /api/v1/health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + {{- toYaml .Values.ui.resources | nindent 10 }} + volumeMounts: + - name: starlake-cli + mountPath: /usr/local/bin/starlake + subPath: starlake.sh + - name: projects + mountPath: /projects + {{- if .Values.persistence.externalProjects.enabled }} + - name: external-projects + mountPath: /external_projects + {{- end }} + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + mountPath: /etc/hadoop/conf + {{- end }} + - name: tmp + mountPath: /tmp + volumes: + - name: starlake-cli + configMap: + name: {{ include "starlake.fullname" . }}-scripts + defaultMode: 0755 + - name: projects + # Always use PVC for local filesystem storage + # SeaweedFS provides S3 infrastructure but doesn't change deployment mode + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-projects + {{- if .Values.persistence.externalProjects.enabled }} + - name: external-projects + persistentVolumeClaim: + claimName: {{ include "starlake.fullname" . }}-external-projects + {{- end }} + - name: tmp + emptyDir: {} + {{- if .Values.seaweedfs.enabled }} + - name: hadoop-config + configMap: + name: {{ include "starlake.fullname" . }}-hadoop-config + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/starlake/templates/ui/service.yaml b/helm/starlake/templates/ui/service.yaml new file mode 100644 index 0000000..25102c1 --- /dev/null +++ b/helm/starlake/templates/ui/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.ui.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "starlake.fullname" . }}-ui + labels: + {{- include "starlake.componentLabels" (dict "component" "ui" "context" .) | nindent 4 }} +spec: + type: {{ .Values.ui.service.type }} + ports: + - name: http + port: {{ .Values.ui.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "starlake.componentSelectorLabels" (dict "component" "ui" "context" .) | nindent 4 }} +{{- end }} diff --git a/helm/starlake/values-development.yaml b/helm/starlake/values-development.yaml new file mode 100644 index 0000000..f9d134d --- /dev/null +++ b/helm/starlake/values-development.yaml @@ -0,0 +1,81 @@ +# Example: Development/Local Deployment +# Usage: helm install starlake ./helm/starlake -f values-development.yaml + +# Use internal PostgreSQL (simpler for dev) +postgresql: + external: + enabled: false + internal: + enabled: true + persistence: + enabled: true + size: 10Gi # Smaller for dev + storageClass: "standard" + credentials: + username: dbuser + password: dbuser123 + +# Smaller resources for development +ui: + replicas: 1 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + +airflow: + webserver: + replicas: 1 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + scheduler: + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + admin: + username: airflow + password: airflow + +agent: + replicas: 1 + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + +# Smaller projects volume +persistence: + projects: + enabled: true + size: 20Gi + storageClass: "nfs-client" # Adjust for your local setup + +# Use LoadBalancer or port-forward +proxy: + service: + type: NodePort # Or LoadBalancer if available + +ingress: + enabled: false + +# Disable optional services to save resources +gizmo: + enabled: false + +minio: + enabled: false diff --git a/helm/starlake/values-external-postgres.yaml b/helm/starlake/values-external-postgres.yaml new file mode 100644 index 0000000..52ab34b --- /dev/null +++ b/helm/starlake/values-external-postgres.yaml @@ -0,0 +1,64 @@ +# Example: Using External PostgreSQL (RDS, CloudSQL, etc.) +# Usage: helm install starlake ./helm/starlake -f values-external-postgres.yaml + +postgresql: + # Use external PostgreSQL + external: + enabled: true + host: "my-postgres.example.com" # CHANGE THIS + port: 5432 + starlakeDatabase: starlake + airflowDatabase: airflow + + # Disable internal PostgreSQL + internal: + enabled: false + + # Credentials (use existing secret in production!) + credentials: + username: starlake_user + password: "ChangeThisPassword123!" + # For production, use: + # existingSecret: "my-postgres-secret" + # usernameKey: "username" + # passwordKey: "password" + +# Adjust storage class for your cloud provider +persistence: + projects: + enabled: true + storageClass: "" # Leave empty to use default, or specify: "efs-sc", "filestore-csi", "azurefile" + size: 100Gi + +# Disable proxy LoadBalancer if using Ingress +proxy: + service: + type: ClusterIP + +# Enable Ingress +ingress: + enabled: true + className: nginx + host: starlake.mycompany.com + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + tls: + enabled: true + secretName: starlake-tls + +# Production settings +ui: + replicas: 2 + resources: + requests: + memory: "2Gi" + cpu: "1000m" + limits: + memory: "8Gi" + cpu: "4000m" + +airflow: + webserver: + replicas: 2 + admin: + password: "ChangeThisAirflowPassword!" diff --git a/helm/starlake/values.yaml b/helm/starlake/values.yaml new file mode 100644 index 0000000..aa5b2ef --- /dev/null +++ b/helm/starlake/values.yaml @@ -0,0 +1,472 @@ +# Starlake Helm Chart Values +# This is the default configuration for the Starlake Data Stack with Airflow + +# Global settings +global: + # Storage class to use for all PVCs (override per component if needed) + storageClass: "" + +# Utility images used by init containers and jobs +initImages: + busybox: + repository: busybox + tag: "1.36" # CVE-2022-28391 fix (upgrade from 1.35) + awsCli: + repository: amazon/aws-cli + tag: "2.15.0" + +# PostgreSQL Database Configuration +postgresql: + # Use external PostgreSQL database (managed service like RDS, CloudSQL, etc.) + external: + enabled: false + host: "" # e.g., "my-postgres.abc123.us-east-1.rds.amazonaws.com" + port: 5432 + # Databases names (will be created if using internal postgres) + starlakeDatabase: starlake + airflowDatabase: airflow + + # Deploy PostgreSQL as a StatefulSet in the cluster + internal: + enabled: true + image: + repository: postgres + # Security fix: Pin PostgreSQL to patch version for security tracking + tag: "17.2" + pullPolicy: IfNotPresent + + # Resources + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + + # Persistence + persistence: + enabled: true + storageClass: "" # Use global.storageClass if empty + size: 50Gi + # existingClaim: "" # Use existing PVC instead of creating new one + + # PostgreSQL configuration + config: + maxConnections: 200 + sharedBuffers: "256MB" + + # Credentials (used for both external and internal) + # WARNING: Override these in production with secure values! + credentials: + username: dbuser + password: dbuser123 + # Existing secret to use instead of creating one + # existingSecret: "" + # Keys in the existing secret + # usernameKey: "postgres-user" + # passwordKey: "postgres-password" + +# Starlake UI Configuration +ui: + enabled: true + + image: + repository: starlakeai/starlake-1.5-ui + tag: "1.5" + pullPolicy: Always + + replicas: 1 + + # Application type: "ducklake" for DuckDB, "web" for cloud data warehouses + appType: ducklake + + # Resources + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "2000m" + + # Service configuration + # UI is now the main entry point (proxy merged into UI in docker-compose) + service: + type: ClusterIP # Use Ingress for external access (safer than LoadBalancer) + port: 80 # External port (maps to internal 9900) + targetPort: 9900 # Internal container port + + # Environment variables (additional custom env vars) + env: [] + # - name: CUSTOM_VAR + # value: "custom_value" + + # Demo mode + demo: false + + # Log level: debug, info, warn, error + logLevel: info + + # Mail configuration (optional) + mail: + enabled: false + host: smtp.sendgrid.net + port: 587 + user: apikey + password: "" # Stored in Secret, not visible in deployment + from: contact@starlake.ai + + # Documentation access credentials + docs: + user: starlake + password: "" # Set a secure password, stored in Secret + + # File upload configuration + fileUpload: + maxContentLength: 21474836480 # 20 GB + maxUserSpaceMB: 0 # 0 = unlimited + + # Security context for UI container (runs as non-root) + securityContext: + runAsUser: 1000 + runAsNonRoot: true + + # Frontend URL override for port-forward scenarios + # If empty, uses auto-detected URL based on ingress/service config + # For local testing with port-forward on port 8088, set to "http://localhost:8088" + frontendUrl: "" + +# Airflow Configuration +airflow: + enabled: true + + # Airflow version (2 or 3) + version: 2 + + image: + # Using official Apache Airflow image + # For local custom image built from: + # - Dockerfile_airflow (Docker mode): repository: starlake-airflow, tag: local + # - Dockerfile_airflow_k8s (K8s Job mode): repository: starlake-airflow-k8s, tag: local + # When using custom image, set installPythonPackages: false and use pullPolicy: Never + repository: apache/airflow + tag: "2.10.4-python3.11" + pullPolicy: IfNotPresent + + # Install Python packages at startup (starlake-airflow, providers, etc.) + # Set to false if using a custom image that already has packages installed + # (e.g., image built from Dockerfile_airflow or Dockerfile_airflow_k8s) + installPythonPackages: true + + # Secret key for Airflow webserver session signing + # IMPORTANT: Must be the same across all Airflow components (webserver, scheduler, workers) + # Generate a new one for production: python -c "import secrets; print(secrets.token_hex(32))" + secretKey: "starlake-airflow-secret-key-change-in-production" + + # Admin user credentials + admin: + username: airflow + password: airflow + firstname: Airflow + lastname: Admin + email: admin@example.com + + # Webserver configuration + # Note: This pod runs webserver + scheduler + pip install, requires significant memory + webserver: + replicas: 1 + resources: + requests: + memory: "4Gi" + cpu: "1000m" + limits: + memory: "16Gi" + cpu: "2000m" + service: + type: ClusterIP + port: 8080 + + # Scheduler configuration (combined with webserver in Airflow 2) + scheduler: + resources: + requests: + memory: "4Gi" + cpu: "500m" + limits: + memory: "8Gi" + cpu: "2000m" + + # Executor type + executor: LocalExecutor + + # Log level + logLevel: INFO + + # DAG processing interval + dagDirListInterval: 30 + dagMinFileProcessInterval: 5 + + # Base URL for Airflow webserver (used for redirects) + # If empty, uses frontendUrl/airflow (e.g., http://localhost:80/airflow) + # For local testing with port-forward, set to "http://localhost:8080/airflow" + baseUrl: "" + + # Logs persistence + logs: + persistence: + enabled: true + storageClass: "" + size: 20Gi + + # Job Runner - Execute Starlake tasks as Kubernetes Jobs + # This offloads heavy processing from the Airflow pod to dedicated Jobs + jobRunner: + enabled: false # Set to true to enable K8s Job execution mode + + # Use built-in wrapper from Dockerfile_airflow_k8s (recommended) + # When true: expects the Airflow image to have starlake-k8s built-in + # When false: mounts starlake-k8s wrapper script from ConfigMap + useBuiltinWrapper: true + + # Starlake image for Jobs (must have CLI at /app/starlake/starlake) + # NOTE: --scheduledDate option requires starlake >= 1.5.3-SNAPSHOT + image: + repository: starlakeai/starlake + tag: "1.5.3-SNAPSHOT" + pullPolicy: IfNotPresent + + # Resources per Job (each Starlake task gets its own pod) + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + + # Cleanup: delete Jobs after completion + # Increased to 300s to ensure exit code capture before TTL cleanup + ttlSecondsAfterFinished: 300 # 5 minutes after completion + + # Backoff limit for failed jobs + backoffLimit: 0 # No retries (Airflow handles retries) + +# Starlake Agent (AI Assistant) +agent: + enabled: true + + image: + repository: starlakeai/starlake-1.5-ask + tag: "0.1" + pullPolicy: Always + + replicas: 1 + + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + + service: + type: ClusterIP + port: 8000 + + # Application key for API authentication + # SECURITY: Change this value in production! + applicationKey: "change-me-in-production" + +# Gizmo (Optional - SQL on-demand service) +gizmo: + enabled: false + + image: + repository: starlakeai/gizmo-on-demand + tag: snapshot-slim + pullPolicy: Always + + replicas: 1 + + # Expose SQL ports 11900-12000 on host network + # Required for external clients (DBeaver, etc.) to connect to Gizmo SQL instances + # Note: With hostNetwork=true, only 1 replica per node is allowed (port conflict) + # For production multi-replica setup, use Gateway API instead + hostNetwork: true + + # Node selector for Gizmo pod (optional) + # In K3d multi-node: use node-role.kubernetes.io/master: "true" to schedule on server node + # This ensures ports are accessible via K3d port mapping + nodeSelector: {} + # nodeSelector: + # node-role.kubernetes.io/master: "true" + + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + + service: + type: ClusterIP + port: 10900 + + # Port range for Gizmo processes + minPort: 11900 + maxPort: 12000 + maxProcesses: 10 + + # Credentials + apiKey: a_secret_api_key + sql: + username: gizmosql_user + password: gizmosql_password + jwt: + secretKey: a_very_secret_key + +# SeaweedFS (Optional - S3-compatible object storage) +# Recommended over MinIO (MinIO in maintenance mode since Dec 2025) +seaweedfs: + enabled: false + + image: + repository: chrislusf/seaweedfs + tag: "3.80" + pullPolicy: IfNotPresent + + # DuckDB CLI version (installed at UI startup for S3 secret creation) + duckdb: + version: "1.1.3" + + # Volume size limit (MB) - max size per volume file + volumeSizeLimitMB: 1000 + + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1000m" + + service: + type: ClusterIP + s3Port: 8333 # S3 API (main endpoint for apps) + masterPort: 9333 # Master coordination + filerPort: 8888 # Filer HTTP API + + persistence: + enabled: true + storageClass: "" + accessMode: ReadWriteOnce + size: 100Gi + + # S3 API credentials and bucket + s3: + accessKey: seaweedfs + secretKey: seaweedfs123 + bucket: starlake # Bucket name for SL_ROOT (created on first PUT) + + # Note: SeaweedFS creates buckets automatically on first access + + nodeSelector: {} + tolerations: [] + +# Persistence for shared volumes +persistence: + # Projects directory (shared between UI, Airflow, etc.) + projects: + enabled: true + storageClass: "" # Must support ReadWriteMany (e.g., NFS, EFS, Azure Files) + size: 100Gi + # existingClaim: "" # Use existing PVC + + # External projects directory (optional) + externalProjects: + enabled: false + storageClass: "" + size: 50Gi + +# Ingress configuration (alternative to proxy service) +ingress: + enabled: false + className: nginx + annotations: {} + # cert-manager.io/cluster-issuer: letsencrypt-prod + # nginx.ingress.kubernetes.io/ssl-redirect: "true" + + host: starlake.example.com + + tls: + enabled: false + secretName: starlake-tls + + # Path-based routing (similar to Docker Compose proxy) + paths: + ui: / + airflow: /airflow + agent: /agent + gizmo: /gizmo + +# Demo Projects Initialization +# Demo projects (starbake, tpch001) are ALWAYS stored on local PVC (/projects) +# regardless of storage mode (local, S3, SeaweedFS, etc.) +# This provides working examples out-of-the-box in all deployment modes. +# For production S3/SeaweedFS projects, users create them manually via UI. +demo: + enabled: true # Enable demo projects for immediate working examples + +# Service Account +serviceAccount: + create: true + annotations: {} + name: "" + # Annotations for cloud IAM (e.g., AWS IRSA, GCP Workload Identity) + # annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT:role/ROLE_NAME + +# Pod Security Context +podSecurityContext: + fsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + +# Security Context +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + +# Node selector +nodeSelector: {} + +# Tolerations +tolerations: [] + +# Affinity +affinity: {} + +# Network Policy - enabled by default for security +# Restricts pod communication to only what's necessary +networkPolicy: + enabled: true + policyTypes: + - Ingress + - Egress + +# Security settings +security: + # Enable credential validation to block deployment with insecure defaults + # When true, deployment will fail if default credentials are used: + # - postgresql.credentials.password: "dbuser123" + # - airflow.admin.password: "airflow" + # - airflow.secretKey: default value + # - gizmo.apiKey: "a_secret_api_key" + # RECOMMENDED: Set to true for production deployments + validateCredentials: false diff --git a/helm/test-helm-chart.sh b/helm/test-helm-chart.sh new file mode 100755 index 0000000..c6b99c7 --- /dev/null +++ b/helm/test-helm-chart.sh @@ -0,0 +1,1270 @@ +#!/bin/bash +# Script de test automatisé du Helm Chart Starlake avec K3s +# +# Ce script supporte deux modes de cluster: +# - Single-node (défaut): Utilise local-path storage (RWO) +# - Multi-node: Cluster avec agents, local-path storage (avec limitations) +# +# Usage: +# ./test-helm-chart.sh # Dev single-node (credentials par défaut) +# ./test-helm-chart.sh --production # Production single-node (credentials sécurisés) +# ./test-helm-chart.sh --multi-node # Multi-nœuds local-path (1 server + 3 agents) +# ./test-helm-chart.sh --multi-node --seaweedfs # Multi-nœuds avec S3 (SeaweedFS) +# ./test-helm-chart.sh --production --multi-node --seaweedfs # Full production-like +# ./test-helm-chart.sh --security-only # Validation sécurité seulement (pas de cluster) +# +# Prérequis: +# - k3d (brew install k3d) +# - helm (brew install helm) +# - kubectl (inclus avec k3d) + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +CLUSTER_NAME="starlake-test" +NAMESPACE="starlake" +CHART_PATH="./starlake" +TIMEOUT="15m" + +# Mode de test (dev par défaut) +PRODUCTION_MODE=false +SECURITY_ONLY=false +MULTI_NODE=false +AGENT_COUNT=3 # Nombre d'agents pour multi-node +SEAWEEDFS_ENABLED=false # Object storage S3 + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --production|-p) + PRODUCTION_MODE=true + shift + ;; + --security-only|-s) + SECURITY_ONLY=true + shift + ;; + --multi-node|-m) + MULTI_NODE=true + shift + ;; + --agents) + AGENT_COUNT=$2 + shift 2 + ;; + --seaweedfs) + SEAWEEDFS_ENABLED=true + shift + ;; + *) + echo "Usage: $0 [--production|-p] [--security-only|-s] [--multi-node|-m] [--agents N] [--seaweedfs]" + exit 1 + ;; + esac +done + +# Génération de credentials sécurisés pour le mode production +generate_secure_credentials() { + # Générer des mots de passe aléatoires + SECURE_PG_PASSWORD=$(openssl rand -base64 24 | tr -dc 'a-zA-Z0-9' | head -c 24) + SECURE_AIRFLOW_PASSWORD=$(openssl rand -base64 24 | tr -dc 'a-zA-Z0-9' | head -c 24) + SECURE_AIRFLOW_SECRET_KEY=$(openssl rand -hex 32) + SECURE_GIZMO_API_KEY=$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid 2>/dev/null || openssl rand -hex 16) + SECURE_AGENT_KEY=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 32) +} + +# Credentials à utiliser +if [ "$PRODUCTION_MODE" = true ]; then + generate_secure_credentials + PG_PASSWORD="$SECURE_PG_PASSWORD" + AIRFLOW_PASSWORD="$SECURE_AIRFLOW_PASSWORD" + AIRFLOW_SECRET_KEY="$SECURE_AIRFLOW_SECRET_KEY" + GIZMO_API_KEY="$SECURE_GIZMO_API_KEY" + AGENT_APPLICATION_KEY="$SECURE_AGENT_KEY" + VALIDATE_CREDENTIALS="true" +else + # Mode dev - credentials par défaut + PG_PASSWORD="dbuser123" + AIRFLOW_PASSWORD="airflow" + AIRFLOW_SECRET_KEY="starlake-airflow-secret-key-change-in-production" + GIZMO_API_KEY="a_secret_api_key" + AGENT_APPLICATION_KEY="change-me-in-production" + VALIDATE_CREDENTIALS="false" +fi + +# Timeouts configurables (en secondes) +CLUSTER_READY_TIMEOUT=120 # Attente cluster ready +HEADLAMP_READY_TIMEOUT=120 # Attente Headlamp ready +POD_READY_MAX_ATTEMPTS=240 # Max attempts pour pods (240 * 5s = 20 min) +HEALTH_CHECK_SLEEP=5 # Pause entre health checks +PORT_FORWARD_SLEEP=5 # Pause après port-forward + +# Fonction pour afficher les messages +log_info() { + echo -e "${BLUE}ℹ${NC} $1" +} + +log_success() { + echo -e "${GREEN}✓${NC} $1" +} + +log_error() { + echo -e "${RED}✗${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}⚠${NC} $1" +} + +# Fonction de nettoyage +cleanup() { + log_info "Nettoyage en cours..." + + # Supprimer le release Helm si existe + if helm list -n $NAMESPACE 2>/dev/null | grep -q starlake; then + helm uninstall starlake -n $NAMESPACE 2>/dev/null || true + fi + + # Supprimer le namespace + kubectl delete namespace $NAMESPACE --wait=false 2>/dev/null || true + + # Supprimer le cluster K3s + if k3d cluster list 2>/dev/null | grep -q $CLUSTER_NAME; then + k3d cluster delete $CLUSTER_NAME 2>/dev/null || true + fi + + # Tuer les processus de port-forward + pkill -f "kubectl port-forward" 2>/dev/null || true +} + +# Trap pour nettoyer en cas d'erreur +trap cleanup EXIT + +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🧪 Test Automatisé du Helm Chart Starlake" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +# 0. Vérifier les prérequis +log_info "Vérification des prérequis..." + +if ! command -v k3d &> /dev/null; then + log_error "k3d n'est pas installé. Installez-le avec: brew install k3d" + exit 1 +fi +log_success "k3d est installé" + +if ! command -v helm &> /dev/null; then + log_error "Helm n'est pas installé. Installez-le avec: brew install helm" + exit 1 +fi +log_success "Helm est installé" + +if ! command -v kubectl &> /dev/null; then + log_error "kubectl n'est pas installé" + exit 1 +fi +log_success "kubectl est installé" + +if [ ! -d "$CHART_PATH" ]; then + log_error "Chart directory not found: $CHART_PATH" + exit 1 +fi +log_success "Chart trouvé: $CHART_PATH" + +echo "" + +# Afficher le mode de test +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +if [ "$MULTI_NODE" = true ]; then + echo " 🌐 MODE MULTI-NODE - $AGENT_COUNT agents + local-path storage" +else + echo " 📦 MODE SINGLE-NODE - local-path storage" +fi +if [ "$PRODUCTION_MODE" = true ]; then + echo " 🔒 PRODUCTION - Credentials sécurisés" +else + echo " 🔧 DEV - Credentials par défaut" +fi +if [ "$SEAWEEDFS_ENABLED" = true ]; then + echo " 📦 SEAWEEDFS - Object storage S3 activé" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +if [ "$PRODUCTION_MODE" = true ]; then + log_info "PostgreSQL password: ${PG_PASSWORD:0:8}..." + log_info "Airflow password: ${AIRFLOW_PASSWORD:0:8}..." + log_info "Airflow secret key: ${AIRFLOW_SECRET_KEY:0:16}..." + log_info "Gizmo API key: ${GIZMO_API_KEY:0:8}..." + log_info "Agent application key: ${AGENT_APPLICATION_KEY:0:8}..." + log_info "security.validateCredentials: true" + echo "" +else + log_info "Mode DEV - credentials par défaut (airflow/airflow, dbuser123)" +fi + +# 0.5. Test de validation sécurité (helm template) +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🔐 Test de Validation Sécurité" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +# Test 1: Validation doit ÉCHOUER avec credentials par défaut + validateCredentials=true +log_info "Test 1: Validation bloque les credentials par défaut..." +SECURITY_TEST_OUTPUT=$(helm template test-security $CHART_PATH \ + --set security.validateCredentials=true \ + --set postgresql.credentials.password=dbuser123 \ + --set airflow.admin.password=airflow \ + --set airflow.secretKey="starlake-airflow-secret-key-change-in-production" \ + 2>&1) && SECURITY_TEST_RESULT=$? || SECURITY_TEST_RESULT=$? + +if [ $SECURITY_TEST_RESULT -ne 0 ] && grep -q "SECURITY ERROR" <<< "$SECURITY_TEST_OUTPUT"; then + log_success " ✓ Validation bloque correctement les credentials par défaut" +else + log_error " ✗ Validation devrait bloquer les credentials par défaut!" + head -5 <<< "$SECURITY_TEST_OUTPUT" + if [ "$SECURITY_ONLY" = true ]; then exit 1; fi +fi + +# Test 2: Validation doit RÉUSSIR avec credentials sécurisés +log_info "Test 2: Validation accepte les credentials sécurisés..." +SECURITY_TEST_OUTPUT=$(helm template test-security $CHART_PATH \ + --set security.validateCredentials=true \ + --set postgresql.credentials.password=SecurePassword123 \ + --set airflow.admin.password=SecureAirflowPass456 \ + --set airflow.secretKey="$(openssl rand -hex 32)" \ + --set gizmo.enabled=false \ + --set agent.applicationKey=SecureAgentKey123 \ + 2>&1) && SECURITY_TEST_RESULT=$? || SECURITY_TEST_RESULT=$? + +if [ $SECURITY_TEST_RESULT -eq 0 ]; then + log_success " ✓ Validation accepte les credentials sécurisés" +else + log_error " ✗ Validation devrait accepter les credentials sécurisés!" + head -10 <<< "$SECURITY_TEST_OUTPUT" + if [ "$SECURITY_ONLY" = true ]; then exit 1; fi +fi + +# Test 3: Validation PostgreSQL password spécifique +log_info "Test 3: Validation bloque postgresql.credentials.password=dbuser123..." +SECURITY_TEST_OUTPUT=$(helm template test-security $CHART_PATH \ + --set security.validateCredentials=true \ + --set postgresql.credentials.password=dbuser123 \ + --set airflow.admin.password=SecurePass \ + --set airflow.secretKey="$(openssl rand -hex 32)" \ + 2>&1) && SECURITY_TEST_RESULT=$? || SECURITY_TEST_RESULT=$? + +if [ $SECURITY_TEST_RESULT -ne 0 ] && grep -q "postgresql.credentials.password" <<< "$SECURITY_TEST_OUTPUT"; then + log_success " ✓ Validation bloque postgresql password par défaut" +else + log_error " ✗ Validation devrait bloquer postgresql password par défaut!" +fi + +# Test 4: Vérifier que les Secrets sont créés correctement +log_info "Test 4: Vérification création des Secrets Kubernetes..." +SECRETS_OUTPUT=$(helm template test-secrets $CHART_PATH \ + --set airflow.enabled=true \ + 2>&1) + +if grep -q "kind: Secret" <<< "$SECRETS_OUTPUT" && \ + grep -q "starlake-airflow" <<< "$SECRETS_OUTPUT" && \ + grep -q "admin-password" <<< "$SECRETS_OUTPUT" && \ + grep -q "secret-key" <<< "$SECRETS_OUTPUT"; then + log_success " ✓ Secret Airflow créé avec admin-password et secret-key" +else + log_error " ✗ Secret Airflow mal configuré!" +fi + +# Test 5: Vérifier que le deployment utilise secretKeyRef +log_info "Test 5: Vérification utilisation secretKeyRef dans deployment..." +if grep -q "secretKeyRef" <<< "$SECRETS_OUTPUT" && \ + grep -q "AIRFLOW_ADMIN_PASSWORD" <<< "$SECRETS_OUTPUT"; then + log_success " ✓ Deployment utilise secretKeyRef pour le password" +else + log_error " ✗ Deployment devrait utiliser secretKeyRef!" +fi + +echo "" +log_success "Tests de validation sécurité terminés!" +echo "" + +# Si --security-only, on s'arrête ici +if [ "$SECURITY_ONLY" = true ]; then + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " ✅ Tests de sécurité terminés (--security-only)" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + trap - EXIT # Désactiver cleanup + exit 0 +fi + +echo "" + +# 1. Créer le cluster K3s (supprimer l'ancien si existant) +if k3d cluster list 2>/dev/null | grep -q "$CLUSTER_NAME" 2>/dev/null; then + log_info "Cluster '$CLUSTER_NAME' existant détecté, suppression..." + k3d cluster delete "$CLUSTER_NAME" 2>/dev/null || true + sleep 2 +fi + +if [ "$MULTI_NODE" = true ]; then + log_info "Création du cluster K3s '$CLUSTER_NAME' (multi-node: 1 server + $AGENT_COUNT agents)..." + log_info " Note: Multi-node utilise local-path (RWX recommandé en production)" + log_info " Note: Ports 11900-11920 exposés pour Gizmo SQL (hostNetwork)" + k3d cluster create $CLUSTER_NAME \ + --servers 1 \ + --agents $AGENT_COUNT \ + --port "8080:80@loadbalancer" \ + --port "11900-11920:11900-11920@server:0" \ + --wait || { + log_error "Échec de la création du cluster" + exit 1 + } + log_success "Cluster K3s créé: 1 server + $AGENT_COUNT agents" +else + log_info "Création du cluster K3s '$CLUSTER_NAME' (single-node)..." + log_info " Note: Single-node requis car local-path ne supporte que RWO" + log_info " Note: Ports 11900-11920 exposés pour Gizmo SQL (hostNetwork)" + k3d cluster create $CLUSTER_NAME \ + --servers 1 \ + --agents 0 \ + --port "8080:80@loadbalancer" \ + --port "11900-11920:11900-11920@server:0" \ + --wait || { + log_error "Échec de la création du cluster" + exit 1 + } + log_success "Cluster K3s créé avec ports Gizmo SQL exposés (11900-11920)" +fi + +# Attendre que le cluster soit prêt +log_info "Attente que le cluster soit prêt..." +kubectl wait --for=condition=ready node --all --timeout=${CLUSTER_READY_TIMEOUT}s +log_success "Cluster prêt" + +# Afficher les nœuds +kubectl get nodes -o wide + +# 1.1 Configuration du Storage Class +# Note: Pour les tests locaux multi-node, on utilise local-path avec une limitation: +# - Les pods partageant un PVC doivent être sur le même nœud +# - En production, utiliser un storage RWX (EFS, Filestore, Azure Files, NFS externe) +if [ "$MULTI_NODE" = true ]; then + echo "" + log_info "Mode multi-node: utilisation de local-path storage" + log_warning " Note: Le PVC /projects sera sur un seul nœud (limitation local-path)" + log_warning " En production, utiliser un storage RWX (EFS, Filestore, Azure Files)" + echo "" + + # Forcer les pods avec PVC partagé sur le même nœud via nodeAffinity + # Le premier pod à démarrer (PostgreSQL) déterminera le nœud + STORAGE_CLASS="local-path" + + # Afficher les nœuds disponibles + log_info "Nœuds disponibles dans le cluster:" + kubectl get nodes -o wide + echo "" +else + STORAGE_CLASS="local-path" +fi + +echo "" + +# 1.5. Construire et importer les images locales dans k3d +log_info "Construction et import des images locales..." + +# Chemin vers le répertoire racine du projet +PROJECT_ROOT="$(cd .. && pwd)" + +# Variables pour les images locales +AIRFLOW_IMAGE_LOCAL="starlake-airflow:local" +PROJECTS_IMAGE_LOCAL="starlake-projects:local" +UI_IMAGE_LOCAL="starlake-ui:local" +AGENT_IMAGE_LOCAL="starlake-agent:local" + +USE_LOCAL_AIRFLOW_IMAGE="" +USE_LOCAL_PROJECTS_IMAGE="" +USE_LOCAL_UI_IMAGE="" +USE_LOCAL_AGENT_IMAGE="" + +# Fonction pour construire et importer une image +build_and_import_image() { + local dockerfile=$1 + local image_tag=$2 + local description=$3 + + log_info " Construction de l'image $description..." + if [ -f "$PROJECT_ROOT/$dockerfile" ]; then + docker build -t $image_tag -f "$PROJECT_ROOT/$dockerfile" "$PROJECT_ROOT" || { + log_warning "Construction de l'image $description a échoué" + return 1 + } + + if docker image inspect $image_tag > /dev/null 2>&1; then + log_info " Import de l'image $description dans k3d..." + # Capturer la sortie pour détecter les erreurs même si le code de retour est 0 + local import_output + import_output=$(k3d image import $image_tag -c $CLUSTER_NAME 2>&1) + local import_status=$? + + # Vérifier le code de retour ET la présence d'erreurs dans la sortie + if [ $import_status -ne 0 ] || grep -qi "error\|failed" <<< "$import_output"; then + log_warning "Import de l'image $description a échoué" + head -5 <<< "$import_output" + return 1 + fi + log_success "Image $description importée: $image_tag" + return 0 + fi + else + log_warning "$dockerfile non trouvé" + return 1 + fi +} + +# Fonction pour importer une image existante depuis le registre local Docker +import_existing_image() { + local image_name=$1 + local local_tag=$2 + local description=$3 + + log_info " Recherche de l'image $description dans Docker local..." + if docker image inspect $image_name > /dev/null 2>&1; then + # Tagger l'image avec un tag local + docker tag $image_name $local_tag + log_info " Import de l'image $description dans k3d..." + # Capturer la sortie pour détecter les erreurs même si le code de retour est 0 + local import_output + import_output=$(k3d image import $local_tag -c $CLUSTER_NAME 2>&1) + local import_status=$? + + # Vérifier le code de retour ET la présence d'erreurs dans la sortie + if [ $import_status -ne 0 ] || grep -qi "error\|failed" <<< "$import_output"; then + log_warning "Import de l'image $description a échoué" + head -5 <<< "$import_output" + return 1 + fi + log_success "Image $description importée: $local_tag" + return 0 + else + log_info " Image $description non trouvée localement" + return 1 + fi +} + +# 1. Construire l'image Airflow depuis Dockerfile_airflow_k8s (K8s Job execution mode) +# Note: On utilise Dockerfile_airflow_k8s qui crée des K8s Jobs pour chaque commande starlake +# au lieu de Dockerfile_airflow qui utilise docker exec (incompatible avec K8s) +# Cette image inclut kubectl et le wrapper starlake qui crée des Jobs K8s +if build_and_import_image "Dockerfile_airflow_k8s" "$AIRFLOW_IMAGE_LOCAL" "Airflow (K8s)"; then + USE_LOCAL_AIRFLOW_IMAGE="true" +fi + +# 2. Construire l'image Projects depuis Dockerfile_projects +if build_and_import_image "Dockerfile_projects" "$PROJECTS_IMAGE_LOCAL" "Projects"; then + USE_LOCAL_PROJECTS_IMAGE="true" +fi + +# 3. Importer l'image UI si elle existe localement (pas de Dockerfile, image pré-construite) +UI_IMAGES=( + "starlakeai/starlake-1.5-ui:1.5" + "starlakeai/starlake-1.5-ui:latest" + "starlakeai/starlake-ui:latest" +) +for ui_img in "${UI_IMAGES[@]}"; do + if import_existing_image "$ui_img" "$UI_IMAGE_LOCAL" "UI"; then + USE_LOCAL_UI_IMAGE="true" + break + fi +done + +# 4. Importer l'image Agent (Ask) si elle existe localement +AGENT_IMAGES=( + "starlakeai/starlake-1.5-ask:0.1" + "starlakeai/starlake-1.5-ask:1.5" + "starlakeai/starlake-1.5-ask:latest" + "starlakeai/starlake-ask:latest" +) +for agent_img in "${AGENT_IMAGES[@]}"; do + if import_existing_image "$agent_img" "$AGENT_IMAGE_LOCAL" "Agent"; then + USE_LOCAL_AGENT_IMAGE="true" + break + fi +done + +# 5. Importer l'image Gizmo si elle existe localement +GIZMO_IMAGE_LOCAL="starlake-gizmo:local" +USE_LOCAL_GIZMO_IMAGE="" +GIZMO_IMAGES=( + "starlakeai/gizmo-on-demand:snapshot-slim" + "starlakeai/gizmo-on-demand:latest" + "starlakeai/gizmo-on-demand:1.0" +) +for gizmo_img in "${GIZMO_IMAGES[@]}"; do + if import_existing_image "$gizmo_img" "$GIZMO_IMAGE_LOCAL" "Gizmo"; then + USE_LOCAL_GIZMO_IMAGE="true" + break + fi +done + +# Note: PostgreSQL (postgres:17) est une image publique légère, +# on la laisse être téléchargée directement par k3d depuis Docker Hub +# car l'import local peut échouer avec des erreurs de digest sur les images multi-arch +log_info " PostgreSQL: sera téléchargée depuis Docker Hub (image publique légère)" + +# Résumé des images locales +echo "" +log_info "Résumé des images locales:" +[ "$USE_LOCAL_AIRFLOW_IMAGE" = "true" ] && log_success " ✓ Airflow: $AIRFLOW_IMAGE_LOCAL" || log_info " ✗ Airflow: image par défaut" +[ "$USE_LOCAL_PROJECTS_IMAGE" = "true" ] && log_success " ✓ Projects: $PROJECTS_IMAGE_LOCAL" || log_info " ✗ Projects: image par défaut" +[ "$USE_LOCAL_UI_IMAGE" = "true" ] && log_success " ✓ UI: $UI_IMAGE_LOCAL" || log_info " ✗ UI: image par défaut" +[ "$USE_LOCAL_AGENT_IMAGE" = "true" ] && log_success " ✓ Agent: $AGENT_IMAGE_LOCAL" || log_info " ✗ Agent: image par défaut" +[ "$USE_LOCAL_GIZMO_IMAGE" = "true" ] && log_success " ✓ Gizmo: $GIZMO_IMAGE_LOCAL" || log_info " ✗ Gizmo: image par défaut" +log_info " ✗ PostgreSQL: postgres:17 (téléchargée depuis Docker Hub)" + +echo "" + +# 2. Installer Headlamp (interface web Kubernetes) +log_info "Installation de Headlamp (interface web Kubernetes)..." +helm repo add headlamp https://headlamp-k8s.github.io/headlamp/ 2>/dev/null || true +helm repo update headlamp 2>/dev/null || true + +helm install my-headlamp headlamp/headlamp \ + --namespace kube-system \ + --wait \ + --timeout 5m || { + log_warning "Installation de Headlamp a échoué (non bloquant)" + } + +# Créer un ServiceAccount avec les permissions admin pour Headlamp +kubectl apply -f - </dev/null || { + log_warning "Headlamp n'est pas encore prêt (non bloquant)" +} + +log_success "Headlamp installé" + +# Démarrer le port-forward Headlamp maintenant pour pouvoir suivre l'installation +log_info "Démarrage du port-forward Headlamp..." +kubectl port-forward -n kube-system svc/my-headlamp 9999:80 > /dev/null 2>&1 & +HEADLAMP_PF_PID=$! +sleep 2 + +# Afficher le token et les informations d'accès +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🖥️ Headlamp - Suivez l'installation en temps réel" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +log_success "Headlamp accessible: http://localhost:9999" +echo "" +log_info "Token d'authentification:" +HEADLAMP_TOKEN=$(kubectl create token headlamp-admin --namespace kube-system 2>/dev/null || echo "Erreur: impossible de créer le token") +echo "" +echo -e "${GREEN}$HEADLAMP_TOKEN${NC}" +echo "" +log_warning "Copiez ce token et ouvrez http://localhost:9999 pour suivre l'installation" +echo "" + +# 3. Lint du chart Starlake +log_info "Validation du chart (helm lint)..." +helm lint $CHART_PATH || { + log_error "Helm lint a échoué" + exit 1 +} +log_success "Chart valide" + +echo "" + +# 4. Installer le chart Starlake +log_info "Installation du chart Helm..." +log_info " Namespace: $NAMESPACE" +log_info " Storage: local-path (K3s built-in)" + +# Préparer les options d'images locales +LOCAL_IMAGE_OPTS="" + +# Image Airflow (K8s Job mode) +# L'image Dockerfile_airflow_k8s inclut: +# - kubectl pour créer des K8s Jobs +# - Le wrapper starlake qui crée des Jobs au lieu d'exécuter localement +# - starlake-airflow 0.4.x pré-installé (compatible Airflow 2) +if [ "$USE_LOCAL_AIRFLOW_IMAGE" = "true" ]; then + log_info " Image Airflow: $AIRFLOW_IMAGE_LOCAL (K8s Job mode, packages pré-installés)" + LOCAL_IMAGE_OPTS="$LOCAL_IMAGE_OPTS --set airflow.image.repository=starlake-airflow --set airflow.image.tag=local --set airflow.image.pullPolicy=Never --set airflow.installPythonPackages=false" +else + log_info " Image Airflow: apache/airflow (par défaut, pip install au démarrage)" +fi + +# Image UI +if [ "$USE_LOCAL_UI_IMAGE" = "true" ]; then + log_info " Image UI: $UI_IMAGE_LOCAL (locale)" + LOCAL_IMAGE_OPTS="$LOCAL_IMAGE_OPTS --set ui.image.repository=starlake-ui --set ui.image.tag=local --set ui.image.pullPolicy=Never" +else + log_info " Image UI: starlakeai/starlake-1.5-ui (par défaut)" +fi + +# Image Agent +if [ "$USE_LOCAL_AGENT_IMAGE" = "true" ]; then + log_info " Image Agent: $AGENT_IMAGE_LOCAL (locale)" + LOCAL_IMAGE_OPTS="$LOCAL_IMAGE_OPTS --set agent.image.repository=starlake-agent --set agent.image.tag=local --set agent.image.pullPolicy=Never" +else + log_info " Image Agent: starlakeai/starlake-1.5-ask (par défaut)" +fi + +# Image Gizmo +if [ "$USE_LOCAL_GIZMO_IMAGE" = "true" ]; then + log_info " Image Gizmo: $GIZMO_IMAGE_LOCAL (locale)" + LOCAL_IMAGE_OPTS="$LOCAL_IMAGE_OPTS --set gizmo.image.repository=starlake-gizmo --set gizmo.image.tag=local --set gizmo.image.pullPolicy=Never" +else + log_info " Image Gizmo: starlakeai/gizmo-on-demand (par défaut)" +fi + +# PostgreSQL: image publique légère, téléchargée automatiquement par k3d +log_info " Image PostgreSQL: postgres:17 (téléchargée depuis Docker Hub)" + +log_info " Note: Les images Starlake nécessitent ~2-3 minutes pour démarrer" + +# Installation avec les paramètres optimisés pour K3s +# Note: --wait=false car les init jobs prennent du temps (airflow db init, demo data load) +# La boucle de surveillance ci-dessous attend que les pods soient prêts + +# Préparer les options de credentials +CREDENTIAL_OPTS="" +CREDENTIAL_OPTS="$CREDENTIAL_OPTS --set postgresql.credentials.password=$PG_PASSWORD" +CREDENTIAL_OPTS="$CREDENTIAL_OPTS --set airflow.admin.password=$AIRFLOW_PASSWORD" +CREDENTIAL_OPTS="$CREDENTIAL_OPTS --set airflow.secretKey=$AIRFLOW_SECRET_KEY" +CREDENTIAL_OPTS="$CREDENTIAL_OPTS --set gizmo.apiKey=$GIZMO_API_KEY" +CREDENTIAL_OPTS="$CREDENTIAL_OPTS --set agent.applicationKey=$AGENT_APPLICATION_KEY" +CREDENTIAL_OPTS="$CREDENTIAL_OPTS --set security.validateCredentials=$VALIDATE_CREDENTIALS" + +log_info " Credentials: $([ "$PRODUCTION_MODE" = true ] && echo "sécurisés (mode production)" || echo "par défaut (mode dev)")" +log_info " Validation: $VALIDATE_CREDENTIALS" + +# Déterminer le storage class +if [ -z "$STORAGE_CLASS" ]; then + STORAGE_CLASS="local-path" +fi + +log_info " Storage class: $STORAGE_CLASS" +log_info " Mode: $([ "$MULTI_NODE" = true ] && echo "multi-node ($AGENT_COUNT agents)" || echo "single-node")" + +# Options spécifiques multi-node +MULTINODE_OPTS="" +if [ "$MULTI_NODE" = true ]; then + # Note: En multi-node avec local-path storage, on NE PEUT PAS forcer Gizmo sur le server + # car le PVC a une affinité vers le nœud où il a été créé (limitation local-path) + # Les ports Gizmo (11900+) sont accessibles via port-forward: + log_warning " Gizmo: port-forward requis en multi-node (limitation storage local-path)" + log_info " Commande: kubectl port-forward deploy/starlake-gizmo 11900:11900 -n starlake" + # Ne pas ajouter de nodeSelector - laisser Gizmo se scheduler où le PVC est disponible +fi + +# Options SeaweedFS (object storage S3) +SEAWEEDFS_OPTS="" +if [ "$SEAWEEDFS_ENABLED" = true ]; then + log_info " SeaweedFS: Object storage S3 activé (pour projets utilisateur)" + log_info " Note: Initialisation du bucket S3 via hook post-install (~30-60s)" + SEAWEEDFS_OPTS="--set seaweedfs.enabled=true" + SEAWEEDFS_OPTS="$SEAWEEDFS_OPTS --set seaweedfs.persistence.storageClass=$STORAGE_CLASS" +fi + +# Demo projects - always enabled and stored on local PVC +# (regardless of storage mode - provides working examples out-of-the-box) +DEMO_ENABLED="true" +log_info " Demo: Activé (toujours en local PVC pour exemples fonctionnels)" + +helm install starlake $CHART_PATH \ + --namespace $NAMESPACE \ + --create-namespace \ + --timeout 10m \ + --set postgresql.internal.persistence.size=2Gi \ + --set postgresql.internal.persistence.storageClass=$STORAGE_CLASS \ + --set persistence.projects.size=2Gi \ + --set persistence.projects.storageClass=$STORAGE_CLASS \ + --set airflow.webserver.resources.requests.memory=4Gi \ + --set airflow.webserver.resources.limits.memory=16Gi \ + --set ui.resources.requests.memory=512Mi \ + --set ui.resources.limits.memory=2Gi \ + --set agent.resources.requests.memory=256Mi \ + --set agent.resources.limits.memory=1Gi \ + --set airflow.logs.persistence.enabled=false \ + --set gizmo.enabled=true \ + --set gizmo.resources.requests.memory=512Mi \ + --set gizmo.resources.limits.memory=2Gi \ + --set ui.service.type=ClusterIP \ + --set ingress.enabled=true \ + --set ingress.className="" \ + --set ingress.host="" \ + --set demo.enabled=$DEMO_ENABLED \ + --set ui.frontendUrl=http://localhost:8080 \ + --set airflow.baseUrl=http://localhost:8080/airflow \ + --set airflow.jobRunner.enabled=true \ + $CREDENTIAL_OPTS \ + $LOCAL_IMAGE_OPTS \ + $MULTINODE_OPTS \ + $SEAWEEDFS_OPTS || { + log_error "Installation du chart a échoué" + exit 1 + } + +log_success "Chart soumis à Kubernetes" + +echo "" + +# 5. Surveiller le déploiement avec logs en temps réel +log_info "Surveillance du déploiement..." +log_info " Temps estimé: 2-5 minutes (téléchargement des images + démarrage)" +echo "" + +# Fonction pour afficher l'état des pods +show_pod_status() { + echo "" + log_info "=== État des pods ===" + kubectl get pods -n $NAMESPACE -o wide + echo "" + + # Vérifier les PVCs + log_info "=== État des PVCs ===" + kubectl get pvc -n $NAMESPACE + echo "" +} + +# Fonction pour afficher les logs des pods en erreur +show_error_logs() { + local pod_name=$1 + echo "" + log_warning "=== Logs de $pod_name ===" + kubectl logs "$pod_name" -n $NAMESPACE --tail=30 2>/dev/null || \ + echo "Pas de logs disponibles" +} + +# Boucle de surveillance +MAX_ATTEMPTS=$POD_READY_MAX_ATTEMPTS # Configurable en haut du script +ATTEMPT=0 +ALL_READY=false +CONSECUTIVE_ERRORS=0 + +while [ $ATTEMPT -lt $MAX_ATTEMPTS ]; do + ATTEMPT=$((ATTEMPT + 1)) + + # Récupérer l'état des pods + PODS_STATUS=$(kubectl get pods -n $NAMESPACE --no-headers 2>/dev/null) + + if [ -z "$PODS_STATUS" ]; then + log_info "[$ATTEMPT/$MAX_ATTEMPTS] Attente de la création des pods..." + sleep 5 + continue + fi + + # Compter les pods par état (exclure les jobs Completed) + # Note: pipefail + grep = exit 1 si pas de match, d'où les sous-shells + TOTAL=$(grep -vc "Completed" <<< "$PODS_STATUS" || true) + RUNNING=$(grep -c "Running" <<< "$PODS_STATUS" || true) + PENDING=$(grep -c "Pending" <<< "$PODS_STATUS" || true) + CRASHLOOP=$(grep -c "CrashLoopBackOff\|ImagePullBackOff" <<< "$PODS_STATUS" || true) + ERROR=$({ grep -E "Error" <<< "$PODS_STATUS" | grep -vc "Completed"; } || true) + INIT=$(grep -c "Init:" <<< "$PODS_STATUS" || true) + READY=$({ grep -E "[0-9]+/[0-9]+.*Running" <<< "$PODS_STATUS" | awk '{split($2,a,"/"); if(a[1]==a[2]) print}' | wc -l | tr -d ' '; } || true) + + echo -ne "\r[$ATTEMPT/$MAX_ATTEMPTS] Pods: $READY/$TOTAL Ready, $RUNNING Running, $INIT Init, $PENDING Pending, $CRASHLOOP CrashLoop " + + # Vérifier si tous les pods principaux sont ready (exclure les jobs) + # On attend au moins 5 pods: postgresql, airflow, ui, agent, gizmo (proxy removed) + if [ "$READY" -ge 5 ] && [ "$CRASHLOOP" -eq 0 ]; then + echo "" + ALL_READY=true + break + fi + + # Si des pods sont en CrashLoopBackOff depuis plusieurs itérations + if [ "$CRASHLOOP" -gt 0 ]; then + CONSECUTIVE_ERRORS=$((CONSECUTIVE_ERRORS + 1)) + + # Attendre 3 itérations avant de considérer comme échec (laisser le temps aux restarts) + if [ $CONSECUTIVE_ERRORS -ge 6 ]; then + echo "" + log_error "Pods en CrashLoopBackOff depuis plus de 30 secondes" + show_pod_status + + # Afficher les logs des pods en erreur + while IFS= read -r line; do + POD_NAME=$(echo "$line" | awk '{print $1}') + POD_STATUS=$(echo "$line" | awk '{print $3}') + + if [[ "$POD_STATUS" == *"CrashLoopBackOff"* ]] || [[ "$POD_STATUS" == *"ImagePullBackOff"* ]]; then + show_error_logs "$POD_NAME" + + # Events du pod + log_warning "=== Events du pod $POD_NAME ===" + kubectl describe pod "$POD_NAME" -n $NAMESPACE | grep -A 15 "Events:" || true + fi + done <<< "$PODS_STATUS" + + log_error "Des pods sont en erreur. Consultez les logs ci-dessus." + log_info "Pour débugger manuellement:" + echo " kubectl get pods -n $NAMESPACE" + echo " kubectl logs -n $NAMESPACE" + echo " kubectl describe pod -n $NAMESPACE" + exit 1 + fi + else + CONSECUTIVE_ERRORS=0 + fi + + sleep 5 +done + +echo "" + +if [ "$ALL_READY" = true ]; then + log_success "Tous les pods sont prêts!" +else + log_warning "Timeout atteint, vérifions l'état actuel..." + show_pod_status + + # Vérifier si c'est acceptable (certains pods peuvent avoir des restarts) + READY=$(kubectl get pods -n $NAMESPACE --no-headers | grep -E "[0-9]+/[0-9]+.*Running" | awk '{split($2,a,"/"); if(a[1]==a[2]) print}' | wc -l | tr -d ' ') + if [ "$READY" -ge 5 ]; then + log_warning "La plupart des pods sont prêts ($READY/6), on continue..." + else + log_error "Pas assez de pods prêts, arrêt du test" + exit 1 + fi +fi + +# Afficher l'état final +echo "" +log_info "État final des pods:" +kubectl get pods -n $NAMESPACE -o wide + +echo "" +log_info "État des PVCs:" +kubectl get pvc -n $NAMESPACE + +echo "" +log_info "État des services:" +kubectl get svc -n $NAMESPACE + +log_success "Ressources déployées" + +# 5.4 Validation multi-node (si activé) +if [ "$MULTI_NODE" = true ]; then + echo "" + log_info "=== Validation Multi-Node ===" + + # Afficher la distribution des pods par nœud + log_info "Distribution des pods par nœud:" + for node in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do + pod_count=$(kubectl get pods -n $NAMESPACE --field-selector spec.nodeName=$node --no-headers 2>/dev/null | wc -l | tr -d ' ') + echo " $node: $pod_count pods" + done + + # Compter le nombre de nœuds utilisés + NODES_WITH_PODS=$(kubectl get pods -n $NAMESPACE -o jsonpath='{.items[*].spec.nodeName}' | tr ' ' '\n' | sort -u | wc -l | tr -d ' ') + TOTAL_NODES=$(kubectl get nodes --no-headers | wc -l | tr -d ' ') + + echo "" + log_info "Résumé: Pods distribués sur $NODES_WITH_PODS/$TOTAL_NODES nœuds" + + if [ "$NODES_WITH_PODS" -gt 1 ]; then + log_success "✓ Distribution multi-nœuds validée" + else + log_warning "⚠ Tous les pods sont sur un seul nœud (possible si affinité ou resources limitées)" + fi + + # Vérifier que le PVC projects est accessible (RWX) + log_info "Vérification du storage RWX..." + PVC_ACCESS_MODE=$(kubectl get pvc starlake-projects -n $NAMESPACE -o jsonpath='{.spec.accessModes[0]}' 2>/dev/null || echo "N/A") + if [ "$PVC_ACCESS_MODE" = "ReadWriteMany" ] || [ "$STORAGE_CLASS" = "nfs-client" ]; then + log_success "✓ Storage RWX configuré (StorageClass: $STORAGE_CLASS)" + else + log_info " PVC access mode: $PVC_ACCESS_MODE (StorageClass: $STORAGE_CLASS)" + fi + + echo "" +fi + +echo "" + +# 5.5 Configurer les projets en local_mode pour éviter les erreurs de chemin +log_info "Configuration des projets en local_mode (fix path resolution)..." +# Attendre que PostgreSQL soit prêt +sleep 5 +kubectl exec starlake-postgresql-0 -n $NAMESPACE -- \ + psql -U dbuser -d starlake -c "UPDATE slk_project SET local_mode = true WHERE local_mode = false;" 2>/dev/null || \ + log_warning "Pas de projets à mettre à jour (table vide ou non créée)" +log_success "Projets configurés" + +echo "" + +# 6. Tests fonctionnels +log_info "Exécution des tests fonctionnels..." + +# Test 1: PostgreSQL +log_info "Test 1/6: Connexion PostgreSQL..." +if kubectl exec starlake-postgresql-0 -n $NAMESPACE -- \ + psql -U dbuser -d starlake -c "SELECT 1" > /dev/null 2>&1; then + log_success " PostgreSQL: OK" +else + log_warning " PostgreSQL: En cours de démarrage..." +fi + +# Test 2: Bases de données créées +log_info "Test 2/6: Vérification des bases de données..." +DB_COUNT=$(kubectl exec starlake-postgresql-0 -n $NAMESPACE -- \ + psql -U dbuser -c "\l" 2>/dev/null | grep -E "starlake|airflow" | wc -l || echo "0") +if [ "$DB_COUNT" -ge 2 ]; then + log_success " Bases de données: OK ($DB_COUNT trouvées)" +else + log_warning " Bases de données: En cours de création..." +fi + +# Test 3: API Airflow accessible +log_info "Test 3/6: API Airflow..." +AIRFLOW_POD=$(kubectl get pod -n $NAMESPACE -l app.kubernetes.io/component=airflow -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") +if [ -n "$AIRFLOW_POD" ]; then + API_RESPONSE=$(kubectl exec $AIRFLOW_POD -n $NAMESPACE -- \ + curl -s -u airflow:airflow http://localhost:8080/airflow/api/v1/dags 2>/dev/null || echo "") + if grep -q "dags" <<< "$API_RESPONSE"; then + log_success " API Airflow: OK" + else + log_warning " API Airflow: En cours de démarrage..." + fi +else + log_warning " API Airflow: Pod non trouvé" +fi + +# Test 4: Health check UI (direct) +log_info "Test 4/6: Health check UI (via port-forward direct)..." +# Note: Service expose port 80 which maps to container port 9900 +kubectl port-forward svc/starlake-ui 8888:80 -n $NAMESPACE > /dev/null 2>&1 & +PF_PID=$! +sleep $PORT_FORWARD_SLEEP + +UI_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8888/api/v1/health 2>/dev/null || echo "000") +if [ "$UI_HEALTH" = "200" ]; then + log_success " UI: OK (HTTP $UI_HEALTH)" +else + log_warning " UI: HTTP $UI_HEALTH (peut prendre plus de temps)" +fi + +kill $PF_PID 2>/dev/null || true + +# Test 5: Health check Airflow (direct) +log_info "Test 5/6: Health check Airflow (via port-forward direct)..." +kubectl port-forward svc/starlake-airflow 8889:8080 -n $NAMESPACE > /dev/null 2>&1 & +PF_PID=$! +sleep $PORT_FORWARD_SLEEP + +AIRFLOW_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8889/airflow/health 2>/dev/null || echo "000") +if [ "$AIRFLOW_HEALTH" = "200" ]; then + log_success " Airflow: OK (HTTP $AIRFLOW_HEALTH)" +else + log_warning " Airflow: HTTP $AIRFLOW_HEALTH (peut prendre plus de temps)" +fi + +kill $PF_PID 2>/dev/null || true + +# Test 6: Health check Gizmo (direct) +log_info "Test 6/6: Health check Gizmo (via port-forward direct)..." +kubectl port-forward svc/starlake-gizmo 10999:10900 -n $NAMESPACE > /dev/null 2>&1 & +PF_PID=$! +sleep $PORT_FORWARD_SLEEP + +GIZMO_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:10999/health 2>/dev/null || echo "000") +if [ "$GIZMO_HEALTH" = "200" ]; then + log_success " Gizmo: OK (HTTP $GIZMO_HEALTH)" +else + log_warning " Gizmo: HTTP $GIZMO_HEALTH (peut prendre plus de temps)" +fi + +kill $PF_PID 2>/dev/null || true + +# Test 7: Health check SeaweedFS (si activé) +if [ "$SEAWEEDFS_ENABLED" = true ]; then + log_info "Test 7/7: Health check SeaweedFS (via port-forward direct)..." + kubectl port-forward svc/starlake-seaweedfs 8399:8333 -n $NAMESPACE > /dev/null 2>&1 & + PF_PID=$! + sleep $PORT_FORWARD_SLEEP + + # Test S3 API health + SEAWEEDFS_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8399/ 2>/dev/null || echo "000") + if [ "$SEAWEEDFS_HEALTH" = "200" ] || [ "$SEAWEEDFS_HEALTH" = "403" ]; then + log_success " SeaweedFS S3: OK (HTTP $SEAWEEDFS_HEALTH)" + else + log_warning " SeaweedFS S3: HTTP $SEAWEEDFS_HEALTH (peut prendre plus de temps)" + fi + + kill $PF_PID 2>/dev/null || true +fi + +echo "" + +# 8. Test d'upgrade (optionnel, rapide) +log_info "Test d'upgrade du chart..." +helm upgrade starlake $CHART_PATH \ + --namespace $NAMESPACE \ + --reuse-values \ + --set ui.replicas=1 \ + --timeout 5m || { + log_warning "Upgrade a échoué (peut être normal si des pods redémarrent)" + } +log_success "Upgrade soumis" + +# Vérifier l'historique +REVISION_COUNT=$(helm history starlake -n $NAMESPACE 2>/dev/null | tail -n +2 | wc -l | tr -d ' ') +if [ "$REVISION_COUNT" -ge 2 ]; then + log_success "Historique: $REVISION_COUNT révisions" +fi + +echo "" + +# 8. Résumé +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 📊 Résumé des Tests" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +log_success "Chart installé et testé avec succès!" +echo "" +log_info "Composants déployés:" +echo " - PostgreSQL (StatefulSet)" +echo " - Airflow Webserver + Scheduler" +echo " - Starlake UI" +echo " - Starlake Agent (AI)" +echo " - Gizmo (SQL on-demand)" +if [ "$SEAWEEDFS_ENABLED" = true ]; then + echo " - SeaweedFS (S3 Object Storage)" +fi +echo " - Headlamp (Interface Web Kubernetes)" +echo "" + +# 9. Démarrage des port-forwards pour Starlake +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🚀 Démarrage des Port-Forwards Starlake" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +log_info "Démarrage des port-forwards..." +log_success " Headlamp: http://localhost:9999 (déjà actif)" + +# Démarrer Starlake UI port-forward sur port 8080 (service port 80 maps to container 9900) +# Note: UI proxie /airflow vers le service Airflow interne, pas besoin de port-forward séparé +kubectl port-forward svc/starlake-ui 8080:80 -n $NAMESPACE > /dev/null 2>&1 & +UI_PF_PID=$! +log_success " Starlake UI: http://localhost:8080 (PID: $UI_PF_PID)" +log_success " Airflow: http://localhost:8080/airflow (via UI proxy)" + +# Démarrer Agent port-forward (port 8000) +kubectl port-forward svc/starlake-agent 8000:8000 -n $NAMESPACE > /dev/null 2>&1 & +AGENT_PF_PID=$! +log_success " Agent: http://localhost:8000 (PID: $AGENT_PF_PID)" + +# Démarrer Gizmo port-forward (port 10900 API + ports 11900-11909 SQL) +kubectl port-forward svc/starlake-gizmo 10900:10900 -n $NAMESPACE > /dev/null 2>&1 & +GIZMO_PF_PID=$! +log_success " Gizmo API: http://localhost:10900 (PID: $GIZMO_PF_PID)" + +# Port-forward Gizmo SQL ports (11900-11909) via le pod directement (hostNetwork) +GIZMO_POD=$(kubectl get pod -n $NAMESPACE -l app.kubernetes.io/component=gizmo -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") +if [ -n "$GIZMO_POD" ]; then + for port in $(seq 11900 11909); do + kubectl port-forward "pod/$GIZMO_POD" "$port:$port" -n $NAMESPACE > /dev/null 2>&1 & + done + log_success " Gizmo SQL: localhost:11900-11909 (Arrow Flight SQL, TLS)" + log_info " JDBC: jdbc:arrow-flight-sql://localhost:11900?useEncryption=true&disableCertificateVerification=true" +fi + +# Démarrer SeaweedFS port-forwards (si activé) +if [ "$SEAWEEDFS_ENABLED" = true ]; then + # S3 API (8333) + kubectl port-forward svc/starlake-seaweedfs 8333:8333 -n $NAMESPACE > /dev/null 2>&1 & + SEAWEEDFS_S3_PF_PID=$! + log_success " SeaweedFS S3 API: http://localhost:8333 (PID: $SEAWEEDFS_S3_PF_PID)" + + # Master UI (9333) - Interface web cluster status + kubectl port-forward svc/starlake-seaweedfs 9333:9333 -n $NAMESPACE > /dev/null 2>&1 & + SEAWEEDFS_MASTER_PF_PID=$! + log_success " SeaweedFS Master UI: http://localhost:9333 (PID: $SEAWEEDFS_MASTER_PF_PID)" + + # Filer UI (8888) - File browser + kubectl port-forward svc/starlake-seaweedfs 8888:8888 -n $NAMESPACE > /dev/null 2>&1 & + SEAWEEDFS_FILER_PF_PID=$! + log_success " SeaweedFS Filer UI: http://localhost:8888 (PID: $SEAWEEDFS_FILER_PF_PID)" + + log_info " S3 Endpoint: http://localhost:8333" + log_info " Bucket: starlake (SL_ROOT=s3a://starlake)" + log_info " Credentials: seaweedfs / seaweedfs123" +fi + +sleep $PORT_FORWARD_SLEEP + +# 10. Vérification des accès +echo "" +log_info "Vérification des accès..." + +HEADLAMP_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9999/ 2>/dev/null || echo "000") +if [ "$HEADLAMP_CHECK" = "200" ] || [ "$HEADLAMP_CHECK" = "304" ]; then + log_success " Headlamp: OK (HTTP $HEADLAMP_CHECK)" +else + log_warning " Headlamp: HTTP $HEADLAMP_CHECK - Vérifiez le pod: kubectl get pods -n kube-system -l app.kubernetes.io/name=headlamp" +fi + +UI_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/api/v1/health 2>/dev/null || echo "000") +if [ "$UI_CHECK" = "200" ]; then + log_success " Starlake UI: OK (HTTP $UI_CHECK)" +else + log_warning " Starlake UI: HTTP $UI_CHECK - Service port 80 -> container 9900" + log_info " Vérifiez le pod: kubectl get pods -n $NAMESPACE -l app.kubernetes.io/component=ui" +fi + +# Airflow via UI proxy (same port 8080) +AIRFLOW_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/airflow/health 2>/dev/null || echo "000") +if [ "$AIRFLOW_CHECK" = "200" ]; then + log_success " Airflow: OK (HTTP $AIRFLOW_CHECK)" +else + log_warning " Airflow: HTTP $AIRFLOW_CHECK - Vérifiez le pod: kubectl get pods -n $NAMESPACE -l app.kubernetes.io/component=airflow" +fi + +AGENT_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ask/health 2>/dev/null || echo "000") +if [ "$AGENT_CHECK" = "200" ]; then + log_success " Agent: OK (HTTP $AGENT_CHECK)" +else + log_warning " Agent: HTTP $AGENT_CHECK - Vérifiez le pod: kubectl get pods -n $NAMESPACE -l app.kubernetes.io/component=agent" +fi + +GIZMO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:10900/health 2>/dev/null || echo "000") +if [ "$GIZMO_CHECK" = "200" ]; then + log_success " Gizmo: OK (HTTP $GIZMO_CHECK)" +else + log_warning " Gizmo: HTTP $GIZMO_CHECK - Vérifiez le pod: kubectl get pods -n $NAMESPACE -l app.kubernetes.io/component=gizmo" +fi + +if [ "$SEAWEEDFS_ENABLED" = true ]; then + # Check S3 API (403 is expected without auth) + SEAWEEDFS_S3_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8333/ 2>/dev/null || echo "000") + if [ "$SEAWEEDFS_S3_CHECK" = "200" ] || [ "$SEAWEEDFS_S3_CHECK" = "403" ]; then + log_success " SeaweedFS S3 API: OK (HTTP $SEAWEEDFS_S3_CHECK)" + else + log_warning " SeaweedFS S3 API: HTTP $SEAWEEDFS_S3_CHECK" + fi + + # Check Master UI + SEAWEEDFS_MASTER_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9333/ 2>/dev/null || echo "000") + if [ "$SEAWEEDFS_MASTER_CHECK" = "200" ]; then + log_success " SeaweedFS Master UI: OK (HTTP $SEAWEEDFS_MASTER_CHECK)" + else + log_warning " SeaweedFS Master UI: HTTP $SEAWEEDFS_MASTER_CHECK" + fi + + # Check Filer UI + SEAWEEDFS_FILER_CHECK=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8888/ 2>/dev/null || echo "000") + if [ "$SEAWEEDFS_FILER_CHECK" = "200" ]; then + log_success " SeaweedFS Filer UI: OK (HTTP $SEAWEEDFS_FILER_CHECK)" + else + log_warning " SeaweedFS Filer UI: HTTP $SEAWEEDFS_FILER_CHECK" + fi +fi + +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🌐 Applications Accessibles" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +echo " Headlamp: http://localhost:9999" +echo " Starlake UI: http://localhost:8080" +echo " Airflow: http://localhost:8080/airflow (via UI proxy)" +echo " Agent: http://localhost:8000" +echo " Gizmo API: http://localhost:10900" +echo " Gizmo SQL: localhost:11900-11909 (Arrow Flight SQL, TLS)" +echo " JDBC: jdbc:arrow-flight-sql://localhost:11900?useEncryption=true&disableCertificateVerification=true" +if [ "$SEAWEEDFS_ENABLED" = true ]; then + echo "" + echo " SeaweedFS:" + echo " Master UI: http://localhost:9333 (cluster status)" + echo " Filer UI: http://localhost:8888 (file browser)" + echo " S3 API: http://localhost:8333 (requires auth)" +fi +echo "" +if [ "$PRODUCTION_MODE" = true ]; then + echo " 🔒 Mode Production - Credentials sécurisés:" + echo " Airflow: airflow / $AIRFLOW_PASSWORD" + echo " PostgreSQL: dbuser / $PG_PASSWORD" + echo " Gizmo API KEY: $SECURE_GIZMO_API_KEY" +else + echo " Credentials Airflow: airflow / airflow" + echo " Credentials PostgreSQL: dbuser / dbuser123" +fi +if [ "$SEAWEEDFS_ENABLED" = true ]; then + echo "" + echo " 📦 SeaweedFS Object Storage:" + echo " Master UI: http://localhost:9333 (cluster status)" + echo " Filer UI: http://localhost:8888 (file browser)" + echo " S3 API: http://localhost:8333" + echo " Bucket: starlake" + echo " SL_ROOT: s3a://starlake" + echo " Access Key: seaweedfs" + echo " Secret Key: seaweedfs123" +fi +echo "" + +log_info "Pour voir les logs:" +echo " kubectl logs -n $NAMESPACE -l app.kubernetes.io/component=ui -f" +echo " kubectl logs -n $NAMESPACE -l app.kubernetes.io/component=airflow -f" +echo "" + +# 11. Option pour garder le cluster +read -p "Voulez-vous garder le cluster pour inspecter? (y/N) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + log_info "Cluster conservé: $CLUSTER_NAME" + echo "" + echo " Pour arrêter les port-forwards: pkill -f 'kubectl port-forward'" + echo " Pour supprimer le cluster: k3d cluster delete $CLUSTER_NAME" + echo "" + + trap - EXIT # Désactiver le cleanup automatique + exit 0 +fi + +echo "" +log_info "Nettoyage automatique..." + +# Tuer les port-forwards avant cleanup +pkill -f "kubectl port-forward" 2>/dev/null || true + +cleanup +trap - EXIT # Désactiver le trap + +echo "" +log_success "✅ Tous les tests ont réussi!" +echo "" diff --git a/helm/test-s3-regression.sh b/helm/test-s3-regression.sh new file mode 100755 index 0000000..d062bfa --- /dev/null +++ b/helm/test-s3-regression.sh @@ -0,0 +1,886 @@ +#!/usr/bin/env bash +set -euo pipefail + +# S3/SeaweedFS Regression Test Script for Starlake Helm Chart +# +# Run AFTER the cluster is deployed (after test-helm-chart.sh --seaweedfs completes). +# Tests the full lifecycle: auth -> project -> domain -> schema -> load -> verify +# with special emphasis on the 86-byte directory marker corruption bug. +# +# Usage: +# ./test-s3-regression.sh # Run all tests (default) +# ./test-s3-regression.sh --cleanup # Delete test artifacts after run +# ./test-s3-regression.sh --api-url http://x # Custom API URL +# ./test-s3-regression.sh --namespace ns # Custom K8s namespace +# ./test-s3-regression.sh --skip-s3 # Skip S3 direct checks (tests 8-9) +# ./test-s3-regression.sh --verbose # Show curl response bodies + +# ============================================================ +# Configuration +# ============================================================ + +API_URL="${API_URL:-http://localhost:8080}" +NAMESPACE="${NAMESPACE:-starlake}" +S3_ACCESS_KEY="${S3_ACCESS_KEY:-seaweedfs}" +S3_SECRET_KEY="${S3_SECRET_KEY:-seaweedfs123}" +S3_BUCKET="${S3_BUCKET:-starlake}" +SEAWEEDFS_S3_PORT=18333 # Local port for S3 port-forward +SEAWEEDFS_FILER_PORT=18888 # Local port for Filer port-forward +CLEANUP_AFTER=false +SKIP_S3=false +VERBOSE=false + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +BOLD='\033[1m' +NC='\033[0m' + +# Counters +PASS=0 +FAIL=0 +SKIP=0 + +# Unique domain name to avoid collisions +DOMAIN_NAME="regtest_$(date +%s)" + +# Cookie file for session +COOKIE_FILE=$(mktemp) + +# Track port-forward PIDs for cleanup +PF_PIDS=() + +# CSV file - search known locations +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CSV_FILE="" +for candidate in \ + "$PROJECT_ROOT/tpch001/datasets/stage/tpch/orders-001.csv" \ + "$SCRIPT_DIR/../tpch001/datasets/stage/tpch/orders-001.csv" \ + "./tpch001/datasets/stage/tpch/orders-001.csv"; do + if [[ -f "$candidate" ]]; then + CSV_FILE="$(cd "$(dirname "$candidate")" && pwd)/$(basename "$candidate")" + break + fi +done + +# ============================================================ +# Argument parsing +# ============================================================ + +while [[ $# -gt 0 ]]; do + case $1 in + --cleanup) + CLEANUP_AFTER=true + shift + ;; + --api-url) + API_URL="$2" + shift 2 + ;; + --namespace) + NAMESPACE="$2" + shift 2 + ;; + --skip-s3) + SKIP_S3=true + shift + ;; + --verbose|-v) + VERBOSE=true + shift + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --cleanup Delete test domain after run" + echo " --api-url URL API base URL (default: http://localhost:8080)" + echo " --namespace NS Kubernetes namespace (default: starlake)" + echo " --skip-s3 Skip direct S3/SeaweedFS checks (tests 8-9)" + echo " --verbose, -v Show curl response bodies" + echo " --help, -h Show this help" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage." + exit 1 + ;; + esac +done + +# ============================================================ +# Helper functions +# ============================================================ + +pass() { + echo -e " ${GREEN}PASS${NC}: $1" + PASS=$((PASS + 1)) +} + +fail() { + echo -e " ${RED}FAIL${NC}: $1" + FAIL=$((FAIL + 1)) +} + +skip() { + echo -e " ${YELLOW}SKIP${NC}: $1" + SKIP=$((SKIP + 1)) +} + +info() { + echo -e " ${BLUE}INFO${NC}: $1" +} + +verbose_log() { + if [[ "$VERBOSE" = true ]]; then + echo -e " ${BLUE}BODY${NC}: $1" + fi +} + +# Perform a curl request and capture HTTP code + body +# Usage: api_call METHOD PATH [EXTRA_CURL_ARGS...] +# Sets: HTTP_CODE, HTTP_BODY +api_call() { + local method="$1" + local path="$2" + shift 2 + + local url="${API_URL}${path}" + local response_file + response_file=$(mktemp) + + HTTP_CODE=$(curl -s -o "$response_file" -w "%{http_code}" \ + -X "$method" \ + -b "$COOKIE_FILE" -c "$COOKIE_FILE" \ + "$url" "$@" 2>/dev/null) || HTTP_CODE="000" + + HTTP_BODY=$(cat "$response_file" 2>/dev/null || echo "") + rm -f "$response_file" + + verbose_log "$(echo "$HTTP_BODY" | head -c 500)" +} + +cleanup_port_forwards() { + for pid in "${PF_PIDS[@]}"; do + kill "$pid" 2>/dev/null || true + done + PF_PIDS=() +} + +cleanup() { + rm -f "$COOKIE_FILE" + cleanup_port_forwards +} + +trap cleanup EXIT + +# ============================================================ +# Banner +# ============================================================ + +echo "" +echo "================================================================" +echo " S3/SeaweedFS Regression Tests - Starlake Helm Chart" +echo "================================================================" +echo "" +echo " API URL: $API_URL" +echo " Namespace: $NAMESPACE" +echo " Domain: $DOMAIN_NAME" +echo " Cleanup: $CLEANUP_AFTER" +echo " Skip S3: $SKIP_S3" +if [[ -n "$CSV_FILE" ]]; then + echo " CSV File: $CSV_FILE" +else + echo " CSV File: NOT FOUND (tests 5-6 will be skipped)" +fi +echo "" + +# ============================================================ +# Prerequisites +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Prerequisites" +echo "----------------------------------------------------------------" + +# Check kubectl +if ! command -v kubectl &>/dev/null; then + fail "kubectl not found in PATH" + echo "Cannot continue without kubectl." + exit 1 +fi +pass "kubectl available" + +# Check cluster pods +POD_STATUS=$(kubectl get pods -n "$NAMESPACE" --no-headers 2>/dev/null || echo "") +if [[ -z "$POD_STATUS" ]]; then + fail "No pods found in namespace '$NAMESPACE'" + echo "Deploy the cluster first: ./test-helm-chart.sh --seaweedfs" + exit 1 +fi + +RUNNING_COUNT=$(echo "$POD_STATUS" | grep -c "Running" || true) +if [[ "$RUNNING_COUNT" -ge 4 ]]; then + pass "Cluster running ($RUNNING_COUNT pods in Running state)" +else + fail "Only $RUNNING_COUNT pods running (expected >= 4)" + echo "$POD_STATUS" + exit 1 +fi + +# Check API accessibility +API_HEALTH=$(curl -s -o /dev/null -w "%{http_code}" "${API_URL}/api/v1/health" 2>/dev/null || echo "000") +if [[ "$API_HEALTH" = "200" ]]; then + pass "API accessible at ${API_URL} (HTTP $API_HEALTH)" +else + fail "API not accessible at ${API_URL} (HTTP $API_HEALTH)" + echo "Ensure port-forward is running: kubectl port-forward svc/starlake-ui 8080:80 -n $NAMESPACE" + exit 1 +fi + +# Check jq +if ! command -v jq &>/dev/null; then + fail "jq not found in PATH (required for JSON parsing)" + echo "Install with: brew install jq" + exit 1 +fi +pass "jq available" + +echo "" + +# ============================================================ +# Test 1: Authentication +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 1: Authentication" +echo "----------------------------------------------------------------" + +api_call POST "/api/v1/auth/basic/signin" \ + -H "Content-Type: application/json" \ + -d '{"email":"admin@localhost.local","password":"admin"}' + +if [[ "$HTTP_CODE" = "200" ]]; then + pass "Authentication successful (HTTP $HTTP_CODE)" + + # Extract user info if available + USER_EMAIL=$(echo "$HTTP_BODY" | jq -r '.email // .user.email // empty' 2>/dev/null || echo "") + if [[ -n "$USER_EMAIL" ]]; then + info "Authenticated as: $USER_EMAIL" + fi +else + fail "Authentication failed (HTTP $HTTP_CODE)" + verbose_log "$HTTP_BODY" + echo "Cannot continue without authentication." + exit 1 +fi + +echo "" + +# ============================================================ +# Test 2: Select S3 Project +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 2: Select S3 Project" +echo "----------------------------------------------------------------" + +# List existing projects +api_call GET "/api/v1/projects" + +PROJECT_ID="" +if [[ "$HTTP_CODE" = "200" ]]; then + pass "Projects listed (HTTP $HTTP_CODE)" + + # Find an S3 project (look for s3a in root or any project) + # Parse projects - try to find one with S3 storage + PROJECT_COUNT=$(echo "$HTTP_BODY" | jq 'if type == "array" then length else 0 end' 2>/dev/null || echo "0") + info "Found $PROJECT_COUNT project(s)" + + if [[ "$PROJECT_COUNT" -gt 0 ]]; then + # Try to find an S3 project first (root contains s3a) + S3_PROJECT_ID=$(echo "$HTTP_BODY" | jq -r '[.[] | select(.root != null and (.root | contains("s3a")))][0].id // empty' 2>/dev/null || echo "") + + if [[ -n "$S3_PROJECT_ID" ]]; then + PROJECT_ID="$S3_PROJECT_ID" + PROJECT_NAME=$(echo "$HTTP_BODY" | jq -r ".[] | select(.id == $PROJECT_ID) | .name" 2>/dev/null || echo "unknown") + info "Selected S3 project: $PROJECT_NAME (id=$PROJECT_ID)" + else + # Fall back to the first project + PROJECT_ID=$(echo "$HTTP_BODY" | jq -r '.[0].id // empty' 2>/dev/null || echo "") + PROJECT_NAME=$(echo "$HTTP_BODY" | jq -r '.[0].name // "unknown"' 2>/dev/null || echo "unknown") + info "No S3 project found, using first project: $PROJECT_NAME (id=$PROJECT_ID)" + fi + fi +else + fail "Failed to list projects (HTTP $HTTP_CODE)" +fi + +if [[ -n "$PROJECT_ID" ]]; then + # Select the project (updates session cookie) + api_call GET "/api/v1/projects/$PROJECT_ID" + + if [[ "$HTTP_CODE" = "200" ]]; then + pass "Project selected: id=$PROJECT_ID" + else + fail "Failed to select project $PROJECT_ID (HTTP $HTTP_CODE)" + echo "Some tests may fail without an active project." + fi +else + skip "No project available to select" + echo " Create an S3 project manually or redeploy with --seaweedfs." +fi + +echo "" + +# ============================================================ +# Test 3: Create Domain +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 3: Create Domain" +echo "----------------------------------------------------------------" + +api_call POST "/api/v1/load/false" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"$DOMAIN_NAME\",\"tags\":[],\"comment\":\"S3 regression test domain\"}" + +if [[ "$HTTP_CODE" = "200" ]]; then + pass "Domain '$DOMAIN_NAME' created (HTTP $HTTP_CODE)" +else + fail "Failed to create domain '$DOMAIN_NAME' (HTTP $HTTP_CODE)" + verbose_log "$HTTP_BODY" +fi + +# Verify domain appears in list +api_call GET "/api/v1/load/names" + +if [[ "$HTTP_CODE" = "200" ]]; then + if echo "$HTTP_BODY" | jq -e ".[] | select(. == \"$DOMAIN_NAME\")" &>/dev/null || \ + echo "$HTTP_BODY" | jq -e ".[] | select(.name == \"$DOMAIN_NAME\")" &>/dev/null || \ + echo "$HTTP_BODY" | grep -q "$DOMAIN_NAME"; then + pass "Domain '$DOMAIN_NAME' visible in domain list" + else + fail "Domain '$DOMAIN_NAME' not found in domain list" + verbose_log "$HTTP_BODY" + fi +else + fail "Failed to list domains (HTTP $HTTP_CODE)" +fi + +echo "" + +# ============================================================ +# Test 4: Check Empty Domain (86-Byte Bug Test) +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 4: Empty Domain Check (86-byte bug detection)" +echo "----------------------------------------------------------------" + +# Check file counts +api_call GET "/api/v1/schemas/files-count/$DOMAIN_NAME" + +if [[ "$HTTP_CODE" = "200" ]]; then + pass "File counts retrieved (HTTP $HTTP_CODE)" + + # Parse counts - handle both flat and nested formats + STAGE_COUNT=$(echo "$HTTP_BODY" | jq '.stage // .stageCount // 0' 2>/dev/null || echo "-1") + INCOMING_COUNT=$(echo "$HTTP_BODY" | jq '.incoming // .incomingCount // 0' 2>/dev/null || echo "-1") + UNRESOLVED_COUNT=$(echo "$HTTP_BODY" | jq '.unresolved // .unresolvedCount // 0' 2>/dev/null || echo "-1") + INGESTING_COUNT=$(echo "$HTTP_BODY" | jq '.ingesting // .ingestingCount // 0' 2>/dev/null || echo "-1") + ARCHIVE_COUNT=$(echo "$HTTP_BODY" | jq '.archive // .archiveCount // 0' 2>/dev/null || echo "-1") + + info "Counts - stage=$STAGE_COUNT incoming=$INCOMING_COUNT unresolved=$UNRESOLVED_COUNT ingesting=$INGESTING_COUNT archive=$ARCHIVE_COUNT" + + ALL_ZERO=true + for count_name in STAGE_COUNT INCOMING_COUNT UNRESOLVED_COUNT INGESTING_COUNT ARCHIVE_COUNT; do + count_val="${!count_name}" + if [[ "$count_val" != "0" && "$count_val" != "-1" && "$count_val" != "null" ]]; then + ALL_ZERO=false + fi + done + + if [[ "$ALL_ZERO" = true ]]; then + pass "All file counts are 0 for empty domain" + else + fail "Non-zero file counts in empty domain (possible 86-byte bug)" + verbose_log "$HTTP_BODY" + fi +else + fail "Failed to get file counts (HTTP $HTTP_CODE)" +fi + +# Check stage file listing +api_call GET "/api/v1/schemas/files/$DOMAIN_NAME?type=stage" + +if [[ "$HTTP_CODE" = "200" ]]; then + FILE_LIST_LENGTH=$(echo "$HTTP_BODY" | jq 'if type == "array" then length else -1 end' 2>/dev/null || echo "-1") + + if [[ "$FILE_LIST_LENGTH" = "0" ]]; then + pass "Stage file list is empty (no phantom files)" + elif [[ "$FILE_LIST_LENGTH" = "-1" ]]; then + # Might be an empty string or non-array response + if [[ "$HTTP_BODY" = "[]" || -z "$HTTP_BODY" ]]; then + pass "Stage file list is empty" + else + fail "Unexpected stage file list format" + verbose_log "$HTTP_BODY" + fi + else + # Check if any file has 86 bytes (the bug signature) + HAS_86_BYTE=$(echo "$HTTP_BODY" | jq '[.[] | select(.fileSizeInBytes == 86 or .size == 86)] | length' 2>/dev/null || echo "0") + if [[ "$HAS_86_BYTE" != "0" ]]; then + fail "CRITICAL: 86-byte phantom file detected in stage (chunked encoding corruption bug)" + verbose_log "$HTTP_BODY" + else + fail "Stage file list is not empty ($FILE_LIST_LENGTH files found in empty domain)" + verbose_log "$HTTP_BODY" + fi + fi +else + fail "Failed to list stage files (HTTP $HTTP_CODE)" +fi + +echo "" + +# ============================================================ +# Test 5: Infer Schema and Create Table +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 5: Infer Schema and Create Table" +echo "----------------------------------------------------------------" + +if [[ -z "$CSV_FILE" ]]; then + skip "CSV file not found - cannot infer schema" + skip "CSV file not found - cannot create table" + SCHEMA_CREATED=false +else + SCHEMA_CREATED=false + + # Infer schema from CSV (raw body, NOT multipart) + api_call POST "/api/v1/schemas/infer-schema-attach?domain=$DOMAIN_NAME&schema=orders&pattern=orders-.*.csv&comment=orders&header=true&filename=orders-001.csv&variant=false" \ + -H "Content-Type: text/csv" \ + --data-binary "@$CSV_FILE" + + if [[ "$HTTP_CODE" = "200" ]]; then + pass "Schema inferred from CSV (HTTP $HTTP_CODE)" + + # Check attribute count + ATTR_COUNT=$(echo "$HTTP_BODY" | jq '.attributes | length' 2>/dev/null || echo "0") + if [[ "$ATTR_COUNT" -eq 9 ]]; then + pass "Schema has 9 attributes (correct for orders table)" + elif [[ "$ATTR_COUNT" -gt 0 ]]; then + info "Schema has $ATTR_COUNT attributes (expected 9)" + pass "Schema has attributes ($ATTR_COUNT found)" + else + fail "Schema has no attributes" + verbose_log "$HTTP_BODY" + fi + + # Create the table using the inferred schema + # The infer-schema-attach endpoint returns the schema body we need to POST + api_call POST "/api/v1/schemas/$DOMAIN_NAME/false/orders" \ + -H "Content-Type: application/json" \ + -d "$HTTP_BODY" + + if [[ "$HTTP_CODE" = "200" ]]; then + pass "Table 'orders' created in domain '$DOMAIN_NAME'" + SCHEMA_CREATED=true + else + fail "Failed to create table 'orders' (HTTP $HTTP_CODE)" + verbose_log "$HTTP_BODY" + fi + else + fail "Schema inference failed (HTTP $HTTP_CODE)" + verbose_log "$HTTP_BODY" + fi +fi + +echo "" + +# ============================================================ +# Test 6: Load Data +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 6: Load Data" +echo "----------------------------------------------------------------" + +DATA_LOADED=false + +if [[ -z "$CSV_FILE" ]]; then + skip "CSV file not found - cannot load data" +elif [[ "$SCHEMA_CREATED" = false ]]; then + skip "Table not created - cannot load data" +else + # Load file via multipart upload + api_call POST "/api/v1/schemas/$DOMAIN_NAME/orders/false/false/sl_none/load" \ + -F "file=@$CSV_FILE;type=text/csv" + + if [[ "$HTTP_CODE" = "200" ]]; then + pass "Data loaded (HTTP $HTTP_CODE)" + + # Check accepted count + ACCEPTED=$(echo "$HTTP_BODY" | jq '.acceptedCount // .accepted // -1' 2>/dev/null || echo "-1") + REJECTED=$(echo "$HTTP_BODY" | jq '.rejectedCount // .rejected // 0' 2>/dev/null || echo "0") + + if [[ "$ACCEPTED" -gt 0 ]]; then + pass "Load accepted $ACCEPTED rows (rejected: $REJECTED)" + DATA_LOADED=true + elif [[ "$ACCEPTED" = "-1" ]]; then + info "Could not parse acceptedCount from response" + verbose_log "$HTTP_BODY" + # Treat as loaded since HTTP 200 + DATA_LOADED=true + else + fail "Load accepted 0 rows" + verbose_log "$HTTP_BODY" + fi + else + fail "Data load failed (HTTP $HTTP_CODE)" + verbose_log "$HTTP_BODY" + fi +fi + +echo "" + +# ============================================================ +# Test 7: Verify File Areas After Load +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 7: Verify File Areas After Load" +echo "----------------------------------------------------------------" + +if [[ "$DATA_LOADED" = false ]]; then + skip "Data not loaded - cannot verify file areas" + skip "Data not loaded - cannot verify archive" + skip "Data not loaded - cannot check for 86-byte files" +else + # Wait a moment for file movement to complete + info "Waiting 3 seconds for file processing..." + sleep 3 + + # Check file counts after load + api_call GET "/api/v1/schemas/files-count/$DOMAIN_NAME" + + if [[ "$HTTP_CODE" = "200" ]]; then + STAGE_COUNT=$(echo "$HTTP_BODY" | jq '.stage // .stageCount // 0' 2>/dev/null || echo "0") + INGESTING_COUNT=$(echo "$HTTP_BODY" | jq '.ingesting // .ingestingCount // 0' 2>/dev/null || echo "0") + UNRESOLVED_COUNT=$(echo "$HTTP_BODY" | jq '.unresolved // .unresolvedCount // 0' 2>/dev/null || echo "0") + ARCHIVE_COUNT=$(echo "$HTTP_BODY" | jq '.archive // .archiveCount // 0' 2>/dev/null || echo "0") + + info "Post-load counts - stage=$STAGE_COUNT ingesting=$INGESTING_COUNT unresolved=$UNRESOLVED_COUNT archive=$ARCHIVE_COUNT" + + if [[ "$ARCHIVE_COUNT" -ge 1 ]]; then + pass "Archive has $ARCHIVE_COUNT file(s) after load" + else + fail "Archive has 0 files after load (expected >= 1)" + fi + + if [[ "$STAGE_COUNT" = "0" ]]; then + pass "Stage is empty after load (file moved correctly)" + else + info "Stage has $STAGE_COUNT files (may still be processing)" + fi + + if [[ "$INGESTING_COUNT" = "0" ]]; then + pass "Ingesting is empty after load (no stuck files)" + else + info "Ingesting has $INGESTING_COUNT files (may still be processing)" + fi + + if [[ "$UNRESOLVED_COUNT" = "0" ]]; then + pass "Unresolved is empty (no rejected files)" + else + fail "Unresolved has $UNRESOLVED_COUNT files (data quality issue?)" + fi + else + fail "Failed to get post-load file counts (HTTP $HTTP_CODE)" + fi + + # Check archive file listing for 86-byte bug + api_call GET "/api/v1/schemas/files/$DOMAIN_NAME?type=archive" + + if [[ "$HTTP_CODE" = "200" ]]; then + ARCHIVE_FILES=$(echo "$HTTP_BODY" | jq 'length' 2>/dev/null || echo "0") + + if [[ "$ARCHIVE_FILES" -ge 1 ]]; then + # Check for the presence of orders-001.csv + HAS_ORDERS=$(echo "$HTTP_BODY" | jq '[.[] | select(.name != null and (.name | contains("orders")))] | length' 2>/dev/null || echo "0") + if [[ "$HAS_ORDERS" -ge 1 ]]; then + pass "Archive contains orders file(s)" + else + info "Archive has files but none matching 'orders'" + verbose_log "$HTTP_BODY" + fi + + # Check file sizes - no file should be exactly 86 bytes + FILES_86_BYTES=$(echo "$HTTP_BODY" | jq '[.[] | select(.fileSizeInBytes == 86 or .size == 86)] | length' 2>/dev/null || echo "0") + if [[ "$FILES_86_BYTES" = "0" ]]; then + pass "No 86-byte files in archive (chunked encoding bug NOT present)" + else + fail "CRITICAL: Found $FILES_86_BYTES file(s) with exactly 86 bytes in archive (chunked encoding corruption)" + echo "$HTTP_BODY" | jq '.[] | select(.fileSizeInBytes == 86 or .size == 86)' 2>/dev/null || true + fi + + # Check that archived file is reasonably sized (orders-001.csv is ~164KB) + LARGE_FILES=$(echo "$HTTP_BODY" | jq '[.[] | select((.fileSizeInBytes // .size // 0) > 1000)] | length' 2>/dev/null || echo "0") + if [[ "$LARGE_FILES" -ge 1 ]]; then + pass "Archive contains files > 1KB (data integrity OK)" + else + fail "No files > 1KB in archive (possible data corruption)" + verbose_log "$HTTP_BODY" + fi + else + fail "No files in archive listing" + fi + else + fail "Failed to list archive files (HTTP $HTTP_CODE)" + fi +fi + +echo "" + +# ============================================================ +# Test 8: Verify S3 Directory Markers +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 8: S3 Directory Markers (chunked encoding check)" +echo "----------------------------------------------------------------" + +if [[ "$SKIP_S3" = true ]]; then + skip "S3 checks skipped (--skip-s3)" +elif ! command -v aws &>/dev/null; then + skip "aws CLI not found (install with: brew install awscli)" +else + # Port-forward SeaweedFS S3 API + info "Setting up port-forward to SeaweedFS S3 (localhost:$SEAWEEDFS_S3_PORT)..." + + # Check if SeaweedFS service exists + SEAWEEDFS_SVC=$(kubectl get svc starlake-seaweedfs -n "$NAMESPACE" --no-headers 2>/dev/null || echo "") + if [[ -z "$SEAWEEDFS_SVC" ]]; then + skip "SeaweedFS service not found in namespace '$NAMESPACE'" + else + kubectl port-forward svc/starlake-seaweedfs "$SEAWEEDFS_S3_PORT":8333 -n "$NAMESPACE" >/dev/null 2>&1 & + S3_PF_PID=$! + PF_PIDS+=("$S3_PF_PID") + sleep 3 + + # Verify port-forward is alive + if ! kill -0 "$S3_PF_PID" 2>/dev/null; then + fail "Port-forward to SeaweedFS S3 failed (port $SEAWEEDFS_S3_PORT may be in use)" + else + pass "SeaweedFS S3 port-forward active on localhost:$SEAWEEDFS_S3_PORT" + + # Determine the project root prefix in S3 + # Projects are stored under a path like: /datasets/... + # We need to find what prefix the current project uses + S3_ENDPOINT="http://localhost:$SEAWEEDFS_S3_PORT" + export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" + export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" + + # List bucket contents to find the project prefix + BUCKET_CONTENTS=$(aws --endpoint-url "$S3_ENDPOINT" s3 ls "s3://$S3_BUCKET/" --no-sign-request 2>/dev/null || \ + aws --endpoint-url "$S3_ENDPOINT" s3 ls "s3://$S3_BUCKET/" 2>/dev/null || echo "") + + if [[ -n "$BUCKET_CONTENTS" ]]; then + pass "S3 bucket '$S3_BUCKET' accessible" + verbose_log "$BUCKET_CONTENTS" + else + info "Could not list S3 bucket (may require specific prefix)" + fi + + # Check directory markers for the test domain + # Directory markers are objects with the same name as the "directory" + # In S3, directories are virtual - they should NOT exist as objects, + # or if they do, they should have ContentLength == 0 (not 86) + MARKER_DIRS=("stage/$DOMAIN_NAME" "ingesting/$DOMAIN_NAME" "archive/$DOMAIN_NAME") + MARKER_BUG_FOUND=false + + # Try common prefixes + PREFIXES=("" "datasets/") + + # Also try to find project-specific prefix from the project info + if [[ -n "$PROJECT_ID" ]]; then + PREFIXES+=("${PROJECT_ID}/datasets/" "${PROJECT_NAME}/datasets/") + fi + + for prefix in "${PREFIXES[@]}"; do + for marker_dir in "${MARKER_DIRS[@]}"; do + FULL_KEY="${prefix}${marker_dir}" + + HEAD_OUTPUT=$(aws --endpoint-url "$S3_ENDPOINT" \ + s3api head-object \ + --bucket "$S3_BUCKET" \ + --key "$FULL_KEY" 2>/dev/null || echo "NOT_FOUND") + + if [[ "$HEAD_OUTPUT" = "NOT_FOUND" ]]; then + # Object doesn't exist - this is fine (preferred behavior) + continue + fi + + CONTENT_LENGTH=$(echo "$HEAD_OUTPUT" | jq -r '.ContentLength // 0' 2>/dev/null || echo "0") + + if [[ "$CONTENT_LENGTH" = "86" ]]; then + fail "CRITICAL: Directory marker '$FULL_KEY' has ContentLength=86 (chunked encoding corruption)" + MARKER_BUG_FOUND=true + elif [[ "$CONTENT_LENGTH" = "0" ]]; then + info "Directory marker '$FULL_KEY' exists with ContentLength=0 (acceptable)" + else + info "Object '$FULL_KEY' has ContentLength=$CONTENT_LENGTH" + fi + done + done + + if [[ "$MARKER_BUG_FOUND" = false ]]; then + pass "No 86-byte directory markers found (chunked encoding bug NOT present)" + fi + + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + fi + fi +fi + +echo "" + +# ============================================================ +# Test 9: SeaweedFS Filer Accessibility +# ============================================================ + +echo "----------------------------------------------------------------" +echo " Test 9: SeaweedFS Filer Accessibility" +echo "----------------------------------------------------------------" + +if [[ "$SKIP_S3" = true ]]; then + skip "S3 checks skipped (--skip-s3)" +else + SEAWEEDFS_SVC=$(kubectl get svc starlake-seaweedfs -n "$NAMESPACE" --no-headers 2>/dev/null || echo "") + if [[ -z "$SEAWEEDFS_SVC" ]]; then + skip "SeaweedFS service not found in namespace '$NAMESPACE'" + else + info "Setting up port-forward to SeaweedFS Filer (localhost:$SEAWEEDFS_FILER_PORT)..." + kubectl port-forward svc/starlake-seaweedfs "$SEAWEEDFS_FILER_PORT":8888 -n "$NAMESPACE" >/dev/null 2>&1 & + FILER_PF_PID=$! + PF_PIDS+=("$FILER_PF_PID") + sleep 3 + + if ! kill -0 "$FILER_PF_PID" 2>/dev/null; then + fail "Port-forward to SeaweedFS Filer failed (port $SEAWEEDFS_FILER_PORT may be in use)" + else + # Check Filer UI root + FILER_ROOT_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$SEAWEEDFS_FILER_PORT/" 2>/dev/null || echo "000") + if [[ "$FILER_ROOT_CODE" = "200" ]]; then + pass "SeaweedFS Filer UI accessible (HTTP $FILER_ROOT_CODE)" + else + fail "SeaweedFS Filer UI not accessible (HTTP $FILER_ROOT_CODE)" + fi + + # Check Filer for the bucket path + FILER_BUCKET_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$SEAWEEDFS_FILER_PORT/buckets/$S3_BUCKET/" 2>/dev/null || echo "000") + if [[ "$FILER_BUCKET_CODE" = "200" ]]; then + pass "Filer bucket path '/buckets/$S3_BUCKET/' accessible (HTTP $FILER_BUCKET_CODE)" + else + info "Filer bucket path HTTP $FILER_BUCKET_CODE (bucket may use different path)" + fi + + # Try to fetch the archive directory via Filer JSON API + if [[ "$DATA_LOADED" = true ]]; then + # SeaweedFS Filer provides JSON listing with ?pretty=y + FILER_ARCHIVE_BODY=$(curl -s "http://localhost:$SEAWEEDFS_FILER_PORT/buckets/$S3_BUCKET/" \ + -H "Accept: application/json" 2>/dev/null || echo "") + + if [[ -n "$FILER_ARCHIVE_BODY" ]] && echo "$FILER_ARCHIVE_BODY" | jq '.' &>/dev/null; then + FILER_ENTRIES=$(echo "$FILER_ARCHIVE_BODY" | jq '.Entries // .entries // [] | length' 2>/dev/null || echo "0") + if [[ "$FILER_ENTRIES" -gt 0 ]]; then + pass "Filer shows $FILER_ENTRIES entries in bucket root" + else + info "Filer shows 0 entries (files may be in subdirectories)" + fi + else + info "Filer response is not JSON (HTML UI returned instead)" + # The HTML UI being returned is fine - it means the Filer is working + if echo "$FILER_ARCHIVE_BODY" | grep -qi "html\|SeaweedFS" 2>/dev/null; then + pass "Filer returns HTML UI (service is functional)" + fi + fi + fi + fi + fi +fi + +echo "" + +# ============================================================ +# Cleanup (optional) +# ============================================================ + +if [[ "$CLEANUP_AFTER" = true ]]; then + echo "----------------------------------------------------------------" + echo " Cleanup" + echo "----------------------------------------------------------------" + + # Delete the test domain + api_call POST "/api/v1/load/$DOMAIN_NAME/delete" \ + -H "Content-Type: application/json" + + # Try alternative delete endpoint if first fails + if [[ "$HTTP_CODE" != "200" ]]; then + api_call DELETE "/api/v1/load/$DOMAIN_NAME" + fi + + if [[ "$HTTP_CODE" = "200" ]]; then + pass "Test domain '$DOMAIN_NAME' deleted" + else + info "Could not auto-delete domain '$DOMAIN_NAME' (HTTP $HTTP_CODE)" + info "Delete manually via the UI or API." + fi + + echo "" +fi + +# ============================================================ +# Summary +# ============================================================ + +echo "================================================================" +echo " Results" +echo "================================================================" +echo "" +echo -e " ${GREEN}PASSED${NC}: $PASS" +echo -e " ${RED}FAILED${NC}: $FAIL" +echo -e " ${YELLOW}SKIPPED${NC}: $SKIP" +echo "" + +TOTAL=$((PASS + FAIL + SKIP)) +echo " Total: $TOTAL tests" +echo "" + +if [[ $FAIL -gt 0 ]]; then + echo -e " ${RED}${BOLD}REGRESSION DETECTED${NC} - $FAIL test(s) failed" + echo "" + echo " Hints:" + echo " - 86-byte bug: Check Transfer-Encoding handling in UI S3 proxy" + echo " - Empty domain phantom files: S3 directory marker created as object" + echo " - Load failures: Verify SeaweedFS S3 API is configured correctly" + echo " - Run with --verbose for detailed response bodies" + echo "" + exit 1 +else + echo -e " ${GREEN}${BOLD}ALL TESTS PASSED${NC}" + echo "" + if [[ "$CLEANUP_AFTER" = false ]]; then + echo " Test domain '$DOMAIN_NAME' was NOT cleaned up." + echo " Run with --cleanup to auto-delete, or delete manually." + fi + echo "" + exit 0 +fi diff --git a/helm/validate-chart.sh b/helm/validate-chart.sh new file mode 100755 index 0000000..3f1c060 --- /dev/null +++ b/helm/validate-chart.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Script de validation du Helm chart Starlake + +set -e + +CHART_DIR="./starlake" +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Validation du Helm Chart Starlake" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +# Check if helm is installed +if ! command -v helm &> /dev/null; then + echo -e "${RED}✗ Helm n'est pas installé${NC}" + echo " Installer avec: brew install helm (macOS) ou voir https://helm.sh/docs/intro/install/" + exit 1 +fi +echo -e "${GREEN}✓ Helm est installé${NC} ($(helm version --short))" + +# Check if chart directory exists +if [ ! -d "$CHART_DIR" ]; then + echo -e "${RED}✗ Répertoire du chart non trouvé: $CHART_DIR${NC}" + exit 1 +fi +echo -e "${GREEN}✓ Répertoire du chart trouvé${NC}" + +# Lint the chart +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 1. Validation de la syntaxe (helm lint)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +if helm lint "$CHART_DIR"; then + echo -e "${GREEN}✓ Lint réussi${NC}" +else + echo -e "${RED}✗ Erreurs de lint détectées${NC}" + exit 1 +fi + +# Validate templates +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 2. Validation des templates" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# Test with default values +echo -e "${YELLOW}[Test 1/4]${NC} Configuration par défaut (PostgreSQL interne)" +helm template test-starlake "$CHART_DIR" > /dev/null +echo -e "${GREEN}✓ Templates valides avec configuration par défaut${NC}" + +# Test with external PostgreSQL +echo -e "${YELLOW}[Test 2/4]${NC} Configuration avec PostgreSQL externe" +helm template test-starlake "$CHART_DIR" \ + --set postgresql.external.enabled=true \ + --set postgresql.external.host=my-postgres.example.com \ + --set postgresql.internal.enabled=false > /dev/null +echo -e "${GREEN}✓ Templates valides avec PostgreSQL externe${NC}" + +# Test with Ingress enabled +echo -e "${YELLOW}[Test 3/4]${NC} Configuration avec Ingress activé" +helm template test-starlake "$CHART_DIR" \ + --set ingress.enabled=true \ + --set ingress.host=starlake.example.com \ + --set proxy.service.type=ClusterIP > /dev/null +echo -e "${GREEN}✓ Templates valides avec Ingress${NC}" + +# Test with development values +echo -e "${YELLOW}[Test 4/4]${NC} Configuration développement" +helm template test-starlake "$CHART_DIR" \ + --values "$CHART_DIR/values-development.yaml" > /dev/null +echo -e "${GREEN}✓ Templates valides avec values-development.yaml${NC}" + +# Check required files +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 3. Vérification des fichiers requis" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +required_files=( + "$CHART_DIR/Chart.yaml" + "$CHART_DIR/values.yaml" + "$CHART_DIR/templates/_helpers.tpl" + "$CHART_DIR/templates/secrets.yaml" + "$CHART_DIR/templates/configmap.yaml" + "$CHART_DIR/templates/pvc.yaml" + "$CHART_DIR/templates/database/statefulset.yaml" + "$CHART_DIR/templates/database/service.yaml" + "$CHART_DIR/templates/ui/deployment.yaml" + "$CHART_DIR/templates/ui/service.yaml" + "$CHART_DIR/templates/airflow/deployment.yaml" + "$CHART_DIR/templates/airflow/service.yaml" + "$CHART_DIR/templates/airflow/init-job.yaml" + "$CHART_DIR/templates/agent/deployment.yaml" + "$CHART_DIR/templates/agent/service.yaml" + "$CHART_DIR/templates/proxy/deployment.yaml" + "$CHART_DIR/templates/proxy/service.yaml" + "$CHART_DIR/templates/ingress.yaml" + "$CHART_DIR/templates/serviceaccount.yaml" + "$CHART_DIR/templates/NOTES.txt" +) + +missing_files=0 +for file in "${required_files[@]}"; do + if [ -f "$file" ]; then + echo -e "${GREEN}✓${NC} $file" + else + echo -e "${RED}✗${NC} $file ${RED}(manquant)${NC}" + ((missing_files++)) + fi +done + +if [ $missing_files -gt 0 ]; then + echo -e "${RED}✗ $missing_files fichiers manquants${NC}" + exit 1 +fi + +# Check scripts +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 4. Vérification des scripts" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +required_scripts=( + "$CHART_DIR/scripts/init-airflow-database.sh" + "$CHART_DIR/scripts/starlake.sh" +) + +missing_scripts=0 +for script in "${required_scripts[@]}"; do + if [ -f "$script" ]; then + echo -e "${GREEN}✓${NC} $script" + else + echo -e "${RED}✗${NC} $script ${RED}(manquant)${NC}" + ((missing_scripts++)) + fi +done + +if [ $missing_scripts -gt 0 ]; then + echo -e "${RED}✗ $missing_scripts scripts manquants${NC}" + exit 1 +fi + +# Dry-run install test +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 5. Test d'installation (dry-run)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +if helm install starlake-test "$CHART_DIR" --dry-run --debug > /dev/null 2>&1; then + echo -e "${GREEN}✓ Dry-run install réussi${NC}" +else + echo -e "${RED}✗ Dry-run install échoué${NC}" + helm install starlake-test "$CHART_DIR" --dry-run --debug + exit 1 +fi + +# Success +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo -e "${GREEN}✓ Toutes les validations ont réussi !${NC}" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +echo "Le chart est prêt à être déployé !" +echo "" +echo "Prochaines étapes :" +echo " 1. Tester sur un cluster local : helm install starlake ./starlake -n starlake --create-namespace" +echo " 2. Personnaliser values.yaml pour votre environnement" +echo " 3. Déployer en production avec : helm install starlake ./starlake -f values-production.yaml" +echo "" diff --git a/helm/values-k3s-test.yaml b/helm/values-k3s-test.yaml new file mode 100644 index 0000000..38d91f2 --- /dev/null +++ b/helm/values-k3s-test.yaml @@ -0,0 +1,95 @@ +# K3s Test Values Override +# This file is specifically for local testing with K3s +# WARNING: DO NOT use these values in production! + +# Use local-path storage (K3s built-in) +global: + storageClass: local-path + +# PostgreSQL with reduced resources and RWO storage +postgresql: + internal: + enabled: true + persistence: + enabled: true + storageClass: local-path + size: 2Gi + # Note: local-path only supports ReadWriteOnce + +# Projects volume - TEMPORARILY using RWO for K3s testing +# In production, this MUST be ReadWriteMany +persistence: + projects: + enabled: true + storageClass: local-path + size: 2Gi + # HACK: We'll use RWO and force all pods to same node via affinity + +# Disable logs persistence to simplify +airflow: + logs: + persistence: + enabled: false + +# Reduced resources for local testing +ui: + replicas: 1 + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1000m + +airflow: + webserver: + replicas: 1 + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1000m + scheduler: + resources: + requests: + memory: 256Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 1000m + +agent: + replicas: 1 + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 512Mi + cpu: 500m + +proxy: + replicas: 1 + resources: + requests: + memory: 64Mi + cpu: 50m + limits: + memory: 128Mi + cpu: 200m + +# Force all pods to same node for shared RWO volume +# This is a HACK for testing only! +affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - starlake + topologyKey: kubernetes.io/hostname diff --git a/s3-dags-stack.sh b/s3-dags-stack.sh index e8c10c3..2143a69 100755 --- a/s3-dags-stack.sh +++ b/s3-dags-stack.sh @@ -1 +1,4 @@ -SL_API_APP_TYPE=ducklake docker compose --profile airflow --profile minio --profile gizmo up --build \ No newline at end of file +#!/usr/bin/env bash +set -euo pipefail + +SL_API_APP_TYPE=ducklake docker compose --profile airflow --profile minio --profile gizmo up --build \ No newline at end of file diff --git a/scripts/kubernetes/starlake.sh b/scripts/kubernetes/starlake.sh new file mode 100644 index 0000000..f1ad04f --- /dev/null +++ b/scripts/kubernetes/starlake.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# Starlake CLI wrapper for Kubernetes - creates K8s Jobs for task execution +# This script is used by Dockerfile_airflow_k8s to offload starlake commands to separate K8s Jobs +# +# Required environment variables: +# STARLAKE_NAMESPACE - Kubernetes namespace for Jobs (default: from service account) +# SL_ROOT - Starlake root directory (default: /projects) +# +# Optional environment variables: +# JOB_TEMPLATE_PATH - Path to job template YAML (default: /etc/starlake/job-template.yaml) +# STARLAKE_IMAGE - Docker image for Jobs (default: same as current pod) +# STARLAKE_IMAGE_TAG - Image tag (default: latest) +# +# The script requires: +# - kubectl in PATH or common locations +# - ServiceAccount token mounted (for in-cluster auth) +# - Job template YAML at JOB_TEMPLATE_PATH + +set -e + +# Check if at least one argument is passed +if [ "$#" -eq 0 ]; then + echo "No arguments provided. Usage: starlake [args...]" + exit 1 +fi + +# Handle arguments from starlake-airflow: +# - --options may contain both: +# - SL_* variables (SL_ROOT, SL_DATASETS, etc.) -> export as env vars +# - Other options (date_min, date_max, etc.) -> pass to starlake --options +# - --scheduledDate is a native starlake option -> pass as-is +old_ifs="$IFS" +raw_options="" # Raw --options value from starlake-airflow +command="" +arguments=() + +# First argument is always the command +command="$1" +shift + +# Parse remaining arguments +while [ $# -gt 0 ]; do + case "$1" in + -o|--options) raw_options="$2"; shift 2 ;; + *) arguments+=("$1"); shift ;; + esac +done + +# Separate SL_* env vars from starlake options +env_vars=() # SL_* variables to export +starlake_options=() # Other options to pass to starlake --options + +if [ -n "$raw_options" ]; then + IFS=',' read -ra opt_array <<< "$raw_options" + for opt in "${opt_array[@]}"; do + name="${opt%%=*}" + value="${opt#*=}" + # Remove surrounding quotes (both single and double) from value + value="${value%\"}" + value="${value#\"}" + value="${value%\'}" + value="${value#\'}" + + if [[ "$name" == SL_* ]]; then + # SL_* variables -> export as environment variables + export "$name=$value" + env_vars+=("$name=$value") + else + # Other options -> pass to starlake --options + starlake_options+=("$name=$value") + fi + done + IFS="$old_ifs" +fi + +# Log what we're doing +if [ ${#env_vars[@]} -gt 0 ]; then + echo "=== Exported env vars: ${env_vars[*]} ===" +fi +if [ ${#starlake_options[@]} -gt 0 ]; then + echo "=== Starlake options: ${starlake_options[*]} ===" +fi + +# Reconstruct full args array with command first +# --scheduledDate and other native options are already in arguments[] +# Add --options only if we have starlake options to pass +if [ ${#starlake_options[@]} -gt 0 ]; then + # Join starlake_options with commas + options_str=$(IFS=','; echo "${starlake_options[*]}") + FULL_ARGS=("$command" "--options" "$options_str" "${arguments[@]}") +else + FULL_ARGS=("$command" "${arguments[@]}") +fi + +# Configuration from environment +JOB_TEMPLATE="${JOB_TEMPLATE_PATH:-/etc/starlake/job-template.yaml}" +NAMESPACE="${STARLAKE_NAMESPACE:-default}" + +# Try to get namespace from service account if not set +if [ "$NAMESPACE" = "default" ] && [ -f /var/run/secrets/kubernetes.io/serviceaccount/namespace ]; then + NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) +fi + +# Find kubectl binary +KUBECTL="" +for loc in "/shared-tools/bin/kubectl" "/usr/local/bin/kubectl" "/usr/bin/kubectl" "kubectl"; do + if command -v "$loc" >/dev/null 2>&1 || [ -x "$loc" ]; then + KUBECTL="$loc" + break + fi +done + +if [ -z "$KUBECTL" ]; then + echo "ERROR: kubectl not found. Searched: /shared-tools/bin/kubectl, /usr/local/bin/kubectl, /usr/bin/kubectl, PATH" + exit 1 +fi + +# Configure kubectl for in-cluster authentication +# Use kubernetes.default.svc as fallback when env vars not available (e.g., in Airflow subprocess) +if [ -n "$KUBERNETES_SERVICE_HOST" ]; then + KUBE_API_SERVER="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" +else + KUBE_API_SERVER="https://kubernetes.default.svc:443" +fi + +# Check if service account token exists +if [ ! -f /var/run/secrets/kubernetes.io/serviceaccount/token ]; then + echo "ERROR: ServiceAccount token not found at /var/run/secrets/kubernetes.io/serviceaccount/token" + echo "Make sure automountServiceAccountToken is enabled for this pod" + exit 1 +fi + +KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) +KUBE_CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + +# kubectl wrapper function with in-cluster config +kubectl_cmd() { + "$KUBECTL" --server="$KUBE_API_SERVER" --token="$KUBE_TOKEN" --certificate-authority="$KUBE_CA_CERT" "$@" +} + +# Check if job template exists +if [ ! -f "$JOB_TEMPLATE" ]; then + echo "ERROR: Job template not found at $JOB_TEMPLATE" + echo "Set JOB_TEMPLATE_PATH environment variable to specify a different location" + exit 1 +fi + +# Generate unique job name based on command and timestamp +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +RANDOM_SUFFIX=$(printf '%04x' $RANDOM) +JOB_NAME="sl-${command}-${TIMESTAMP}-${RANDOM_SUFFIX}" +# Kubernetes job names must be lowercase and max 63 chars +JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | cut -c1-63) + +# Build args array for YAML (JSON array format) +# Proper JSON escaping to prevent command injection +ARGS_JSON="[" +FIRST=true +for arg in "${FULL_ARGS[@]}"; do + if [ "$FIRST" = true ]; then + FIRST=false + else + ARGS_JSON="${ARGS_JSON}, " + fi + # Properly escape for JSON: backslashes first, then quotes, tabs, newlines + ESCAPED_ARG=$(printf '%s' "$arg" | sed 's/\\/\\\\/g; s/"/\\"/g; s/ /\\t/g' | tr '\n' ' ') + ARGS_JSON="${ARGS_JSON}\"${ESCAPED_ARG}\"" +done +ARGS_JSON="${ARGS_JSON}]" + +# Create temporary job manifest +TEMP_JOB=$(mktemp /tmp/starlake-job-XXXXXX.yaml) +trap "rm -f $TEMP_JOB" EXIT + +# Build env vars YAML (12 spaces indent to match template) +ENV_FILE=$(mktemp /tmp/starlake-env-XXXXXX.yaml) +trap "rm -f $TEMP_JOB $ENV_FILE" EXIT + +for var in $(env | grep -E "^SL_" | grep -v "^SL_ROOT=" | cut -d= -f1); do + value="${!var}" + # Escape for YAML string: backslashes first, then double quotes + escaped_value=$(printf '%s' "$value" | sed 's/\\/\\\\/g; s/"/\\"/g') + # 12 spaces for - name:, 14 spaces for value: (matching template) + echo " - name: ${var}" >> "$ENV_FILE" + echo " value: \"${escaped_value}\"" >> "$ENV_FILE" +done + +# Read template and create job manifest +# Replace simple placeholders +sed -e "s|__JOB_NAME__|${JOB_NAME}|g" \ + -e "s|__STARLAKE_ARGS__|${ARGS_JSON}|g" \ + -e "s|__SL_ROOT__|${SL_ROOT:-/projects}|g" \ + "$JOB_TEMPLATE" > "$TEMP_JOB" + +# Replace __ENV_VARS__ with actual env vars from file +if [ -s "$ENV_FILE" ]; then + # Use awk to replace __ENV_VARS__ with file contents + awk -v envfile="$ENV_FILE" ' + /__ENV_VARS__/ { + while ((getline line < envfile) > 0) print line + close(envfile) + next + } + {print} + ' "$TEMP_JOB" > "${TEMP_JOB}.tmp" && mv "${TEMP_JOB}.tmp" "$TEMP_JOB" +else + # No env vars, just remove the placeholder + sed -i '/__ENV_VARS__/d' "$TEMP_JOB" 2>/dev/null || sed -i '' '/__ENV_VARS__/d' "$TEMP_JOB" +fi + +echo "=== Creating Kubernetes Job: ${JOB_NAME} ===" +echo "Command: starlake ${FULL_ARGS[*]}" +echo "Namespace: ${NAMESPACE}" +echo "SL_ROOT: ${SL_ROOT:-/projects}" + +# Create the job (--validate=false to avoid openapi download issues) +kubectl_cmd apply -f "$TEMP_JOB" -n "$NAMESPACE" --validate=false + +# Wait for pod to be created and get its name +echo "=== Waiting for pod to start ===" +POD_NAME="" +for i in $(seq 1 60); do + POD_NAME=$(kubectl_cmd get pods -n "$NAMESPACE" -l "job-name=${JOB_NAME}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [ -n "$POD_NAME" ]; then + POD_STATUS=$(kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Pending") + if [ "$POD_STATUS" != "Pending" ]; then + break + fi + fi + sleep 2 +done + +if [ -z "$POD_NAME" ]; then + echo "ERROR: Pod not created after 120 seconds" + kubectl_cmd get jobs -n "$NAMESPACE" -l "job-name=${JOB_NAME}" -o yaml + exit 1 +fi + +echo "=== Pod ${POD_NAME} started, streaming logs ===" + +# Stream logs (follow until completion) +kubectl_cmd logs -f "$POD_NAME" -n "$NAMESPACE" -c starlake 2>/dev/null || true + +# IMPORTANT: Capture exit code IMMEDIATELY after logs complete +# kubectl logs -f finishes when the container terminates, so this is the best time to capture +# We need to do this before TTL can delete the job/pod +echo "=== Checking job completion status ===" + +# Small delay to allow Kubernetes to update pod status +sleep 1 + +# Try to get exit code from pod immediately +IMMEDIATE_EXIT_CODE=$(kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" \ + -o jsonpath='{.status.containerStatuses[?(@.name=="starlake")].state.terminated.exitCode}' 2>/dev/null || echo "") + +if [ -n "$IMMEDIATE_EXIT_CODE" ]; then + if [ "$IMMEDIATE_EXIT_CODE" = "0" ]; then + echo "=== Job completed successfully (exit code: 0) ===" + exit 0 + else + echo "=== Job failed (exit code: $IMMEDIATE_EXIT_CODE) ===" + exit "$IMMEDIATE_EXIT_CODE" + fi +fi + +# If we couldn't get the exit code immediately, fall back to polling +# This handles cases where the container hasn't fully terminated yet +echo "=== Waiting for job completion (polling) ===" +MAX_WAIT=3600 # 1 hour max +POLL_INTERVAL=5 +WAITED=0 + +while [ $WAITED -lt $MAX_WAIT ]; do + # Check if job still exists + JOB_EXISTS=$(kubectl_cmd get job "${JOB_NAME}" -n "$NAMESPACE" -o name 2>/dev/null || echo "") + + if [ -z "$JOB_EXISTS" ]; then + # Job was deleted (by TTL after completion) - check pod exit code + echo "Job was deleted (likely completed and cleaned up by TTL)" + EXIT_CODE=$(kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || echo "") + + if [ -z "$EXIT_CODE" ]; then + # Pod also deleted, assume success if we got here (logs were streamed) + echo "=== Job completed (pod also cleaned up) ===" + exit 0 + elif [ "$EXIT_CODE" = "0" ]; then + echo "=== Job completed successfully (exit code: 0) ===" + exit 0 + else + echo "=== Job failed (exit code: $EXIT_CODE) ===" + exit "$EXIT_CODE" + fi + fi + + # Job exists, check its status + JOB_COMPLETE=$(kubectl_cmd get job "${JOB_NAME}" -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || echo "") + JOB_FAILED=$(kubectl_cmd get job "${JOB_NAME}" -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || echo "") + + if [ "$JOB_COMPLETE" = "True" ]; then + echo "=== Job completed successfully ===" + exit 0 + fi + + if [ "$JOB_FAILED" = "True" ]; then + echo "=== Job failed ===" + EXIT_CODE=$(kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.containerStatuses[0].state.terminated.exitCode}' 2>/dev/null || echo "1") + exit "${EXIT_CODE:-1}" + fi + + # Job still running, wait + sleep $POLL_INTERVAL + WAITED=$((WAITED + POLL_INTERVAL)) +done + +echo "=== Timeout waiting for job completion ===" +exit 1 diff --git a/scripts/projects/entrypoint.sh b/scripts/projects/entrypoint.sh index 122f8bf..6df10c4 100755 --- a/scripts/projects/entrypoint.sh +++ b/scripts/projects/entrypoint.sh @@ -25,13 +25,20 @@ if [[ $member_id =~ ^[0-9]+$ ]]; then project_name=$(basename "$zip" | cut -d. -f1) if [ ! -d /projects/$member_id/$project_id ]; then echo "Project $project_name will be created with id $project_id and UUID $project_uuid" - psql -v ON_ERROR_STOP=1 -h "${POSTGRES_HOST}" -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -w <<-EOSQL + # Use psql variables to prevent SQL injection + # Variables: project_id (int), project_uuid (safe), project_name (user input - must be escaped), member_id (int) + psql -v ON_ERROR_STOP=1 \ + -v p_id="$project_id" \ + -v p_uuid="$project_uuid" \ + -v p_name="$project_name" \ + -v m_id="$member_id" \ + -h "${POSTGRES_HOST}" -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -w <<-EOSQL INSERT INTO public.slk_project (id, code, "name", description, repository, active, deleted, created, updated, master, owner, owner_email, access, pat, airflow_role) -OVERRIDING SYSTEM VALUE -VALUES($project_id, '$project_uuid', '$project_name', '$project_name', '', true, false, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, -1, $member_id, 'admin@localhost.local', 'ADMIN', '', 'DEV:OPS,STAGING:OPS,PROD:OPS'); +OVERRIDING SYSTEM VALUE +VALUES(:p_id, :'p_uuid', :'p_name', :'p_name', '', true, false, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, -1, :m_id, 'admin@localhost.local', 'ADMIN', '', 'DEV:OPS,STAGING:OPS,PROD:OPS'); INSERT INTO public.slk_project_props (id, project, properties, created, updated) -OVERRIDING SYSTEM VALUE -VALUES($project_id, $project_id, '[{"envName":"__sl_ignore__"}]', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); +OVERRIDING SYSTEM VALUE +VALUES(:p_id, :p_id, '[{"envName":"__sl_ignore__"}]', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); EOSQL echo "Move project $member_id/$project_id to target folder" mkdir -p /projects/$member_id/$project_id @@ -59,5 +66,6 @@ EOSQL echo "All projects have been unzipped" echo "You can now access Starlake at http://localhost:${SL_UI_PORT:-80}" else + echo "ERROR: Could not find admin member (admin@localhost.local) in database. Is the database initialized?" exit 1 fi