Feature: add Helm chart support

.gitignore

@@ -9,3 +9,17 @@ logs
 pgdata
 projects
 .env
+.idea
+*.iml
+
+# macOS
+.DS_Store
+**/.DS_Store
+
+# Claude Code - Local only (don't track)
+*.local.md
+.claude/settings.local.json
+.claude/scripts/
+.vscode
+**/**/CLAUDE.md
+tpch001

Dockerfile_airflow

@@ -39,10 +39,18 @@ USER airflow
 # Install aws cli
 RUN pip install --no-cache-dir -U awscli
 
-# Configure aws cli
-RUN mkdir -p /home/airflow/.aws
-COPY conf/aws/credentials /home/airflow/.aws/credentials
-COPY conf/aws/config /home/airflow/.aws/config
+# NOTE: AWS credentials should NOT be baked into the image.
+# For Kubernetes: Use IRSA (EKS), Workload Identity (GKE), or mount credentials via K8s Secret.
+# For Docker Compose: Mount ~/.aws as a volume or use environment variables.
+# Example K8s Secret mount:
+#   volumes:
+#     - name: aws-credentials
+#       secret:
+#         secretName: aws-credentials
+#   volumeMounts:
+#     - name: aws-credentials
+#       mountPath: /home/airflow/.aws
+#       readOnly: true
 
 # Install airflow amazon and google providers
 RUN pip install --no-cache-dir \

Dockerfile_airflow3

@@ -41,10 +41,18 @@ USER airflow
 # Install aws cli
 RUN pip install --no-cache-dir -U awscli
 
-# Configure aws cli
-RUN mkdir -p /home/airflow/.aws
-COPY conf/aws/credentials /home/airflow/.aws/credentials
-COPY conf/aws/config /home/airflow/.aws/config
+# NOTE: AWS credentials should NOT be baked into the image.
+# For Kubernetes: Use IRSA (EKS), Workload Identity (GKE), or mount credentials via K8s Secret.
+# For Docker Compose: Mount ~/.aws as a volume or use environment variables.
+# Example K8s Secret mount:
+#   volumes:
+#     - name: aws-credentials
+#       secret:
+#         secretName: aws-credentials
+#   volumeMounts:
+#     - name: aws-credentials
+#       mountPath: /home/airflow/.aws
+#       readOnly: true
 
 # Install airflow amazon, google and fab providers
 RUN pip install --no-cache-dir \

Dockerfile_airflow_k8s

@@ -0,0 +1,102 @@
+# Dockerfile for Airflow on Kubernetes with K8s Job execution
+# Use this instead of Dockerfile_airflow for Kubernetes deployments
+#
+# This image creates K8s Jobs for each starlake command, offloading heavy
+# processing from the Airflow pod. Requires:
+# - ServiceAccount with RBAC permissions to create/manage Jobs
+# - Job template ConfigMap mounted at /etc/starlake/job-template.yaml
+# - automountServiceAccountToken: true
+
+# Stage 1: Copy Starlake CLI from UI image
+FROM starlakeai/starlake-1.5-ui:1.5 AS starlake-cli
+
+# Stage 2: Build Airflow image with Starlake CLI
+FROM apache/airflow:2.11.0
+
+# Switch to root user to install additional packages
+USER root
+
+# Install NFS client utilities and kubectl for K8s Job creation
+# kubectl is downloaded with SHA256 checksum verification for supply chain security
+RUN apt-get update \
+    && apt-get install -y nfs-common \
+    mandoc \
+    less \
+    curl \
+    && KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) \
+    && ARCH=$(dpkg --print-architecture || echo "amd64") \
+    && curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" \
+    && curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl.sha256" \
+    && echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check \
+    && chmod +x kubectl \
+    && mv kubectl /usr/local/bin/kubectl \
+    && rm kubectl.sha256 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD conf/airflow/webserver_config.py /opt/airflow/webserver_config.py
+
+# Required to mount NFS volumes
+RUN echo "airflow ALL=(ALL:ALL) NOPASSWD: ALL"  > /etc/sudoers.d/airflow
+
+# Copy the actual Starlake CLI (Java) from UI image
+COPY --from=starlake-cli /app/starlake /app/starlake
+
+# Install SL CLI wrapper for Kubernetes Job execution
+# This wrapper creates K8s Jobs to offload starlake commands from the Airflow pod
+# IMPORTANT: We rename the original starlake to starlake-original and replace it with our wrapper
+# because Airflow DAGs call /app/starlake/starlake directly (not /usr/local/bin/starlake)
+RUN mv /app/starlake/starlake /app/starlake/starlake-original 2>/dev/null || true
+COPY scripts/kubernetes/starlake.sh /app/starlake/starlake
+RUN chmod +x /app/starlake/starlake \
+    && chmod +x /app/starlake/starlake.sh \
+    && chmod +x /app/starlake/starlake-original 2>/dev/null || true \
+    && ln -sf /app/starlake/starlake /usr/local/bin/starlake \
+    && ln -sf /app/starlake/starlake /usr/local/bin/starlake-k8s
+
+# Copy Java runtime from UI image (required for starlake CLI)
+COPY --from=starlake-cli /opt/java /opt/java
+ENV JAVA_HOME=/opt/java/openjdk
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+
+# Make JAVA_HOME available in all subshells (for Airflow BashOperator/subprocess)
+RUN echo "export JAVA_HOME=/opt/java/openjdk" >> /etc/bash.bashrc \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/bash.bashrc \
+    && echo "export JAVA_HOME=/opt/java/openjdk" >> /etc/profile.d/java.sh \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile.d/java.sh \
+    && chmod +x /etc/profile.d/java.sh
+
+# Install gcloud sdk
+RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz \
+    && mkdir -p /usr/local/gcloud \
+    && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
+    && ln -s /usr/local/gcloud/google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud \
+    && rm /tmp/google-cloud-sdk.tar.gz
+
+# Switch back to the airflow user
+USER airflow
+
+# Install aws cli
+RUN pip install --no-cache-dir -U awscli
+
+# NOTE: AWS credentials should NOT be baked into the image.
+# For Kubernetes: Use IRSA (EKS), Workload Identity (GKE), or mount credentials via K8s Secret.
+# Example K8s Secret mount in Helm values.yaml:
+#   airflow:
+#     extraVolumes:
+#       - name: aws-credentials
+#         secret:
+#           secretName: aws-credentials
+#     extraVolumeMounts:
+#       - name: aws-credentials
+#         mountPath: /home/airflow/.aws
+#         readOnly: true
+
+# Install airflow amazon and google providers
+RUN pip install --no-cache-dir \
+    apache-airflow-providers-amazon \
+    apache-airflow-providers-google
+
+# Install SL Python libraries for Airflow 2 (no docker package needed in K8s)
+# IMPORTANT: Pin starlake-airflow~=0.4 for Airflow 2 compatibility (0.5+ requires Airflow 3)
+RUN pip install --no-cache-dir "starlake-airflow>=0.4,<0.5"

Dockerfile_dagster

@@ -1,4 +1,5 @@
-FROM debian:11-slim
+# Security fix: Debian 12 (bookworm) for latest security patches
+FROM debian:12-slim
 
 # Add Docker's official GPG key:
 RUN apt update \

Dockerfile_projects

@@ -1,4 +1,5 @@
-FROM alpine:latest
+# Security fix: Pin Alpine version for reproducible builds and security tracking
+FROM alpine:3.21
 
 RUN apk add --no-cache --no-progress \
     nfs-utils \

README.md

@@ -91,7 +91,7 @@ Starlake uses Docker Compose **profiles** to manage different configurations (e.
 To start the Pragmatic Duck Data Stack with Airflow and Gizmo on local file system, use the following command:
 
 ```bash
-COMPOSE_PROFILES=airflow,gizmo SL_API_APP_TYPE=ducklake docker compose up  --build
+COMPOSE_PROFILES=airflow,minio,gizmo SL_API_APP_TYPE=ducklake docker compose up  --build
 ```
 
 or simply
@@ -100,12 +100,6 @@ or simply
 $ ./dags-stack.sh
 ```
 
-To start the Pragmatic Duck Data Stack with Airflow & Minio and Gizmo, use the following command:
-
-```bash
-COMPOSE_PROFILES=airflow,minio,gizmo SL_API_APP_TYPE=ducklake docker compose up  --build
-```
-
 To start the stack with a specific profile (e.g., `airflow`) and address any Cloud Datawarehouses, use the following commands:
 
 ```bash
@@ -163,48 +157,11 @@ Once up, the services are accessible at the following default URLs:
 
 > If you are affected by this [Docker issue](https://github.com/docker/for-mac/issues/7583), please upgrade your Docker install.
 
-## Mounting external projects
-
-If you have any starlake container projects and want to mount it:
-
-- run `setup_mac_nfs.sh` if you are on mac in order to expose your folder via NFS.
-  Modify the root folder to share if necessary. By default it is set to /user.
-  This change is not specific to starlake and may be used in other container.
-- comment `- external_projects_data:/external_projects` in the `volumes` section of the starlake-nas container
-- uncomment `- starlake-prj-nfs-mount:/external_projects` right below the line above in the docker compose file
-- go to the end of the file and comment uncomment the `starlake-prj-nfs-mount:` section as follows:
-
-```
-  starlake-prj-nfs-mount:
-    driver: local
-    driver_opts:
-      type: nfs
-      o: addr=host.docker.internal,rw,nolock,hard,nointr,nfsvers=3
-      device: ":/path_to_starlake_project_container" # absolute path to folder on your host where projects are located.
-```
-
-Starlake container folder should contain the starlake project folder:
-
-```
- /path_to_starlake_project_container
-   |
-    - my_first_starlake_project
-        |
-         - metadata
-         - ...
-   |
-    - my_second_starlake_project
-        |
-         - metadata
-         - ...
-```
-
-If you have many container projects, create as many volume as needed.
 
 ## Stopping Starlake UI
 
 To stop Starlake UI, run the following command in the same directory
 
 ```bash
 docker compose down
-```
+```

cloud-data-stack.sh

@@ -1 +1,4 @@
-docker compose --profile airflow up  --build
+#!/usr/bin/env bash
+set -euo pipefail
+
+docker compose --profile airflow up --build

dags-stack.sh

@@ -1 +1,4 @@
-COMPOSE_PROFILES=airflow,gizmo SL_API_APP_TYPE=ducklake docker compose up  --build
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMPOSE_PROFILES=airflow,gizmo SL_API_APP_TYPE=ducklake docker compose up --build

docker-compose-dagster.yml

@@ -4,7 +4,8 @@ version: '3.8'
 services:
 
   starlake-db:
-    image: postgres:17
+    # Security fix: Pin PostgreSQL to patch version for security tracking
+    image: postgres:17.2
     restart: on-failure
     container_name: starlake-db
     ports:
@@ -172,4 +173,4 @@ services:
       - SL_ASK_API_DOCS_PASSWORD=s3cret.Paw
 
 volumes:
-  pgdata:
+  pgdata:

docker-compose-snowflake.yml

@@ -4,7 +4,8 @@ version: '3.8'
 services:
 
   starlake-db:
-    image: postgres:17
+    # Security fix: Pin PostgreSQL to patch version for security tracking
+    image: postgres:17.2
     restart: on-failure
     container_name: starlake-db
     ports:
@@ -137,4 +138,4 @@ services:
       - SL_ASK_API_DOCS_PASSWORD=s3cret.Paw
 
 volumes:
-  pgdata:
+  pgdata:

docker-compose.yml

@@ -31,7 +31,6 @@ x-starlake-airflow:
       - ${PROJECTS_DATA_PATH:-./projects}:/projects
       - ${PROJECTS_DATA_PATH:-./projects}/dags:/opt/airflow/dags
       - ${AIRFLOW_LOGS:-./airflow/logs}:/opt/airflow/logs
-      - /Users/hayssams/.config:/home/airflow/.config
 
 x-starlake-ui-common:
   &starlake-ui-common
@@ -51,7 +50,7 @@ x-starlake-ui-common:
     interval: "5s"
     retries: 60
   ports:
-    - ${SL_PORT:-80}:9900  # starlake-ui default port
+    - ${SL_PORT:-80}:9900 # starlake-ui default port
   environment:
     &starlake-ui-common-env
     SL_API_GIZMO_ON_DEMAND_URL: http://starlake-gizmo:10900
@@ -135,7 +134,8 @@ services:
       - airflow3
       - dagster
       - snowflake
-    image: postgres:17
+    # Security fix: Pin PostgreSQL to patch version for security tracking
+    image: postgres:17.2
     restart: on-failure
     container_name: starlake-db
     ports:
@@ -409,7 +409,8 @@ services:
   minio:
     profiles:
       - minio
-    image: "quay.io/minio/minio:latest"
+    # Security fix: Chainguard MinIO - official quay.io/minio/minio is unmaintained since Oct 2025
+    image: "cgr.dev/chainguard/minio:latest"
     ports:
       - "${SL_MINIO_PORT:-9000}:9000"
       - "${SL_MINIO_CONSOLE_PORT:-9001}:9001"

helm/README.md

@@ -0,0 +1,36 @@
+# Starlake Helm Chart
+
+This directory contains the official Helm chart for deploying the Starlake Data Stack on Kubernetes.
+
+## Documentation
+
+- **[Chart Documentation](starlake/README.md)** - Complete chart documentation (installation, configuration, parameters)
+- **[Quick Start Guide](docs/QUICKSTART.md)** - Step-by-step deployment guides
+- **[Local Testing](docs/LOCAL_TESTING.md)** - Test locally with K3s/K3d
+
+## Quick Start
+
+```bash
+# Automated test script (creates K3s cluster, installs chart, validates)
+./test-helm-chart.sh
+
+# Manual installation
+helm install starlake ./starlake \
+  --namespace starlake \
+  --create-namespace
+```
+
+## Directory Structure
+
+```
+helm/
+├── starlake/              # The Helm chart
+│   ├── Chart.yaml         # Chart metadata
+│   ├── values.yaml        # Default configuration
+│   ├── templates/         # Kubernetes templates
+│   └── README.md          # Chart documentation
+├── docs/                  # Additional documentation
+└── test-helm-chart.sh     # Automated test script
+```
+
+For full documentation, see [starlake/README.md](starlake/README.md).