CI/CD Pipeline k8s

Author: Dev Nirwal

.github/workflows/deploy.yml

@@ -0,0 +1,67 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ "main" ] # Trigger the pipeline on pushes to the main branch
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  # GHCR image name: ghcr.io/owner/repo-name
+  IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/simple-rag
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write # Required for pushing to GHCR
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            ${{ env.IMAGE_NAME }}:latest
+            ${{ env.IMAGE_NAME }}:${{ github.sha }}
+          # Ensure image is lowered for GHCR if repository name contains uppercase
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+
+  # Optional job to automatically update kubernetes deployment.
+  # This requires a kubernetes cluster setup and kubeconfig secret.
+  deploy:
+    needs: build-and-push
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      
+      # Example step using an external tool to deploy, or setup kubectl
+      - name: Set up kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: 'latest'
+          
+      - name: Configure Kubernetes context
+        uses: azure/k8s-set-context@v3
+        with:
+           method: kubeconfig
+           kubeconfig: ${{ secrets.KUBECONFIG }}
+           
+      - name: Update deployment image
+        run: |
+          kubectl set image deployment/simple-rag-deployment simple-rag=${{ env.IMAGE_NAME }}:${{ github.sha }}
+          kubectl rollout status deployment/simple-rag-deployment

.gitignore

@@ -0,0 +1,38 @@
+# Django
+*.log
+*.pot
+*.pyc
+__pycache__/
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+media/
+staticfiles/
+vector_stores/
+
+# OS
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+v/
+
+# IDEs
+.vscode/
+.idea/
+
+# Docker
+.dockerignore
+
+# Kubernetes
+k8s/secrets.yaml

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+
+# Install system dependencies including sqlite3
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    sqlite3 \
+    libsqlite3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Create a data directory that we will mount as a persistent volume
+RUN mkdir -p /app/data
+
+# Expose port
+EXPOSE 8000
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV DJANGO_SETTINGS_MODULE="pdf_chat_project.settings"
+# Default DATA_DIR (can be overridden by k8s but this is a fallback)
+ENV DATA_DIR="/app/data"
+
+# Command to run the application using Gunicorn
+CMD ["sh", "-c", "python manage.py migrate && python manage.py collectstatic --noinput && gunicorn pdf_chat_project.wsgi:application --bind 0.0.0.0:8000 --workers 3"]

chat/views.py

@@ -22,7 +22,7 @@ def upload_pdf(request):
             pdf_doc = form.save()
 
             # Ensure vector_stores directory exists
-            vector_store_root = os.path.join(settings.BASE_DIR, 'vector_stores')
+            vector_store_root = os.path.join(getattr(settings, 'DATA_DIR', settings.BASE_DIR), 'vector_stores')
             if not os.path.exists(vector_store_root):
                 os.makedirs(vector_store_root)
 

k8s/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: simple-rag-deployment
+  labels:
+    app: simple-rag
+spec:
+  replicas: 1 # Important: Because we use SQLite, we shouldn't scale beyond 1 replica safely without a shared volume that supports multiple concurrent writers properly, and even then SQLite handles it poorly. If you need more replicas, migrate to PostgreSQL.
+  selector:
+    matchLabels:
+      app: simple-rag
+  template:
+    metadata:
+      labels:
+        app: simple-rag
+    spec:
+      containers:
+        - name: simple-rag
+          # UPDATE: Replace <GITHUB_OWNER> with your GitHub username or organization name
+          image: ghcr.io/<GITHUB_OWNER>/simple-rag:latest 
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8000
+          env:
+            - name: DATA_DIR
+              value: "/app/data"
+            - name: DEBUG
+              value: "False"
+            - name: ALLOWED_HOSTS
+              value: "*" 
+            - name: GOOGLE_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: simple-rag-secrets
+                  key: google-api-key
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: simple-rag-secrets
+                  key: database-url
+          volumeMounts:
+            - name: data-volume
+              mountPath: /app/data
+      volumes:
+        - name: data-volume
+          persistentVolumeClaim:
+            claimName: simple-rag-pvc

k8s/postgres-deployment.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:15
+          ports:
+            - containerPort: 5432
+          env:
+            - name: POSTGRES_DB
+              value: "simple_rag_db"
+            - name: POSTGRES_USER
+              value: "rag_user"
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: simple-rag-secrets
+                  key: postgres-password
+          volumeMounts:
+            - name: postgres-storage
+              mountPath: /var/lib/postgresql/data
+      volumes:
+        - name: postgres-storage
+          persistentVolumeClaim:
+            claimName: postgres-pvc

k8s/postgres-pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

k8s/postgres-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres-service
+spec:
+  selector:
+    app: postgres
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432

k8s/pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: simple-rag-pvc
+  labels:
+    app: simple-rag
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

k8s/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: simple-rag-service
+  labels:
+    app: simple-rag
+spec:
+  # Use LoadBalancer if you are on a cloud provider like AWS/GCP, or NodePort/ClusterIP for local k8s (minikube)
+  type: LoadBalancer
+  ports:
+    - port: 80
+      targetPort: 8000
+      protocol: TCP
+  selector:
+    app: simple-rag