Init

Author: Andreas Kull

Makefile

@@ -0,0 +1,27 @@
+.ONESHELL:
+
+.DEFAULT_GOAL := install
+
+BIN = .venv/bin
+
+.PHONY: clean
+clean:
+	rm -f requirements.txt
+	rm -rf .venv
+
+.PHONY: install
+install: requirements.in
+	python3 -m venv .venv
+	chmod +x .venv/bin/activate
+	. .venv/bin/activate
+	$(BIN)/pip install pip-tools
+	$(BIN)/pip-compile --strip-extras -q -o requirements.txt requirements.in
+	$(BIN)/pip-sync requirements.txt
+
+.PHONY: lint
+lint: .venv/bin/ruff
+	$(BIN)/ruff check .
+
+.PHONY: test
+test: .venv/bin/pytest
+	$(BIN)/pytest app/

README.md

@@ -0,0 +1,83 @@
+# Deployment for Python
+
+Opinionated shell scripts for deployment, a lot is based on the repository name and thus the `SLUG` environment variable.
+
+Assume we have a repository named `foobar-service` and we didn't set `SLUG` by hand.
+
+Here's the explanation in order of recommended execution:
+
+## Requirements
+
+- 3.12 <= [Python](https://docs.python.org) < 3.13
+
+- [Docker](https://docs.docker.com/)
+
+- [Kubernetes](https://kubernetes.io/docs/home/)
+
+- [AWS](https://docs.aws.amazon.com/)
+
+- [pip-tools](https://github.com/jazzband/pip-tools)
+
+- [ruff](https://docs.astral.sh/ruff/)
+
+## slug.sh
+
+A lot of things are derived from the repository's name, i.e. you either set the `SLUG` environment variable manually or you replace `<REPLACE_WITH_REPOSITORY_NAME_ENV_VAR>` with something provided by your CI/CD like `GITHUB_REPOSITORY_NAME`, `BITBUCKET_REPO_SLUG`, ...
+
+Example: `BITBUCKET_REPO_SLUG=foobar-service` will result in `SLUG=foobar`
+
+## master.sh
+
+Sets up all the environment variables for a specific branch/environment, i.e. rename this file to `production.sh` or duplicate it for whatever environment you build for. It requires some replacements:
+
+- `<REPLACE_WITH_AWS_ACCOUNT_ID>`, e.g. 123456789012 or environment variable
+
+- `<REPLACE_WIHT_AWS_REGION>`, e.g. us-east-1 or environment variable
+
+- `<REPLACE_WITH_API_URL_NAME>`, e.g. api.foobar.com or environment variable
+
+Then we have the `CONTEXT` for our API which I like to base on the `SLUG`, e.g. `api.company.com/foobar` but you can also set it manually.
+
+## build.sh
+
+Triggers `make`, see `Makefile`.
+
+## lint.sh
+
+Triggers `make lint`, see `Makefile`.
+
+## docker.sh
+
+Here we build the docker image and push it to the ECR repository.
+
+It also requires some environment_variables to be set:
+
+- `SLUG`, set by `slug.sh` or manually, e.g. in `master.sh`
+
+- `<REPLACE_WITH_BRANCH_NAME_ENV_VAR>`, e.g. `GITHUB_REF##*/`, `BITBUCKET_BRANCH`, ...
+
+- `<REPLACE_WITH_COMMIT_HASH_ENV_VAR>`, e.g. `GITHUB_SHA`, `BITBUCKET_COMMIT`, ...
+
+- `AWS_ECR_URL`
+
+- `AWS_REGION`
+
+- `AWS_ACCESS_KEY_ID`
+
+- `AWS_SECRET_ACCESS_KEY`
+
+A multi-staged build follows that can use a specific `CONTEXT` (set in `master.sh`) environment variable for the server.
+
+The images pushed assume an existing repository with `SLUG` as it's name, e.g. `foobar`.
+
+It will push the image with the tags `latest` and branch name with shortened commit hash, e.g. `master-1234567`.
+
+## k8s.sh
+
+So one assumption here is that `STAGE` is basically also your k8s namespace, e.g. `(STAGE=PRODUCTION) == (NAMESPACE=production)`.
+
+Yet another assumption is that you've created a secret in the secrets-manager with name `$SLUG-secrets`, e.g. `foobar-secrets`.
+
+I use [envsubst](https://www.gnu.org/software/gettext/manual/html_node/envsubst-Invocation.html) to replace variables in the `*.yml` files with environment variables.
+
+Have a look at the YAML files yourself, they provide a basic skeleton.

build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+make

config.yml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: $SLUG-config
+data:
+  STAGE: $STAGE
+  NAME: $SLUG

docker.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+[[ -z $SLUG ]] && echo "Missing SLUG" && exit 1
+[[ -z $<REPLACE_WITH_BRANCH_NAME_ENV_VAR> ]] && echo "Missing BRANCH_NAME" && exit 1
+[[ -z $<REPLACE_WITH_COMMIT_HASH_ENV_VAR> ]] && echo "Missing COMMIT_HASH" && exit 1
+[[ -z $AWS_ECR_URL ]] && echo "Missing AWS_ECR_URL" && exit 1
+[[ -z $AWS_REGION ]] && echo "Missing AWS_REGION" && exit 1
+[[ -z $AWS_ACCESS_KEY_ID ]] && echo "Missing AWS_ACCESS_KEY_ID" && exit 1
+[[ -z $AWS_SECRET_ACCESS_KEY ]] && echo "Missing AWS_SECRET_ACCESS_KEY" && exit 1
+
+# Create Dockerfile
+cat >Dockerfile <<EOF
+FROM python:3.12-slim as build
+
+ENV PIP_DEFAULT_TIMEOUT=100 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_CACHE_DIR=1
+
+WORKDIR /app
+
+COPY requirements.in ./
+
+RUN pip install pip-tools \
+    && pip-compile --strip-extras -q -o requirements.txt requirements.in
+
+
+FROM python:3.12-slim as runtime
+
+ENV CONTEXT=$CONTEXT
+
+WORKDIR /app
+
+COPY --from=build /app/requirements.txt .
+
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install build-essential -y \
+    && pip install -r requirements.txt \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY ./app app
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--root-path", "/$CONTEXT"]
+EOF
+
+# Create vars
+IMAGE="${AWS_ECR_URL}/${SLUG}"
+TAG="${BRANCH_NAME}-${COMMIT_HASH::7}"
+
+echo -e "\n--- docker ---"
+echo "IMAGE=$IMAGE"
+echo "TAGS=$TAG,latest"
+echo -e "--- docker ---\n"
+
+aws ecr get-login-password --region "${AWS_REGION}" | docker login --username AWS --password-stdin "${AWS_ECR_URL}" &&
+  # Build and push
+  docker build -t "$IMAGE":"$TAG" -t "$IMAGE":latest . &&
+  docker push "$IMAGE":latest &&
+  docker push "$IMAGE":"$TAG"

k8s.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+[[ -z $STAGE ]] && echo "Missing STAGE" && exit 1
+[[ -z $SLUG ]] && echo "Missing SLUG" && exit 1
+
+
+# Create vars
+BASEDIR=$(dirname "$0")
+NAME=$(echo "${STAGE}" | tr '[:upper:]' '[:lower:]')
+NAMESPACE="${NAME}"
+
+# Update config
+aws eks update-kubeconfig --name "$NAME"
+
+secrets () {
+  # Create secrets
+  kubectl -n "$NAMESPACE" create secret generic "${SLUG}"-secrets \
+    --from-env-file=<(aws secretsmanager get-secret-value --secret-id "${SLUG}"-secrets | jq -r .SecretString | jq -r 'to_entries | .[] | .key + "=" + .value') \
+    --dry-run=client -o yaml |
+    kubectl apply -f -
+}
+
+config () {
+  # Apply config map
+  envsubst <"$BASEDIR"/config.yml | kubectl apply -n "$NAMESPACE" -f -
+}
+
+deploy () {
+  # Required to substitute variables in k8s.yml
+  [[ -z $HOST ]] && echo "Missing HOST" && exit 1
+  [[ -z $PORT ]] && echo "Missing PORT" && exit 1
+  [[ -z $MEMORY_LIMIT ]] && echo "Missing MEMORY_LIMIT" && exit 1
+  [[ -z $MEMORY_REQUESTS ]] && echo "Missing MEMORY_REQUESTS" && exit 1
+  [[ -z $AWS_ECR_URL ]] && echo "Missing AWS_ECR_URL" && exit 1
+
+  envsubst <"$BASEDIR"/k8s.yml | kubectl apply -n "$NAMESPACE" -f -
+}
+
+restart () {
+  kubectl rollout restart deployment -n "$NAMESPACE" "$SLUG"
+}
+
+case $1 in
+  "secrets")
+    secrets
+  ;;
+  "config")
+    config
+  ;;
+  "deploy")
+    deploy
+  ;;
+  "restart")
+    restart
+  ;;
+  *)
+    exit 1;
+  ;;
+esac

k8s.yml

@@ -0,0 +1,80 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: $SLUG
+  labels:
+    app: $SLUG
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: $SLUG
+  template:
+    metadata:
+      annotations:
+        linkerd.io/inject: enabled
+      labels:
+        app: $SLUG
+    spec:
+      containers:
+        - name: $SLUG
+          image: $AWS_ECR_URL/$SLUG:latest
+          imagePullPolicy: Always
+          livenessProbe:
+            tcpSocket:
+              port: $PORT
+            initialDelaySeconds: 30
+          readinessProbe:
+            tcpSocket:
+              port: $PORT
+            initialDelaySeconds: 30
+          resources:
+            limits:
+              memory: $MEMORY_LIMIT
+            requests:
+              memory: $MEMORY_REQUESTS
+          ports:
+            - containerPort: $PORT
+              name: $SLUG
+          envFrom:
+            - configMapRef:
+                name: $SLUG-config
+            - secretRef:
+                name: $SLUG-secrets
+---
+kind: Ingress
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: $SLUG-ingress
+  namespace: $NAMESPACE
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+    nginx.ingress.kubernetes.io/limit-connections: "500"
+    nginx.ingress.kubernetes.io/limit-rps: "500"
+    nginx.ingress.kubernetes.io/load-balance: ewma
+    nginx.ingress.kubernetes.io/use-regex: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: $HOST
+      http:
+        paths:
+          - backend:
+              service:
+                name: $SLUG
+                port:
+                  number: $PORT
+            path: /$CONTEXT(/*)(.*)
+            pathType: ImplementationSpecific
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: $SLUG
+spec:
+  selector:
+    app: $SLUG
+  ports:
+    - protocol: TCP
+      port: $PORT
+      targetPort: $PORT

lint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+make lint

master.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export AWS_ECR_URL=<REPLACE_WITH_AWS_ACCOUNT_ID>.dkr.ecr.<REPLACE_WIHT_AWS_REGION>.amazonaws.com
+export AWS_REGION=<REPLACE_WIHT_AWS_REGION>
+
+export STAGE=PRODUCTION
+export HOST=<REPLACE_WITH_API_URL_NAME>
+export PORT=8000
+export CONTEXT=$SLUG
+export MEMORY_LIMIT=512Mi
+export MEMORY_REQUESTS=256Mi
+
+printenv

slug.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# If SLUG is not set, build it from the repository name
+if [[ -z ${SLUG} ]]; then
+  SLUG=$(echo "$<REPLACE_WITH_REPOSITORY_NAME_ENV_VAR>" | cut -f1 -d"-" | head -c 15)
+fi
+
+export SLUG=$SLUG
+echo "SLUG: $SLUG"

test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+STAGE=LOCAL make test