nycmeshnet · james-otten · Apr 27, 2024 · Apr 28, 2024 · Apr 28, 2024 · Apr 28, 2024
diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml
@@ -3,6 +3,8 @@ name: Checkov
 on:
   push:
     branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
   workflow_dispatch:
 
 permissions: read-all
@@ -24,14 +26,14 @@ jobs:
         id: checkov
         uses: bridgecrewio/checkov-action@0549dc60bddd4c55cb85c6c3a07072e3cf2ca48e
         with:
-          skip_check: CKV_DOCKER_2,CKV_DOCKER_3
+          skip_check: CKV_DOCKER_2,CKV_DOCKER_3,CKV_SECRET_6
           quiet: true
           output_format: cli,sarif
           output_file_path: console,results.sarif
           download_external_modules: true
 
       - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3
         if: success() || failure()
         with:
           sarif_file: results.sarif
diff --git a/.github/workflows/helm_lint.yaml b/.github/workflows/helm_lint.yaml
@@ -0,0 +1,47 @@
+name: Lint and Test Chart
+
+on: pull_request
+
+permissions: read-all
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@20d2b4f98d41febe2bbca46408499dbb535b6258 # v3
+        with:
+          version: v3.14.0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+          check-latest: true
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }}
+
+      - name: Create kind cluster
+        if: steps.list-changed.outputs.changed == 'true'
+        uses: helm/[email protected]
+
+      - name: Run chart-testing (install)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct install --target-branch ${{ github.event.repository.default_branch }}
diff --git a/.github/workflows/publish-and-deploy.yaml b/.github/workflows/publish-and-deploy.yaml
@@ -51,3 +51,19 @@ jobs:
         if_key_exists: fail # replace / ignore / fail; optional (defaults to fail)
     - name: Pull new Docker image
       run: ssh ${{ secrets.GRANDSVC_SSH_TARGET }} "cd ${{ secrets.GRANDSVC_PROJECT_PATH }} && git pull && docker compose pull && docker compose up -d"
+
+  deploy_to_dev0:
+    name: Deploy to dev0
+    needs: push_to_registry
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install SSH key
+      uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # v2
+      with:
+        key: ${{ secrets.DEV0_KEY }}
+        name: id_ed25519 # optional
+        known_hosts: ${{ secrets.DEV0_KNOWN_HOSTS }}
+        #config: ${{ secrets.CONFIG }} # ssh_config; optional
+        if_key_exists: fail # replace / ignore / fail; optional (defaults to fail)
+    - name: Pull new Docker image
+      run: ssh ${{ secrets.DEV0_SSH_TARGET }} "cd ${{ secrets.DEV0_PROJECT_PATH }} && git pull && cd infra/helm/meshdb && helm template . -f ../../../../values.yaml -f  ../../../../secret.values.yaml | kubectl apply -f -"
diff --git a/infra/README.md b/infra/README.md
@@ -0,0 +1,65 @@
+# Meshdb Environment Setup
+
+These instructions will set up a 4 node k3s cluster on proxmox.
+- 1 "manager" node for control plane and to be used for deployments.
+- 3 "agent" nodes to run services.
+
+1. Configure a user for the [proxmox provider](https://registry.terraform.io/providers/Telmate/proxmox/latest/docs) and set up env vars. Create an API key in Proxmox, and disable Privilege Separation.
+2. Setup tfvars + ssh keys
+```
+cd meshdb/infra/tf/
+cp example.tfvars your_env.tfvars
+# Modify your_env.tfvars to meet your needs
+ENV_NAME="garfield" # Matching meshdb_env_name
+ssh-keygen -t ed25519 -f ./meshdb$ENV_NAME
+```
+3. Create the VMs that will host k3s
+```
+terraform init
+terraform plan --var-file=your_env.tfvars
+terraform apply --var-file=your_env.tfvars
+```
+4. Login via serial and figure out the IPs that were received from DHCP
+5. One time provisioning for the manager node
+
+```
+target_host="<MGR IP>"
+scp infra/mgr_provision.sh ubuntu@$target_host:/home/ubuntu/mgr_provision.sh
+ssh -t ubuntu@$target_host "sudo bash /home/ubuntu/mgr_provision.sh"
+```
+
+6. Set the IP range for metallb, such as `10.70.90.80/29`, in `/opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml` and then deploy metallb and longhorn from the manager
+```
+cd /opt/meshdb_mgmt/meshdb/infra/cluster/
+terraform init
+terraform plan
+terraform apply
+# update address block in /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml
+kubectl apply -f /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml
+```
+
+7. Setup each node (from the manager)
+
+```
+cd /opt/meshdb_mgmt/meshdb/infra/
+declare -a target_nodes=("10.70.90.XX" "10.70.90.YY" "10.70.90.ZZ")
+
+for n in "${target_nodes[@]}"
+do
+  bash setup_node.sh $n
+done
+```
+
+8. Update values + secrets in `/opt/meshdb_mgmt/values.yaml` and `/opt/meshdb_mgmt/secret.values.yaml`
+
+9. Deploy helm chart. Create the namespace you indicated in `/opt/meshdb_mgmt/values.yaml`
+
+```
+your_ns="meshdbdev0"
+cd /opt/meshdb_mgmt/meshdb/infra/helm/meshdb/
+kubectl create namespace $your_ns
+helm template . -f ../../../../values.yaml -f ../../../../secret.values.yaml | kubectl apply -f -
+kubectl get all -n $your_ns
+```
+
+10. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser`
diff --git a/infra/cluster/longhorn.yaml b/infra/cluster/longhorn.yaml
@@ -0,0 +1,9 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  repo: https://charts.longhorn.io
+  chart: longhorn
+  targetNamespace: longhorn-system
diff --git a/infra/cluster/main.tf b/infra/cluster/main.tf
@@ -0,0 +1,33 @@
+provider "kubernetes" {
+  config_path = "/etc/rancher/k3s/k3s.yaml"
+}
+
+# Create metallb-system
+resource "kubernetes_namespace" "metallb-system-ns" {
+  metadata {
+    name = "metallb-system"
+  }
+}
+
+# Create metallb with the manifest
+resource "kubernetes_manifest" "metallb" {
+  manifest = yamldecode(file("./metallb.yaml"))
+  depends_on = [
+    kubernetes_namespace.metallb-system-ns
+  ]
+}
+
+# Create longhorn-system
+resource "kubernetes_namespace" "longhorn-system-ns" {
+  metadata {
+    name = "longhorn-system"
+  }
+}
+
+# Create longhorn with the manifest
+resource "kubernetes_manifest" "longhorn" {
+  manifest = yamldecode(file("./longhorn.yaml"))
+  depends_on = [
+    kubernetes_namespace.longhorn-system-ns
+  ]
+}
diff --git a/infra/cluster/metallb.yaml b/infra/cluster/metallb.yaml
@@ -0,0 +1,9 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: metallb
+  namespace: metallb-system
+spec:
+  repo: https://metallb.github.io/metallb
+  chart: metallb
+  targetNamespace: metallb-system
diff --git a/infra/cluster/metallb_extra.yaml b/infra/cluster/metallb_extra.yaml
@@ -0,0 +1,17 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: pool-1
+  namespace: metallb-system
+spec:
+  addresses:
+  - "10.70.90.80/29"
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: k3s-l2
+  namespace: metallb-system
+spec:
+  ipAddressPools:
+  - pool-1
diff --git a/infra/helm/meshdb/Chart.yaml b/infra/helm/meshdb/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: meshdb
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
diff --git a/infra/helm/meshdb/README.md b/infra/helm/meshdb/README.md
@@ -0,0 +1,83 @@
+# meshdb
+
+A Helm chart for Kubernetes
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| autoscaling.enabled | bool | `false` |  |
+| autoscaling.maxReplicas | int | `100` |  |
+| autoscaling.minReplicas | int | `1` |  |
+| autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
+| aws.access_key_id | string | `"the_real_value"` | AWS access key id for S3 |
+| aws.secret_access_key | string | `"the_real_value"` | AWS secret access key for S3 |
+| fullnameOverride | string | `"meshdb"` | App name |
+| image.pullPolicy | string | `"Always"` | pullPolicy for all images, should be `Always` |
+| map.base_url | string | `"http://admin-map.grandsvc.mesh.nycmesh.net"` | Map url |
+| meshdb_app_namespace | string | `"meshdbdev0"` | K8s namespace used for all resources |
+| meshweb.affinity | object | `{}` |  |
+| meshweb.backup_s3_base_folder | string | `"meshdb-backups/development/"` | Base folder for django postgres backups |
+| meshweb.backup_s3_bucket_name | string | `"meshdb-data-backups"` | Bucket used for django postgres backups |
+| meshweb.disable_pano_edits | string | `"True"` | Feature flag for disabling panorama edits |
+| meshweb.disable_profiling | string | `"True"` | Disable profiling in meshweb |
+| meshweb.django_secret_key | string | `"the_real_value"` | Django secret key |
+| meshweb.enable_debug | string | `"False"` | Enable `DEBUG` in meshweb |
+| meshweb.image.repository | string | `"willnilges/meshdb"` | Docker image repo for meshweb |
+| meshweb.image.tag | string | `"main"` | Docker image tag for meshweb |
+| meshweb.liveness_probe | string | `"true"` | Enable liveness probe with `true` all other values will disable it |
+| meshweb.nn_assign_psk | string | `"the_real_value"` | Legacy NN assign form password |
+| meshweb.nodeSelector | object | `{}` |  |
+| meshweb.pano_github_token | string | `"the_real_value"` | Github token for downloading panorama |
+| meshweb.podSecurityContext | object | `{}` |  |
+| meshweb.port | int | `8081` | Port used by meshweb (internally) |
+| meshweb.query_psk | string | `"the_real_value"` | Legacy query form password |
+| meshweb.resources | object | `{}` |  |
+| meshweb.securityContext | object | `{}` |  |
+| meshweb.static_pvc_name | string | `"meshdb-static-pvc"` | Name of the PVC for static content |
+| meshweb.static_pvc_size | string | `"1Gi"` | Size of the PVC for static content |
+| meshweb.tolerations | list | `[]` |  |
+| nameOverride | string | `""` |  |
+| nginx.affinity | object | `{}` |  |
+| nginx.nodeSelector | object | `{}` |  |
+| nginx.podSecurityContext | object | `{}` |  |
+| nginx.port | int | `80` | Nginx port |
+| nginx.resources | object | `{}` |  |
+| nginx.securityContext | object | `{}` |  |
+| nginx.server_name | string | `"db.nycmesh.net"` | `server_name` used by nginx |
+| nginx.tolerations | list | `[]` |  |
+| pelias.affinity | object | `{}` |  |
+| pelias.nodeSelector | object | `{}` |  |
+| pelias.podSecurityContext | object | `{}` |  |
+| pelias.port | int | `6800` | Pelias port (internal) |
+| pelias.resources | object | `{}` |  |
+| pelias.securityContext | object | `{}` |  |
+| pelias.tolerations | list | `[]` |  |
+| pg.affinity | object | `{}` |  |
+| pg.dbname | string | `"meshdb"` | Postgres database name |
+| pg.liveness_probe | string | `"true"` | Enable liveness probe with `true` all other values will disable it |
+| pg.nodeSelector | object | `{}` |  |
+| pg.password | string | `"the_real_value"` | Password for postgres |
+| pg.podSecurityContext | object | `{}` |  |
+| pg.port | string | `"5432"` | Postgres port (internal) |
+| pg.pvc_name | string | `"meshdb-postgres-pvc"` | Name of the PVC for postgres |
+| pg.pvc_size | string | `"20Gi"` | Size of the PVC for postgres |
+| pg.resources | object | `{}` |  |
+| pg.securityContext | object | `{}` |  |
+| pg.tolerations | list | `[]` |  |
+| pg.user | string | `"meshdb"` | Postgres user |
+| podAnnotations | object | `{}` |  |
+| podLabels | object | `{}` |  |
+| redis.affinity | object | `{}` |  |
+| redis.liveness_probe | string | `"true"` | Enable liveness probe with `true` all other values will disable it |
+| redis.nodeSelector | object | `{}` |  |
+| redis.podSecurityContext | object | `{}` |  |
+| redis.port | int | `6379` | Redis port (internal) |
+| redis.resources | object | `{}` |  |
+| redis.securityContext | object | `{}` |  |
+| redis.tolerations | list | `[]` |  |
+| uisp.psk | string | `"the_real_value"` | Password for UISP |
+| uisp.url | string | `"https://uisp.mesh.nycmesh.net/nms"` | UISP url |
+| uisp.user | string | `"nycmesh_readonly"` | Username for UISP |
+
+----------------------------------------------
diff --git a/infra/helm/meshdb/secret.values.yaml b/infra/helm/meshdb/secret.values.yaml
@@ -0,0 +1,15 @@
+pg:
+  password: the_real_value
+
+aws:
+  access_key_id: the_real_value
+  secret_access_key: the_real_value
+
+meshweb:
+  django_secret_key: the_real_value
+  nn_assign_psk: the_real_value
+  query_psk: the_real_value
+  pano_github_token: the_real_value
+
+uisp:
+  psk: the_real_value