Shuffle kustomize files around, simplify pipelines, and add docs

.github/workflows/build-and-publish.yml

@@ -1,39 +1,16 @@
 # This is a basic workflow to help you get started with Actions
-name: Bundle building and pushing
+name: Building and Pushing to GHCR
 
 # Controls when the action will run. Triggers the workflow on push or pull request
 # events but only for the main branch
 on: [workflow_dispatch]
 
-env:
-  GO_VERSION: '1.16.1'
-
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  lint:
+  publish:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Linter
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Download modules
-        run: go mod download
-      - name: Build project
-        run: go build ./cmd/aks-periscope
       - name: Get Changelog Entry
         id: changelog_reader
         uses: mindsers/changelog-reader-action@v2
@@ -43,6 +20,7 @@ jobs:
       - name: Display output
         run: |
           echo "Version: ${{ steps.changelog_reader.outputs.version }}"
+          echo "Changes: ${{ steps.changelog_reader.outputs.changes }}"
       # Lowercase my github ownername.
       - name: Set Environment Variables
         run: |
@@ -59,6 +37,5 @@ jobs:
       # Note the below automatically sets the registry to the local GH registry of the org name. 
       # Currently, for pure gitops reconstructing this runtime setting will require looking at the repository owner of the commit
         run: |
-          docker build -f ./builder/Dockerfile -t aks/periscope .
-          docker tag aks/periscope ghcr.io/${{ env.REPO-OWNER }}/aks/periscope:${{ steps.changelog_reader.outputs.version }}
+          docker build -f ./builder/Dockerfile -t ghcr.io/${{ env.REPO-OWNER }}/aks/periscope:${{ steps.changelog_reader.outputs.version }} .
           docker push ghcr.io/${{ env.REPO-OWNER }}/aks/periscope:${{ steps.changelog_reader.outputs.version }}

.github/workflows/build-publish-mcr.yml

@@ -1,26 +1,15 @@
-name: Building and pushing mcr
+name: Building and Pushing to MCR
 on: [workflow_dispatch]
 
 permissions:
       id-token: write
       contents: read
 
-env:
-  GO_VERSION: '1.16.1'
-
 jobs: 
-  build-and-deploy:
+  publish:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Download modules
-        run: go mod download
-      - name: Build project
-        run: go build ./cmd/aks-periscope
       - name: Get Changelog Entry
         id: changelog_reader
         uses: mindsers/changelog-reader-action@v2
@@ -39,8 +28,7 @@ jobs:
           subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
       - name: 'Run Azure CLI commands'
         run: |
-          docker build -f ./builder/Dockerfile -t public/aks/periscope .
-          docker tag public/aks/periscope ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/periscope:${{ steps.changelog_reader.outputs.version }}
+          docker build -f ./builder/Dockerfile -t ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/periscope:${{ steps.changelog_reader.outputs.version }} .
           az acr login -n ${{ secrets.AZURE_REGISTRY_SERVER }}
           docker push ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/periscope:${{ steps.changelog_reader.outputs.version }}
           echo "acr push done"

.github/workflows/ci-pipeline.yaml

@@ -55,8 +55,6 @@ jobs:
         run: |
           curl -Lo ./kind https://kind.sigs.k8s.io/dl/${{ env.KIND_VERSION }}/kind-linux-amd64
           chmod +x ./kind
-
-          curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
       - name: Start kind
         run: |
           docker run -d --restart=always -p "127.0.0.1:5000:5000" --name "registry" registry:2
@@ -106,9 +104,22 @@ jobs:
           helm repo update
           helm install happy-panda bitnami/wordpress --namespace default
       - name: Deploy AKS Periscope
+        # Deploy using the Kustomize functionality built in to kubectl
+        # (placing the overlay in a gitignore'd location so this can be run locally
+        # without editing source-controlled files).
         run: |
-          (cd ./deployment && kustomize edit set image mcr.microsoft.com/aks/periscope=localhost:5000/periscope:${{ github.run_id }})
-          kubectl apply -f <(kustomize build ./deployment)
+          # Create Kustomize overlay folder
+          mkdir -p ./deployment/overlays/temp
+          touch ./deployment/overlays/temp/.env.config
+          touch ./deployment/overlays/temp/.env.secret
+
+          # Build kustomization yaml
+          export IMAGE_NAME=localhost:5000/periscope
+          export IMAGE_TAG=${{ github.run_id }}
+          cat ./deployment/overlays/dynamic-image/kustomization.template.yaml | envsubst > ./deployment/overlays/temp/kustomization.yaml
+
+          # Deploy and wait for readiness
+          kubectl apply -k ./deployment/overlays/temp
           kubectl -n aks-periscope describe ds aks-periscope
           kubectl -n aks-periscope wait po --all --for condition=ready --timeout=240s
       - name: Go tests

.gitignore

@@ -334,3 +334,13 @@ ASALocalRun/
 
 # MFractors (Xamarin productivity tool) working folder 
 .mfractor/
+
+# CI files
+helm
+kind
+*.tar.gz
+deployment/overlays/temp
+
+# Local deployment configuration files
+**/.env
+**/.env.*

README.md

@@ -141,48 +141,64 @@ To locally build this project from the root of this repository:
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor github.com/Azure/aks-periscope/cmd/aks-periscope
 ```
 
-**Tip**: In order to test local changes, user can build the local image via `Dockerfile` and then push it to your local hub. This way, a user should be able to reference this test image in the `deployment\aks-periscope.yaml` `containers` property `image` attribute reference to your published test docker image. 
+**Tip**: To test local changes, there are instructions for running Periscope in a `Kind` cluster in the ['dev' Kustomize overlay notes](./deployment/overlays/dev/README.md). This allows for altering the configuration without touching any source-controlled files.
 
-For example:
-
-```sh
-docker build -f ./builder/Dockerfile -t <some_docker_repo_name>/<aks-periscope-user-selected-test-name> .
-docker push <some_docker_repo_name>/<aks-periscope-user-selected-test-name> 
-```
+**Tip**: To test changes in a GitHub branch, there are instructions for running images published to a local GHCR registry in the ['dynamic-image' Kustomize overlay notes](./deployment/overlays/dynamic-image/README.md#ghcr).
 
 ## Dependent Consuming Tools and Working Contract
 
-`az-cli` and `vscode` both consume the `aks-periscope.yaml` file. If the `aks-periscope.yaml` file is changed, you will introduce breaking changes to `az-cli` and `vscode`.
+Dependent tools need access to an immutable, versioned Periscope resource definition. We provide two ways to obtain this:
+1. [Deprecated] Build the `external` overlay using instructions [here](./deployment/overlays/external/README.md) and include the output as a static resource in consuming tools. This will require runtime string substitution to configure appropriately for any given deployment, before being deployed using `kubectl -f`.
+2. Build a `Kustomize` overlay at runtime, referencing `https://github.com/azure/aks-periscope//deployment/base?ref={RELEASE_TAG}` as the base, and the appropriate MCR image tags for that release, as well as all configuration and secrets. This can then be deployed using `kubectl -k`. Example:
+```yaml
+resources:
+- https://github.com/azure/aks-periscope//deployment/base?ref={RELEASE_TAG}
+images:
+- name: periscope
+  newName: mcr.microsoft.com/aks/periscope
+  newTag: "{IMAGE_TAG}"
+secretGenerator:
+- name: azureblob-secret
+  behavior: replace
+  literals:
+  - AZURE_BLOB_ACCOUNT_NAME={STG_ACCOUNT}
+  - AZURE_BLOB_SAS_KEY=?{STG_SAS}
+  - AZURE_BLOB_CONTAINER_NAME={STG_CONTAINER}
+configMapGenerator:
+- name: diagnostic-config
+  behavior: merge
+  literals:
+  # Only specify those which should be overridden
+  - DIAGNOSTIC_KUBEOBJECTS_LIST={KUBEOBJECTS_OVERRIDE}
+```
 
 ## Debugging Guide
 
 This section intends to add some tips for debugging pod logs using aks-periscope.
 
 Scenario, where `user A` uses **expired** `sas-token` and converts into `base64` to be used in the deployment file.
 
-In the scenario above, the `kubectl apply -f deployment-file.yaml` will show no error but the output which will look like the one below.
+In the scenario above, the `kubectl apply -k ./deployment/overlays/dev` will show no error but the output which will look like the one below.
 
 ```sh
-❯ kubectl apply -f deployment/aks-periscope.yaml
+❯ kubectl apply -k ./deployment/overlays/dev
 namespace/aks-periscope created
 serviceaccount/aks-periscope-service-account created
 clusterrole.rbac.authorization.k8s.io/aks-periscope-role unchanged
 clusterrolebinding.rbac.authorization.k8s.io/aks-periscope-role-binding unchanged
 clusterrolebinding.rbac.authorization.k8s.io/aks-periscope-role-binding-view unchanged
 daemonset.apps/aks-periscope created
 secret/azureblob-secret created
-configmap/containerlogs-config created
-configmap/kubeobjects-config created
-configmap/nodelogs-config created
+configmap/diagnostic-config created
 customresourcedefinition.apiextensions.k8s.io/diagnostics.aks-periscope.azure.github.com unchanged
 ```
 
-To debug the `pod` logs in the `aks-periscope` namespace deployed in the cluster:
+To debug the `pod` logs in the `aks-periscope-dev` namespace deployed in the cluster:
 
-   * To get the pods in `aks-periscope` namespace:
-       * `kubectl get pods -n aks-periscope`
+   * To get the pods in `aks-periscope-dev` namespace:
+       * `kubectl get pods -n aks-periscope-dev`
    * To check the logs in each of the deployed pods:
-       * `kubectl logs <name-of-pod> -n aks-periscope`
+       * `kubectl logs <name-of-pod> -n aks-periscope-dev`
 
 Feel free to contact aksperiscope@microsoft.com or open an issue with any feedback or questions about AKS Periscope. This is currently a work in progress, but look out for more capabilities to come!
 

deployment/daemon-set.yaml → deployment/base/daemon-set.yaml

@@ -16,21 +16,18 @@ spec:
       serviceAccountName: aks-periscope-service-account
       hostPID: true
       nodeSelector:
-        beta.kubernetes.io/os: linux
+        kubernetes.io/os: linux
       containers:
       - name: aks-periscope
-        image: mcr.microsoft.com/aks/periscope
+        image: periscope
         securityContext:
           privileged: true
         imagePullPolicy: Always
-        env: []
         envFrom:
         - configMapRef:
-            name: containerlogs-config
-        - configMapRef:
-            name: kubeobjects-config
-        - configMapRef:
-            name: nodelogs-config
+            name: diagnostic-config
+        - secretRef:
+            name: azureblob-secret
         volumeMounts:
         - name: varlog
           mountPath: /var/log
@@ -54,4 +51,4 @@ spec:
           path: /run/systemd/resolve
       - name: etcvmlog
         hostPath:
-          path: /etc
+          path: /etc

deployment/base/kustomization.yaml

@@ -0,0 +1,29 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: aks-periscope
+
+resources:
+- namespace.yaml
+- cluster-role.yaml
+- cluster-role-binding.yaml
+- crd.yaml
+- daemon-set.yaml
+- service-account.yaml
+
+configMapGenerator:
+- name: diagnostic-config
+  literals:
+  - DIAGNOSTIC_CONTAINERLOGS_LIST=kube-system
+  - DIAGNOSTIC_KUBEOBJECTS_LIST=kube-system/pod kube-system/service kube-system/deployment
+  - DIAGNOSTIC_NODELOGS_LIST="/var/log/azure/cluster-provision.log /var/log/cloud-init.log"
+
+secretGenerator:
+- name: azureblob-secret
+  literals:
+  - AZURE_BLOB_ACCOUNT_NAME=
+  - AZURE_BLOB_SAS_KEY=
+  - AZURE_BLOB_CONTAINER_NAME=
+
+generatorOptions:
+  disableNameSuffixHash: true