Merge pull request #257 from snyk/chore/multi-platform-integration-tests

chore: eks/ecr integration testing preparations continued

Author: Amir Moualem

.circleci/config.yml

@@ -173,6 +173,32 @@ jobs:
             fi
           when: on_fail
 
+  eks_integration_tests:
+    <<: *default_machine_config
+    steps:
+      - checkout
+      - run:
+          name: INTEGRATION TESTS EKS
+          command: |
+            export NVM_DIR="/opt/circleci/.nvm" &&
+            [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" &&
+            nvm install v10 &&
+            npm install &&
+            docker login --username ${DOCKERHUB_USER} --password ${DOCKERHUB_PASSWORD} &&
+            export IMAGE_TAG=$([[ "$CIRCLE_BRANCH" == "staging" ]] && echo "staging-candidate" || echo "discardable") &&
+            export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=snyk/kubernetes-monitor:${IMAGE_TAG}-${CIRCLE_SHA1} &&
+            docker pull ${KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG} &&
+            .circleci/do-exclusively --branch staging npm run test:integration:eks
+      - run:
+          name: Notify Slack on failure
+          command: |
+            if [[ "$CIRCLE_BRANCH" == "staging" ]]; then
+              ./scripts/slack-notify-failure.sh "staging-eks-integration-tests-${CIRCLE_SHA1}"
+            else
+              echo "Current branch is $CIRCLE_BRANCH so skipping notifying Slack"
+            fi
+          when: on_fail
+
 ######################## MERGE TO STAGING ########################
   tag_and_push:
     <<: *default_container_config
@@ -293,6 +319,10 @@ workflows:
           requires:
             - build_image
           <<: *staging_branch_only_filter
+      - eks_integration_tests:
+          requires:
+            - build_image
+          <<: *staging_branch_only_filter
       - package_manager_test_apk:
           requires:
             - build_image

.circleci/do-exclusively

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# copied from https://github.com/bellkev/circle-lock-test
+
+# sets $branch, $tag, $rest
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            -b|--branch) branch="$2" ;;
+            -t|--tag) tag="$2" ;;
+            *) break ;;
+        esac
+        shift 2
+    done
+    rest=("$@")
+}
+
+# reads $branch, $tag, $commit_message
+should_skip() {
+    if [[ "$branch" && "$CIRCLE_BRANCH" != "$branch" ]]; then
+        echo "Not on branch $branch. Skipping..."
+        return 0
+    fi
+
+    if [[ "$tag" && "$commit_message" != *\[$tag\]* ]]; then
+        echo "No [$tag] commit tag found. Skipping..."
+        return 0
+    fi
+
+    return 1
+}
+
+# reads $branch, $tag
+# sets $jq_prog
+make_jq_prog() {
+    local jq_filters=""
+
+    if [[ $branch ]]; then
+        jq_filters+=" and .branch == \"$branch\""
+    fi
+
+    if [[ $tag ]]; then
+        jq_filters+=" and (.subject | contains(\"[$tag]\"))"
+    fi
+
+    jq_prog=".[] | select(.build_num < $CIRCLE_BUILD_NUM and (.status | test(\"running|pending|queued\")) $jq_filters) | .build_num"
+}
+
+
+if [[ "$0" != *bats* ]]; then
+set -e
+set -u
+set -o pipefail
+
+    branch=""
+    tag=""
+    rest=()
+    api_url="https://circleci.com/api/v1/project/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME?circle-token=$CIRCLE_TOKEN&limit=100"
+
+    parse_args "$@"
+    commit_message=$(git log -1 --pretty=%B)
+    if should_skip; then exit 0; fi
+    make_jq_prog
+
+    echo "Checking for running builds..."
+
+    while true; do
+        builds=$(curl -s -H "Accept: application/json" "$api_url" | jq "$jq_prog")
+        if [[ $builds ]]; then
+            echo "Waiting on builds:"
+            echo "$builds"
+        else
+            break
+        fi
+        echo "Retrying in 5 seconds..."
+        sleep 5
+    done
+
+    echo "Acquired lock"
+
+    if [[ "${#rest[@]}" -ne 0 ]]; then
+        "${rest[@]}"
+    fi
+fi

test/helpers/kubectl.ts

@@ -34,6 +34,12 @@ export async function createNamespace(namespace: string): Promise<void> {
   console.log(`Created namespace ${namespace}`);
 }
 
+export async function deleteNamespace(namespace: string): Promise<void> {
+  console.log(`Deleting namespace ${namespace}...`);
+  await exec(`./kubectl delete namespace ${namespace}`);
+  console.log(`Deleted namespace ${namespace}`);
+}
+
 export async function createSecret(
   secretName: string,
   namespace: string,

test/integration/kubernetes.test.ts

@@ -37,7 +37,15 @@ tap.test('deploy snyk-monitor', async (t) => {
 
 // Next we apply some sample workloads
 tap.test('deploy sample workloads', async (t) => {
-  await setup.createSampleDeployments();
+  const servicesNamespace = 'services';
+  const someImageWithSha = 'alpine@sha256:7746df395af22f04212cd25a92c1d6dbc5a06a0ca9579a229ef43008d4d1302a';
+  await Promise.all([
+    kubectl.applyK8sYaml('./test/fixtures/alpine-pod.yaml'),
+    kubectl.applyK8sYaml('./test/fixtures/nginx-replicationcontroller.yaml'),
+    kubectl.applyK8sYaml('./test/fixtures/redis-deployment.yaml'),
+    kubectl.applyK8sYaml('./test/fixtures/centos-deployment.yaml'),
+    kubectl.createDeploymentFromImage('alpine-from-sha', someImageWithSha, servicesNamespace),
+  ]);
   t.pass('successfully deployed sample workloads');
 });
 

test/setup/index.ts

@@ -6,6 +6,9 @@ import platforms from './platforms';
 import * as kubectl from '../helpers/kubectl';
 import * as waiters from './waiters';
 
+const testPlatform = process.env['TEST_PLATFORM'] || 'kind';
+const createCluster = process.env['CREATE_CLUSTER'] === 'true';
+
 function getIntegrationId(): string {
   const integrationId = uuidv4();
   console.log(`Generated new integration ID ${integrationId}`);
@@ -55,9 +58,13 @@ function createTestYamlDeployment(
 
 export async function removeMonitor(): Promise<void> {
   try {
-    await platforms.kind.delete();
+    if (createCluster) {
+      await platforms[testPlatform].delete();
+    } else {
+      await platforms[testPlatform].clean();
+    }
   } catch (error) {
-    console.log(`Could not delete kind cluster: ${error.message}`);
+    console.log(`Could not remove the Kubernetes-Monitor: ${error.message}`);
   }
 
   console.log('Removing KUBECONFIG environment variable...');
@@ -73,20 +80,20 @@ export async function removeMonitor(): Promise<void> {
 
 async function createEnvironment(): Promise<void> {
   // TODO: we probably want to use k8s-api for that, not kubectl
-  const servicesNamespace = 'services';
-  await kubectl.createNamespace(servicesNamespace);
+  await kubectl.createNamespace('services');
   // Small hack to prevent timing problems in CircleCI...
+  // TODO: should be replaced by actively waiting for the namespace to be created
   await sleep(5000);
+}
 
-  // Create imagePullSecrets for pulling private images from gcr.io.
-  // This is needed for deploying gcr.io images in KinD (this is _not_ used by snyk-monitor).
+async function createSecretForGcrIoAccess(): Promise<void> {
   const gcrSecretName = 'gcr-io';
   const gcrKubectlSecretsKeyPrefix = '--';
   const gcrSecretType = 'docker-registry';
   const gcrToken = getEnvVariableOrDefault('GCR_IO_SERVICE_ACCOUNT', '{}');
   await kubectl.createSecret(
     gcrSecretName,
-    servicesNamespace,
+    'services',
     {
       'docker-server': 'https://gcr.io',
       'docker-username': '_json_key',
@@ -129,17 +136,17 @@ export async function deployMonitor(): Promise<string> {
       'snyk/kubernetes-monitor:local',
     );
 
-    const testPlatform = process.env['TEST_PLATFORM'] || 'kind';
-    const createCluster = process.env['CREATE_CLUSTER'] === 'true';
     console.log(`platform chosen is ${testPlatform}, createCluster===${createCluster}`);
 
     await kubectl.downloadKubectl();
     if (createCluster) {
-      await platforms[testPlatform].create(imageNameAndTag);
+      await platforms[testPlatform].create();
     }
+    const remoteImageName = await platforms[testPlatform].loadImage(imageNameAndTag);
     await platforms[testPlatform].config();
     await createEnvironment();
-    const integrationId = await installKubernetesMonitor(imageNameAndTag);
+    await createSecretForGcrIoAccess();
+    const integrationId = await installKubernetesMonitor(remoteImageName);
     await waiters.waitForMonitorToBeReady();
     console.log(`Deployed the snyk-monitor with integration ID ${integrationId}`);
     return integrationId;
@@ -157,15 +164,3 @@ export async function deployMonitor(): Promise<string> {
     throw err;
   }
 }
-
-export async function createSampleDeployments(): Promise<void> {
-  const servicesNamespace = 'services';
-  const someImageWithSha = 'alpine@sha256:7746df395af22f04212cd25a92c1d6dbc5a06a0ca9579a229ef43008d4d1302a';
-  await Promise.all([
-    kubectl.applyK8sYaml('./test/fixtures/alpine-pod.yaml'),
-    kubectl.applyK8sYaml('./test/fixtures/nginx-replicationcontroller.yaml'),
-    kubectl.applyK8sYaml('./test/fixtures/redis-deployment.yaml'),
-    kubectl.applyK8sYaml('./test/fixtures/centos-deployment.yaml'),
-    kubectl.createDeploymentFromImage('alpine-from-sha', someImageWithSha, servicesNamespace),
-  ]);
-}

test/setup/platforms/eks.ts

@@ -1,13 +1,62 @@
-export async function createCluster(imageNameAndTag: string): Promise<void> {
-  exportKubeConfig();
+import { exec } from 'child-process-promise';
+import * as kubectl from '../../helpers/kubectl';
+
+export async function createCluster(): Promise<void> {
   throw new Error('Not implemented');
-  // process.env.KUBECONFIG = 'path-to-/kubeconfig-aws';
 }
 
 export async function deleteCluster(): Promise<void> {
   throw new Error('Not implemented');
 }
 
 export async function exportKubeConfig(): Promise<void> {
-  throw new Error('Not implemented');
+  await exec('aws eks update-kubeconfig --name runtime-experiments --kubeconfig ./kubeconfig');
+  process.env.KUBECONFIG = './kubeconfig';
+}
+
+export async function loadImageInCluster(imageNameAndTag: string): Promise<string> {
+  console.log(`Loading image ${imageNameAndTag} in ECR...`);
+
+  // update the `aws` CLI, the one in CircleCI's default image is outdated and doens't support eks
+  await exec('pip install awscli --ignore-installed six');
+
+  // TODO: assert all the vars are present before starting the setup?
+  // TODO: wipe out the data during teardown?
+  await exec(`aws configure set aws_access_key_id ${process.env['AWS_ACCESS_KEY_ID']}`);
+  await exec(`aws configure set aws_secret_access_key ${process.env['AWS_SECRET_ACCESS_KEY']}`);
+  await exec(`aws configure set region ${process.env['AWS_REGION']}`);
+
+  const ecrLogin = await exec('aws ecr get-login --region us-east-2 --no-include-email');
+
+  // aws ecr get-login returns something that looks like:
+  // docker login -U AWS -p <secret> https://the-address-of-ecr-we-should-use.com
+  // `docker tag` wants just the last part without https://
+  // `docker login` wants everything
+
+  // validate output so we don't execute malicious stuff
+  if (ecrLogin.stdout.indexOf('docker login -u AWS -p') !== 0) {
+    throw new Error('aws ecr get-login returned an unexpected output');
+  }
+
+  const targetImage = targetImageFromLoginDetails(ecrLogin.stdout);
+
+  await exec(`docker tag ${imageNameAndTag} ${targetImage}`);
+  await exec(ecrLogin.stdout);
+  await exec(`docker push ${targetImage}`);
+
+  console.log(`Loaded image ${targetImage} in ECR`);
+  return targetImage;
+}
+
+export async function clean(): Promise<void> {
+  await Promise.all([
+    kubectl.deleteNamespace('services'),
+    kubectl.deleteNamespace('snyk-monitor'),
+  ]);
+}
+
+function targetImageFromLoginDetails(ecrLoginOutput: string): string {
+  const split = ecrLoginOutput.split(' ');
+  const targetImagePrefix = split[split.length - 1].replace('https://', '').trim();
+  return `${targetImagePrefix}/snyk/kubernetes-monitor:local`;
 }

test/setup/platforms/index.ts

@@ -3,23 +3,31 @@ import * as eks from './eks';
 
 interface IPlatformSetup {
   // create a Kubernetes cluster
-  create: (imageNameAndTag: string) => Promise<void>;
+  create: () => Promise<void>;
+  // loads the image so Kubernetes may run it, return the name of the image in its registry's format
+  loadImage: (imageNameAndTag: string) => Promise<string>;
   // delete a Kubernetes cluster
   delete: () => Promise<void>;
   // set KUBECONFIG to point at the tested cluster
   config: () => Promise<void>;
+  // clean up whatever we littered an existing cluster with
+  clean: () => Promise<void>;
 }
 
 const kindSetup: IPlatformSetup = {
   create: kind.createCluster,
+  loadImage: kind.loadImageInCluster,
   delete: kind.deleteCluster,
   config: kind.exportKubeConfig,
+  clean: kind.clean,
 };
 
 const eksSetup: IPlatformSetup = {
   create: eks.createCluster,
+  loadImage: eks.loadImageInCluster,
   delete: eks.deleteCluster,
   config: eks.exportKubeConfig,
+  clean: eks.clean,
 };
 
 export default {

test/setup/platforms/kind.ts

@@ -6,11 +6,10 @@ import * as needle from 'needle';
 
 const clusterName = 'kind';
 
-export async function createCluster(imageNameAndTag: string): Promise<void> {
+export async function createCluster(): Promise<void> {
   const osDistro = platform();
   await download(osDistro);
   await createKindCluster(clusterName);
-  await loadImageInCluster(imageNameAndTag);
 }
 
 export async function deleteCluster(): Promise<void> {
@@ -27,6 +26,18 @@ export async function exportKubeConfig(): Promise<void> {
   console.log('Exported K8s config!');
 }
 
+export async function loadImageInCluster(imageNameAndTag: string): Promise<string> {
+  console.log(`Loading image ${imageNameAndTag} in KinD cluster...`);
+  await exec(`./kind load docker-image ${imageNameAndTag}`);
+  console.log(`Loaded image ${imageNameAndTag}`);
+  return imageNameAndTag;
+}
+
+export async function clean(): Promise<void> {
+  // just delete the cluster instead
+  throw new Error('Not implemented');
+}
+
 async function download(osDistro: string): Promise<void> {
   try {
     accessSync(resolve(process.cwd(), 'kind'), constants.R_OK);
@@ -62,9 +73,3 @@ async function createKindCluster(clusterName, kindImageTag = 'latest'): Promise<
   await exec(`./kind create cluster --name="${clusterName}" ${kindImageArgument}`);
   console.log(`Created cluster ${clusterName}!`);
 }
-
-async function loadImageInCluster(imageNameAndTag): Promise<void> {
-  console.log(`Loading image ${imageNameAndTag} in cluster...`);
-  await exec(`./kind load docker-image ${imageNameAndTag}`);
-  console.log(`Loaded image ${imageNameAndTag}`);
-}