Merge pull request #268 from snyk/feat/pulls-from-ecr

Feat/pulls from ecr

Author: Amir Moualem

package-lock.json

@@ -33,6 +33,7 @@
     "@types/node": "^10.12.5",
     "@types/sinon": "^7.5.1",
     "async": "^2.6.2",
+    "aws-sdk": "^2.596.0",
     "bunyan": "^1.8.12",
     "child-process-promise": "^2.2.1",
     "lru-cache": "^5.1.1",

package.json

@@ -1,10 +1,15 @@
 import { spawn, SpawnPromiseResult } from 'child-process-promise';
 import logger = require('./logger');
 
-export function exec(bin: string, ...args: string[]):
+export interface IProcessArgument {
+  body: string;
+  sanitise: boolean;
+}
+
+export function exec(bin: string, ...processArgs: IProcessArgument[]):
     Promise<SpawnPromiseResult> {
   if (process.env.DEBUG === 'true') {
-    args.push('--debug');
+    processArgs.push({body: '--debug', sanitise: false});
   }
 
   // Ensure we're not passing the whole environment to the shelled out process...
@@ -13,10 +18,12 @@ export function exec(bin: string, ...args: string[]):
     PATH: process.env.PATH,
   };
 
-  return spawn(bin, args, { env, capture: [ 'stdout', 'stderr' ] })
+  const allArguments = processArgs.map((arg) => arg.body);
+  return spawn(bin, allArguments, { env, capture: [ 'stdout', 'stderr' ] })
     .catch((error) => {
       const message = (error && error.stderr) || 'Unknown reason';
-      logger.warn({message, bin, args}, 'could not spawn the process');
+      const loggableArguments = processArgs.filter((arg) => !arg.sanitise).map((arg) => arg.body);
+      logger.warn({message, bin, loggableArguments}, 'child process failure');
       throw error;
     });
 }

src/common/process.ts

@@ -0,0 +1,68 @@
+import * as aws from 'aws-sdk';
+
+import logger = require('../common/logger');
+
+export async function getSourceCredentials(imageSource: string): Promise<string | undefined> {
+  // TODO is this the best way we can determine the image's source?
+  if (imageSource.indexOf('.ecr.') !== -1) {
+    const ecrRegion = ecrRegionFromFullImageName(imageSource);
+    return getEcrCredentials(ecrRegion);
+  }
+  return undefined;
+}
+
+function getEcrCredentials(region: string): Promise<string> {
+  return new Promise(async (resolve, reject) => {
+    const ecr = new aws.ECR({region});
+    return ecr.getAuthorizationToken({}, (err, data) => {
+      if (err) {
+        return reject(err);
+      }
+
+      if (!(
+        data &&
+        data.authorizationData &&
+        Array.isArray(data.authorizationData) &&
+        data.authorizationData.length > 0
+      )) {
+        return reject('unexpected data format from ecr.getAuthorizationToken');
+      }
+
+      const authorizationTokenBase64 = data.authorizationData[0].authorizationToken;
+
+      if (!authorizationTokenBase64) {
+        return reject('empty authorization token from ecr.getAuthorizationToken');
+      }
+
+      const buff = new Buffer(authorizationTokenBase64, 'base64');
+      const userColonPassword = buff.toString('utf-8');
+      return resolve(userColonPassword);
+    });
+  });
+}
+
+export function ecrRegionFromFullImageName(imageFullName: string): string {
+  // should look like this
+  // aws_account_id.dkr.ecr.region.amazonaws.com/my-web-app:latest
+  // https://docs.aws.amazon.com/AmazonECR/latest/userguide/ECR_on_EKS.html
+  try {
+    const [registry, repository] = imageFullName.split('/');
+    if (!repository) {
+      throw new Error('ECR image full name missing repository');
+    }
+
+    const parts = registry.split('.');
+    if (!(
+      parts.length === 6 &&
+      parts[1] === 'dkr' &&
+      parts[2] === 'ecr' &&
+      parts[4] === 'amazonaws'
+    )) {
+      throw new Error('ECR image full name in unexpected format');
+    }
+    return parts[3];
+  } catch (err) {
+    logger.error({err, imageFullName}, 'failed extracting ECR region from image full name');
+    throw err;
+  }
+}

src/images/credentials.ts

@@ -1,7 +1,9 @@
-import { SkopeoRepositoryType } from '../kube-scanner/types';
 import { SpawnPromiseResult } from 'child-process-promise';
-import { exec } from '../common/process';
-import config = require('../common/config');
+
+import * as processWrapper from '../common/process';
+import * as config from'../common/config';
+import * as credentials from './credentials';
+import { SkopeoRepositoryType } from '../kube-scanner/types';
 
 function getUniqueIdentifier(): string {
   const [seconds, nanoseconds] = process.hrtime();
@@ -28,12 +30,27 @@ function prefixRespository(target: string, type: SkopeoRepositoryType): string {
   }
 }
 
-export function pull(
+export async function pull(
   image: string,
   destination: string,
 ): Promise<SpawnPromiseResult> {
-  return exec('skopeo', 'copy',
-    prefixRespository(image, SkopeoRepositoryType.ImageRegistry),
-    prefixRespository(destination, SkopeoRepositoryType.DockerArchive),
-  );
+  const creds = await credentials.getSourceCredentials(image);
+  const credentialsParameters = getCredentialParameters(creds);
+
+  const args: Array<processWrapper.IProcessArgument> = [];
+  args.push({body: 'copy', sanitise: false});
+  args.push(...credentialsParameters);
+  args.push({body: prefixRespository(image, SkopeoRepositoryType.ImageRegistry), sanitise: false});
+  args.push({body: prefixRespository(destination, SkopeoRepositoryType.DockerArchive), sanitise: false});
+
+  return processWrapper.exec('skopeo', ...args);
+}
+
+export function getCredentialParameters(credentials: string | undefined): Array<processWrapper.IProcessArgument> {
+  const credentialsParameters: Array<processWrapper.IProcessArgument> = [];
+  if (credentials) {
+    credentialsParameters.push({body: '--src-creds', sanitise: true});
+    credentialsParameters.push({body: credentials, sanitise: true});
+  }
+  return credentialsParameters;
 }

src/images/skopeo.ts

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: debian-ecr
+  namespace: services
+spec:
+  selector:
+    matchLabels:
+      app: debian-ecr
+  template:
+    metadata:
+      labels:
+        app: debian-ecr
+    spec:
+      containers:
+      - name: debian-ecr
+        image: 291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/debian:10
+        imagePullPolicy: Always
+        securityContext: {}
+        command: ['sh', '-c', 'echo Hello from ECR alpine pod! && sleep 360000']

test/fixtures/private-registries/debian-deployment-ecr.yaml

@@ -134,7 +134,7 @@ tap.test('snyk-monitor sends correct data to homebase after adding another deplo
     'snyk-monitor sent expected data to homebase in the expected timeframe');
 });
 
-tap.test('snyk-monitor pulls images from private registries and sends data to homebase', async (t) => {
+tap.test('snyk-monitor pulls images from a private gcr.io registry and sends data to homebase', async (t) => {
   t.plan(3);
 
   const deploymentName = 'debian-gcr-io';
@@ -144,7 +144,42 @@ tap.test('snyk-monitor pulls images from private registries and sends data to ho
   const imageName = 'gcr.io/snyk-k8s-fixtures/debian';
 
   await kubectl.applyK8sYaml('./test/fixtures/private-registries/debian-deployment-gcr-io.yaml');
-  console.log(`Begin polling upstream for the expected private registry workload with integration ID ${integrationId}...`);
+  console.log(`Begin polling upstream for the expected private gcr.io image with integration ID ${integrationId}...`);
+
+  const validatorFn: WorkloadLocatorValidator = (workloads) => {
+    return workloads !== undefined &&
+      workloads.find((workload) => workload.name === deploymentName &&
+        workload.type === WorkloadKind.Deployment) !== undefined;
+  };
+
+  const homebaseTestResult = await validateHomebaseStoredData(
+    validatorFn, `api/v2/workloads/${integrationId}/${clusterName}/${namespace}`);
+  t.ok(homebaseTestResult, 'snyk-monitor sent expected data to upstream in the expected timeframe');
+
+  const depGraphResult = await getHomebaseResponseBody(
+    `api/v1/dependency-graphs/${integrationId}/${clusterName}/${namespace}/${deploymentType}/${deploymentName}`);
+  t.ok('dependencyGraphResults' in depGraphResult,
+    'expected dependencyGraphResults field to exist in /dependency-graphs response');
+  t.ok('imageMetadata' in JSON.parse(depGraphResult.dependencyGraphResults[imageName]),
+    'snyk-monitor sent expected data to upstream in the expected timeframe');
+});
+
+tap.test('snyk-monitor pulls images from a private ECR and sends data to homebase', async (t) => {
+  if (process.env['TEST_PLATFORM'] !== 'eks') {
+    t.pass('Not testing private ECR images because we\'re not running in EKS');
+    return;
+  }
+
+  t.plan(3);
+
+  const deploymentName = 'debian-ecr';
+  const namespace = 'services';
+  const clusterName = 'Default cluster';
+  const deploymentType = WorkloadKind.Deployment;
+  const imageName = '291964488713.dkr.ecr.us-east-2.amazonaws.com/snyk/debian';
+
+  await kubectl.applyK8sYaml('./test/fixtures/private-registries/debian-deployment-ecr.yaml');
+  console.log(`Begin polling upstream for the expected private ECR image with integration ID ${integrationId}...`);
 
   const validatorFn: WorkloadLocatorValidator = (workloads) => {
     return workloads !== undefined &&