Introduce LINKERD_DOCKER_REGISTRY and flexibilize CI workflows (linke…

…rd#6782) This allows overriding the docker images registry through the `LINKERD_DOCKER_REGISTRY` environment variable, as the CLI's `--registry` flag does (analogously to the `LINKERD_NAMESPACE` env var). In the `integration_tests.yml` workflow we've been wanting to test directly against GH's registry instead of having to endure cr.l5d.io's problems. To achieve this we set `DOCKER_REGISTRY: ghcr.io/linkerd` globally, and all the docker scripts will use it accordingly. And before triggering the integration tests, we copy that value to `LINKERD_DOCKER_REGISTRY`. Note that the CLI and Helm manifests will continue to point to `cr.l5d.io` by default when this is not overridden. `release.yml` still points to `cr.l5.io` because that's what users will experience, so we want to test that right before releasing. Another goal was to be able to run these workflows in forks by overriding the global `DOCKER_REGISTRY` var pointing to the fork's own registry. ## Nice side-effects - This fixes the policy controller's image not being affected by the `--registry` flag when installing through the CLI. - The `--registry` flag (and overriding `LINKERD_DOCKER_REGISTRY` env var) have been added to `linkerd jaeger install`.
abhijeetgauravm · Sep 1, 2021 · cf7ccc3 · cf7ccc3
1 parent 8f15683
commit cf7ccc3
Show file tree

Hide file tree

Showing 16 changed files with 182 additions and 75 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -12,6 +12,7 @@ permissions:
   contents: read
 env:
   GH_ANNOTATION: true
+  DOCKER_REGISTRY: ghcr.io/linkerd
 jobs:
   docker_build:
     runs-on: ubuntu-20.04
@@ -28,9 +29,6 @@ jobs:
       run: |
         . bin/_tag.sh
         echo "TAG=$(CI_FORCE_CLEAN=1 bin/root-tag)" >> $GITHUB_ENV
-
-        . bin/_docker.sh
-        echo "DOCKER_REGISTRY=cr.l5d.io/linkerd" >> $GITHUB_ENV
         echo "DOCKER_BUILDKIT_CACHE=${{ runner.temp }}/.buildx-cache" >> $GITHUB_ENV
     - name: Cache docker layers
       uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
@@ -54,7 +52,7 @@ jobs:
         ARCHIVES: /home/runner/archives
       run: |
         mkdir -p $ARCHIVES
-        docker save "cr.l5d.io/linkerd/${{ matrix.target }}:$TAG" > $ARCHIVES/${{ matrix.target }}.tar
+        docker save "$DOCKER_REGISTRY/${{ matrix.target }}:$TAG" > $ARCHIVES/${{ matrix.target }}.tar
     # `with.path` values do not support environment variables yet, so an
     # absolute path is used here.
     #
@@ -99,9 +97,6 @@ jobs:
       run: |
         . bin/_tag.sh
         echo "TAG=$(CI_FORCE_CLEAN=1 bin/root-tag)" >> $GITHUB_ENV
-
-        . bin/_docker.sh
-        echo "DOCKER_REGISTRY=cr.l5d.io/linkerd" >> $GITHUB_ENV
     - name: Download image archives
       uses: actions/download-artifact@3be87be14a055c47b01d3bd88f8fe02320a9bb60
       with:
@@ -112,11 +107,13 @@ jobs:
     - name: Install CLI
       run: |
         # Copy the CLI out of the local cli-bin container.
-        container_id=$(docker create "cr.l5d.io/linkerd/cli-bin:$TAG")
+        container_id=$(docker create "$DOCKER_REGISTRY/cli-bin:$TAG")
         docker cp $container_id:/out/linkerd-linux-amd64 "$HOME/.linkerd"
 
         # Validate the CLI version matches the current build tag.
         [[ "$TAG" == "$($HOME/.linkerd version --short --client)" ]]
     - name: Run integration tests
+      env:
+        LINKERD_DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}
       run: |
         bin/tests --images archive --cleanup-docker --name ${{ matrix.integration_test }} "$HOME/.linkerd"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,6 +7,7 @@ permissions:
   contents: read
 env:
   GH_ANNOTATION: true
+  DOCKER_REGISTRY: cr.l5d.io/linkerd
 jobs:
   docker_build:
     runs-on: ubuntu-20.04
@@ -23,9 +24,6 @@ jobs:
       run: |
         . bin/_tag.sh
         echo "TAG=$(CI_FORCE_CLEAN=1 bin/root-tag)" >> $GITHUB_ENV
-
-        . bin/_docker.sh
-        echo "DOCKER_REGISTRY=ghcr.io/linkerd" >> $GITHUB_ENV
         echo "DOCKER_BUILDKIT_CACHE=${{ runner.temp }}/.buildx-cache" >> $GITHUB_ENV
     - name: Cache docker layers
       uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
@@ -90,7 +88,7 @@ jobs:
         echo "${{ secrets.DOCKER_GHCR_PAT }}" | docker login ghcr.io -u "${{ secrets.DOCKER_GHCR_USERNAME }}" --password-stdin
         docker buildx build --push ./policy-controller/ \
           -f ./policy-controller/${{ matrix.arch }}.dockerfile \
-          -t ghcr.io/linkerd/policy-controller:$TAG-${{ matrix.arch }}
+          -t $DOCKER_REGISTRY/policy-controller:$TAG-${{ matrix.arch }}
 
   policy_controller_manifest:
     runs-on: ubuntu-20.04
@@ -105,22 +103,22 @@ jobs:
         echo "TAG=$(CI_FORCE_CLEAN=1 bin/root-tag)" >> $GITHUB_ENV
     - name: Create multiarch manifest
       run: |
-        docker manifest create ghcr.io/linkerd/policy-controller:${TAG} \
-            ghcr.io/linkerd/policy-controller:${TAG}-amd64 \
-            ghcr.io/linkerd/policy-controller:${TAG}-arm64 \
-            ghcr.io/linkerd/policy-controller:${TAG}-arm
+        docker manifest create $DOCKER_REGISTRY/policy-controller:$TAG \
+            $DOCKER_REGISTRY/policy-controller:$TAG-amd64 \
+            $DOCKER_REGISTRY/policy-controller:$TAG-arm64 \
+            $DOCKER_REGISTRY/policy-controller:$TAG-arm
     - name: Annotate multiarch manifest
       run: |
-        docker manifest annotate ghcr.io/linkerd/policy-controller:$TAG \
-            ghcr.io/linkerd/policy-controller:${TAG}-amd64 --os=linux --arch=amd64
-        docker manifest annotate ghcr.io/linkerd/policy-controller:$TAG \
-            ghcr.io/linkerd/policy-controller:${TAG}-arm64 --os=linux --arch=arm64
-        docker manifest annotate ghcr.io/linkerd/policy-controller:$TAG \
-            ghcr.io/linkerd/policy-controller:${TAG}-arm --os=linux --arch=arm
+        docker manifest annotate $DOCKER_REGISTRY/policy-controller:$TAG \
+            $DOCKER_REGISTRY/policy-controller:${TAG}-amd64 --os=linux --arch=amd64
+        docker manifest annotate $DOCKER_REGISTRY/policy-controller:$TAG \
+            $DOCKER_REGISTRY/policy-controller:${TAG}-arm64 --os=linux --arch=arm64
+        docker manifest annotate $DOCKER_REGISTRY/policy-controller:$TAG \
+            $DOCKER_REGISTRY/policy-controller:${TAG}-arm --os=linux --arch=arm
     - name: Push multiarch manifest
       run: |
         echo "${{ secrets.DOCKER_GHCR_PAT }}" | docker login ghcr.io -u "${{ secrets.DOCKER_GHCR_USERNAME }}" --password-stdin
-        docker manifest push ghcr.io/linkerd/policy-controller:$TAG
+        docker manifest push $DOCKER_REGISTRY/policy-controller:$TAG
 
   # todo: Keep in sync with `integration_tests.yml`
   windows_static_cli_tests:
@@ -180,6 +178,8 @@ jobs:
         echo "CMD=$CMD" >> $GITHUB_ENV
         echo "TAG=$TAG" >> $GITHUB_ENV
     - name: Run integration tests
+      env:
+        LINKERD_DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}
       run: |
         bin/docker-pull-binaries $TAG
         # Validate the CLI version matches the current build tag.
@@ -220,6 +220,7 @@ jobs:
     - name: Run integration tests
       env:
         RUN_ARM_TEST: 1
+        LINKERD_DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}
       run: bin/tests --name deep --images preload --skip-cluster-create "$CMD"
     - name: CNI tests
       run: |

diff --git a/cli/cmd/inject_test.go b/cli/cmd/inject_test.go
@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	"github.com/linkerd/linkerd2/pkg/charts/linkerd2"
+	"github.com/linkerd/linkerd2/pkg/cmd"
 	"github.com/linkerd/linkerd2/pkg/k8s"
 	"github.com/linkerd/linkerd2/testutil"
 )
@@ -760,7 +761,7 @@ func TestOverwriteRegistry(t *testing.T) {
 	for i, tc := range testCases {
 		tc := tc // pin
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			actual := registryOverride(tc.image, tc.registry)
+			actual := cmd.RegistryOverride(tc.image, tc.registry)
 			if actual != tc.expected {
 				t.Fatalf("expected %q, but got %q", tc.expected, actual)
 			}

diff --git a/cli/cmd/install-cni-plugin.go b/cli/cmd/install-cni-plugin.go
@@ -9,6 +9,8 @@ import (
 	"github.com/linkerd/linkerd2/pkg/charts"
 	cnicharts "github.com/linkerd/linkerd2/pkg/charts/cni"
 	"github.com/linkerd/linkerd2/pkg/charts/static"
+	"github.com/linkerd/linkerd2/pkg/cmd"
+	"github.com/linkerd/linkerd2/pkg/flags"
 	"github.com/linkerd/linkerd2/pkg/version"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -66,8 +68,12 @@ func (options *cniPluginOptions) validate() error {
 }
 
 func (options *cniPluginOptions) pluginImage() string {
+	// env var overrides CLI flag
+	if override := os.Getenv(flags.EnvOverrideDockerRegistry); override != "" {
+		return cmd.RegistryOverride(options.cniPluginImage, override)
+	}
 	if options.dockerRegistry != defaultDockerRegistry {
-		return registryOverride(options.cniPluginImage, options.dockerRegistry)
+		return cmd.RegistryOverride(options.cniPluginImage, options.dockerRegistry)
 	}
 	return options.cniPluginImage
 }
@@ -95,7 +101,8 @@ assumes that the 'linkerd install' command will be executed with the
 	}
 
 	cmd.PersistentFlags().StringVarP(&options.linkerdVersion, "linkerd-version", "v", options.linkerdVersion, "Tag to be used for Linkerd images")
-	cmd.PersistentFlags().StringVar(&options.dockerRegistry, "registry", options.dockerRegistry, "Docker registry to pull images from")
+	cmd.PersistentFlags().StringVar(&options.dockerRegistry, "registry", options.dockerRegistry,
+		fmt.Sprintf("Docker registry to pull images from ($%s)", flags.EnvOverrideDockerRegistry))
 	cmd.PersistentFlags().Int64Var(&options.proxyUID, "proxy-uid", options.proxyUID, "Run the proxy under this user ID")
 	cmd.PersistentFlags().UintVar(&options.inboundPort, "inbound-port", options.inboundPort, "Proxy port to use for inbound traffic")
 	cmd.PersistentFlags().UintVar(&options.outboundPort, "outbound-port", options.outboundPort, "Proxy port to use for outbound traffic")

diff --git a/cli/cmd/options.go b/cli/cmd/options.go
@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"errors"
+	"os"
 
 	"fmt"
 	"io/ioutil"
@@ -12,6 +13,8 @@ import (
 
 	"github.com/linkerd/linkerd2/cli/flag"
 	l5dcharts "github.com/linkerd/linkerd2/pkg/charts/linkerd2"
+	"github.com/linkerd/linkerd2/pkg/cmd"
+	flagspkg "github.com/linkerd/linkerd2/pkg/flags"
 	"github.com/linkerd/linkerd2/pkg/inject"
 	"github.com/linkerd/linkerd2/pkg/issuercerts"
 	"github.com/linkerd/linkerd2/pkg/k8s"
@@ -305,15 +308,6 @@ func makeProxyFlags(defaults *l5dcharts.Values) ([]flag.Flag, *pflag.FlagSet) {
 				return nil
 			}),
 
-		flag.NewStringFlag(proxyFlags, "registry", defaultDockerRegistry, "Docker registry to pull images from",
-			func(values *l5dcharts.Values, value string) error {
-				values.ControllerImage = registryOverride(values.ControllerImage, value)
-				values.DebugContainer.Image.Name = registryOverride(values.DebugContainer.Image.Name, value)
-				values.Proxy.Image.Name = registryOverride(values.Proxy.Image.Name, value)
-				values.ProxyInit.Image.Name = registryOverride(values.ProxyInit.Image.Name, value)
-				return nil
-			}),
-
 		flag.NewStringFlag(proxyFlags, "image-pull-policy", defaults.ImagePullPolicy,
 			"Docker image pull policy", func(values *l5dcharts.Values, value string) error {
 				values.ImagePullPolicy = value
@@ -425,6 +419,21 @@ func makeProxyFlags(defaults *l5dcharts.Values) ([]flag.Flag, *pflag.FlagSet) {
 			}),
 	}
 
+	registryFlag := flag.NewStringFlag(proxyFlags, "registry", defaultDockerRegistry,
+		fmt.Sprintf("Docker registry to pull images from ($%s)", flagspkg.EnvOverrideDockerRegistry),
+		func(values *l5dcharts.Values, value string) error {
+			values.ControllerImage = cmd.RegistryOverride(values.ControllerImage, value)
+			values.PolicyController.Image.Name = cmd.RegistryOverride(values.PolicyController.Image.Name, value)
+			values.DebugContainer.Image.Name = cmd.RegistryOverride(values.DebugContainer.Image.Name, value)
+			values.Proxy.Image.Name = cmd.RegistryOverride(values.Proxy.Image.Name, value)
+			values.ProxyInit.Image.Name = cmd.RegistryOverride(values.ProxyInit.Image.Name, value)
+			return nil
+		})
+	if reg := os.Getenv(flagspkg.EnvOverrideDockerRegistry); reg != "" {
+		registryFlag.Set(reg)
+	}
+	flags = append(flags, registryFlag)
+
 	proxyFlags.MarkDeprecated("proxy-memory", "use --proxy-memory-request instead")
 	proxyFlags.MarkDeprecated("proxy-cpu", "use --proxy-cpu-request instead")
 

diff --git a/cli/cmd/root.go b/cli/cmd/root.go
@@ -11,6 +11,7 @@ import (
 	jaeger "github.com/linkerd/linkerd2/jaeger/cmd"
 	multicluster "github.com/linkerd/linkerd2/multicluster/cmd"
 	pkgcmd "github.com/linkerd/linkerd2/pkg/cmd"
+	"github.com/linkerd/linkerd2/pkg/flags"
 	"github.com/linkerd/linkerd2/pkg/healthcheck"
 	viz "github.com/linkerd/linkerd2/viz/cmd"
 	log "github.com/sirupsen/logrus"
@@ -79,7 +80,7 @@ var RootCmd = &cobra.Command{
 			log.SetLevel(log.PanicLevel)
 		}
 
-		controlPlaneNamespaceFromEnv := os.Getenv("LINKERD_NAMESPACE")
+		controlPlaneNamespaceFromEnv := os.Getenv(flags.EnvOverrideNamespace)
 		if controlPlaneNamespace == defaultLinkerdNamespace && controlPlaneNamespaceFromEnv != "" {
 			controlPlaneNamespace = controlPlaneNamespaceFromEnv
 		}
@@ -93,7 +94,9 @@ var RootCmd = &cobra.Command{
 }
 
 func init() {
-	RootCmd.PersistentFlags().StringVarP(&controlPlaneNamespace, "linkerd-namespace", "L", defaultLinkerdNamespace, "Namespace in which Linkerd is installed ($LINKERD_NAMESPACE)")
+	RootCmd.PersistentFlags().StringVarP(&controlPlaneNamespace, "linkerd-namespace", "L",
+		defaultLinkerdNamespace,
+		fmt.Sprintf("Namespace in which Linkerd is installed ($%s)", flags.EnvOverrideNamespace))
 	RootCmd.PersistentFlags().StringVarP(&cniNamespace, "cni-namespace", "", defaultCNINamespace, "Namespace in which the Linkerd CNI plugin is installed")
 	RootCmd.PersistentFlags().StringVar(&kubeconfigPath, "kubeconfig", "", "Path to the kubeconfig file to use for CLI requests")
 	RootCmd.PersistentFlags().StringVar(&kubeContext, "context", "", "Name of the kubeconfig context to use")
@@ -143,22 +146,6 @@ func deprecateCmd(cmd *cobra.Command) *cobra.Command {
 	return cmd
 }
 
-// registryOverride replaces the registry-portion of the provided image with the provided registry.
-func registryOverride(image, newRegistry string) string {
-	if image == "" {
-		return image
-	}
-	registry := newRegistry
-	if registry != "" && !strings.HasSuffix(registry, slash) {
-		registry += slash
-	}
-	imageName := image
-	if strings.Contains(image, slash) {
-		imageName = image[strings.LastIndex(image, slash)+1:]
-	}
-	return registry + imageName
-}
-
 func flattenFlags(flags ...[]flag.Flag) []flag.Flag {
 	out := []flag.Flag{}
 	for _, f := range flags {