Merge pull request #461 from BenTheElder/containerd

Images Improvements [Breaking Changes]

cmd/kind/load/docker-image/docker-image.go

@@ -114,6 +114,7 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
 	}
 
 	// Load the image into every node
+	// TODO(bentheelder): this should probably be concurrent
 	for _, node := range selectedNodes {
 		if err := loadImage(imageTarPath, &node); err != nil {
 			return err
@@ -123,20 +124,10 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
 }
 
 func loadImage(imageTarName string, node *clusternodes.Node) error {
-	// Copy image tar to each node
 	f, err := os.Open(imageTarName)
 	if err != nil {
 		return errors.Wrap(err, "failed to open image")
 	}
 	defer f.Close()
-
-	// Load image into each node
-	cmd := node.Command(
-		"docker", "load",
-	)
-	cmd.SetStdin(f)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrap(err, "failed to load image")
-	}
-	return nil
+	return node.LoadImageArchive(f)
 }

cmd/kind/load/image-archive/image-archive.go

@@ -98,6 +98,7 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
 	}
 
 	// Load the image into every node
+	// TODO(bentheelder): this should probably be concurrent
 	for _, node := range selectedNodes {
 		if err := loadImage(args[0], &node); err != nil {
 			return err
@@ -107,20 +108,10 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) error {
 }
 
 func loadImage(imageTarName string, node *clusternodes.Node) error {
-	// Copy image tar to each node
 	f, err := os.Open(imageTarName)
 	if err != nil {
 		return errors.Wrap(err, "failed to open image")
 	}
 	defer f.Close()
-
-	// Load image into each node
-	cmd := node.Command(
-		"docker", "load",
-	)
-	cmd.SetStdin(f)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrap(err, "failed to load image")
-	}
-	return nil
+	return node.LoadImageArchive(f)
 }

images/base/Dockerfile

@@ -12,16 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# kind cluster base image, built on ubuntu:18.04
-#
-# To this we add systemd, CNI, and other tools needed to run Kubeadm
+# kind cluster base image
 #
 # For systemd + docker configuration used below, see the following references:
 # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
 # https://developers.redhat.com/blog/2016/09/13/running-systemd-in-a-non-privileged-container/
 
-ARG BASE_IMAGE="ubuntu:18.04"
+ARG BASE_IMAGE="ubuntu:19.04"
 FROM ${BASE_IMAGE}
 
 # setting DEBIAN_FRONTEND=noninteractive stops some apt warnings, this is not 
@@ -34,50 +32,32 @@ RUN chmod +x /usr/local/bin/clean-install
 # Get dependencies
 # The base image already has: ssh, apt, snapd
 # This is broken down into (each on a line):
-# - packages necessary for installing docker
 # - packages needed to run services (systemd)
-# - packages needed for docker / hyperkube / kubernetes components
-# - misc packages (utilities we use in our own tooling)
+# - CRI (containerd)
+# - packages needed for kubernetes components
+# - misc packages kind uses itself
 # Then we cleanup (removing unwanted systemd services)
 # Finally we disable kmsg in journald
 # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/
 RUN clean-install \
-      apt-transport-https ca-certificates curl software-properties-common gnupg2 lsb-release \
       systemd systemd-sysv libsystemd0 \
-      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod aufs-tools \
-      bash rsync \
+      containerd \
+      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \
+      bash ca-certificates curl rsync \
     && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
     && rm -f /lib/systemd/system/multi-user.target.wants/* \
     && rm -f /etc/systemd/system/*.wants/* \
     && rm -f /lib/systemd/system/local-fs.target.wants/* \
     && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
     && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
     && rm -f /lib/systemd/system/basic.target.wants/* \
-    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf
-
-# Install docker, which needs to happen after we install some of the packages above
-# based on https://docs.docker.com/install/linux/docker-ce/ubuntu/#set-up-the-repository
-# and https://kubernetes.io/docs/setup/independent/install-kubeadm/#installing-docker
-# - get docker's GPG key
-# - add the fingerprint
-# - add the repository
-# - update apt, install docker, cleanup
-# NOTE: 18.09 is officially supported by Kubernetes currently, so we pin to that.
-# https://kubernetes.io/docs/tasks/tools/install-kubeadm/
-ARG DOCKER_VERSION="5:18.09.*"
-# another temporary env, not a real argument. setting this to a non-zero value
-# silences this warning from apt-key:
-# "Warning: apt-key output should not be parsed (stdout is not a terminal)"
-ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE="false"
-RUN curl -fsSL "https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg" | apt-key add - \
-    && apt-key fingerprint 0EBFCD88 \
-    && add-apt-repository \
-        "deb https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable" \
-    && clean-install "docker-ce=${DOCKER_VERSION}"
+    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
+    && systemctl enable containerd \
+    && echo "done installing packages"
 
-# Enable CRI for containerd
-RUN mkdir -p /etc/docker && echo '{"cri-containerd": true}' > /etc/docker/daemon.json
-RUN mkdir -p /etc/containerd && containerd config default > /etc/containerd/config.toml
+# debug containerd version and create default config
+RUN containerd --version \
+    && mkdir -p /etc/containerd && containerd config default > /etc/containerd/config.toml
 
 # Install CNI binaries to /opt/cni/bin
 # TODO(bentheelder): doc why / what here
@@ -92,6 +72,11 @@ RUN export ARCH=$(dpkg --print-architecture) \
     && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \
     && rm -rf /tmp/cni.tgz
 
+# Install crictl to /usr/local/bin
+ARG CRICTL_VERSION="v1.14.0"
+RUN export ARCH=$(dpkg --print-architecture) \
+    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin
+
 # tell systemd that it is in docker (it will check for the container env)
 # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 ENV container docker
@@ -103,17 +88,11 @@ STOPSIGNAL SIGRTMIN+3
 # basically this just lets us set up some things before continuing on to systemd
 # while preserving that systemd is PID1
 # for how we leverage this, see pkg/cluster
-COPY [ "entrypoint/entrypoint", "/usr/local/bin/" ]
+COPY [ "entrypoint", "/usr/local/bin/" ]
 # We need systemd to be PID1 to run the various services (docker, kubelet, etc.)
 # NOTE: this is *only* for documentation, the entrypoint is overridden at runtime
 ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 
-# the docker graph must be a volume to avoid overlay on overlay
-# NOTE: we do this last because changing a volume with a Dockerfile must
-# occur before defining it.
-# See: https://docs.docker.com/engine/reference/builder/#volume
-VOLUME [ "/var/lib/docker" ]
-
 # TODO(bentheelder): deal with systemd MAC address assignment
 # https://github.com/systemd/systemd/issues/3374#issuecomment-288882355
 # https://github.com/systemd/systemd/issues/3374#issuecomment-339258483

images/base/clean-install

@@ -26,11 +26,14 @@ if [ $# = 0 ]; then
 fi
 
 apt-get update
-apt-get install -y --no-install-recommends $@
+apt-get install -y --no-install-recommends "$@"
 apt-get clean -y
 rm -rf \
    /var/cache/debconf/* \
    /var/lib/apt/lists/* \
    /var/log/* \
    /tmp/* \
-   /var/tmp/*
+   /var/tmp/* \
+   /usr/share/doc/* \
+   /usr/share/man/* \
+   /usr/share/local/*

images/base/entrypoint

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+configure_proxy() {
+    mkdir -p /etc/systemd/system/containerd.service.d/
+    cat <<EOF >/etc/systemd/system/containerd.service.d/http-proxy.conf
+[Service]
+Environment="HTTP_PROXY=${HTTP_PROXY:-}"
+Environment="HTTPS_PROXY=${HTTPS_PROXY:-}"
+Environment="NO_PROXY=${NO_PROXY:-}"
+EOF
+}
+
+fix_mount() {
+    # necessary only when userns-remap is enabled on the host, but harmless
+    # The binary /bin/mount should be owned by root and have the setuid bit
+    chown root:root /bin/mount
+    chmod -s /bin/mount
+
+    # systemd-in-a-container should have read only /sys
+    # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+    # however, we need other things from `docker run --privileged` ...
+    # and this flag also happens to make /sys rw, amongst other things
+    mount -o remount,ro /sys
+}
+
+fix_machine_id() {
+    # Deletes the machine-id embedded in the node image and generates a new one.
+	# This is necessary because both kubelet and other components like weave net
+	# use machine-id internally to distinguish nodes.
+	rm -f /etc/machine-id
+	systemd-machine-id-setup
+}
+
+fix_product_name() {
+    # this is a small fix to hide the underlying hardware and fix issue #426
+    # https://github.com/kubernetes-sigs/kind/issues/426
+    echo "kind" > /kind/product_name
+    mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
+}
+
+# run pre-init fixups
+fix_mount
+fix_machine_id
+fix_product_name
+configure_proxy
+
+# we want the command (expected to be systemd) to be PID1, so exec to it
+exec "$@"

pkg/build/base/base.go

@@ -20,7 +20,6 @@ package base
 import (
 	"os"
 	"path/filepath"
-	"sigs.k8s.io/kind/pkg/util"
 
 	log "github.com/sirupsen/logrus"
 
@@ -38,9 +37,6 @@ type BuildContext struct {
 	// option fields
 	sourceDir string
 	image     string
-	// non option fields
-	goCmd string // TODO(bentheelder): should be an option possibly
-	arch  string // TODO(bentheelder): should be an option
 }
 
 // Option is BuildContext configuration option supplied to NewBuildContext
@@ -65,8 +61,6 @@ func WithImage(image string) Option {
 func NewBuildContext(options ...Option) *BuildContext {
 	ctx := &BuildContext{
 		image: DefaultImage,
-		goCmd: "go",
-		arch:  util.GetArch(),
 	}
 	for _, option := range options {
 		option(ctx)
@@ -105,36 +99,10 @@ func (c *BuildContext) Build() (err error) {
 
 	log.Infof("Building base image in: %s", buildDir)
 
-	// build the entrypoint binary first
-	if err := c.buildEntrypoint(buildDir); err != nil {
-		return err
-	}
-
 	// then the actual docker image
 	return c.buildImage(buildDir)
 }
 
-// builds the entrypoint binary
-func (c *BuildContext) buildEntrypoint(dir string) error {
-	// NOTE: this binary only uses the go1 stdlib, and is a single file
-	entrypointSrc := filepath.Join(dir, "entrypoint", "main.go")
-	entrypointDest := filepath.Join(dir, "entrypoint", "entrypoint")
-
-	cmd := exec.Command(c.goCmd, "build", "-o", entrypointDest, entrypointSrc)
-	// TODO(bentheelder): we may need to map between docker image arch and GOARCH
-	cmd.SetEnv(append(os.Environ(), "GOOS=linux", "GOARCH="+c.arch)...)
-
-	// actually build
-	log.Info("Building entrypoint binary ...")
-	exec.InheritOutput(cmd)
-	if err := cmd.Run(); err != nil {
-		log.Errorf("Entrypoint build Failed! %v", err)
-		return err
-	}
-	log.Info("Entrypoint build completed.")
-	return nil
-}
-
 func (c *BuildContext) buildImage(dir string) error {
 	// build the image, tagged as tagImageAs, using the our tempdir as the context
 	cmd := exec.Command("docker", "build", "-t", c.image, dir)