diff --git a/modules/flake/k3s/1_27/chart-versions.nix b/modules/flake/k3s/1_27/chart-versions.nix new file mode 100644 index 0000000..d3ff993 --- /dev/null +++ b/modules/flake/k3s/1_27/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-21.2.1+up21.2.0.tgz"; + sha256 = "05j3vyikb7g2z2i07rij9h4ki5lb2hb2rynpiqfd4l1y5qm0qhw9"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-21.2.1+up21.2.0.tgz"; + sha256 = "0gvz0yzph2893scd0q10b938yc7f36b3zqs57pkjgqqpl1d0nwhg"; + }; +} diff --git a/modules/flake/k3s/1_27/versions.nix b/modules/flake/k3s/1_27/versions.nix new file mode 100644 index 0000000..d08c3bb --- /dev/null +++ b/modules/flake/k3s/1_27/versions.nix @@ -0,0 +1,16 @@ +{ + k3sVersion = "1.27.6+k3s1"; + k3sCommit = "bd04941a294793ec92e8703d5e5da14107902e88"; + k3sRepoSha256 = "04chr8gp0yprihigy1yzhvi2baby053fav384gq0sjq6bkp3fzd8"; + # k3sVendorHash = "sha256-LH9OsBK0Pq/NGEHprbIgYKQsslYdR3i4LYVvo5P0K+8="; + # Modified hash for ../../patches/k3s-nix-snapshotter patch + k3sVendorHash = "sha256-ryEnRrmV7xkpPIj55N10tz0+eMPHW62IRfmBIh9IbBw="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.12.2"; + k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; + k3sCNIVersion = "1.3.0-k3s1"; + k3sCNISha256 = "0zma9g4wvdnhs9igs03xlx15bk2nq56j73zns9xgqmfiixd9c9av"; + containerdVersion = "1.7.6-k3s1.27"; + containerdSha256 = "1kzjqw56pcdpsqdkw2k5a3pnpf8n93dh4jc2yybgqz3nyj4fw0a8"; + criCtlVersion = "1.26.0-rc.0-k3s1"; +} diff --git a/modules/flake/k3s/builder.nix b/modules/flake/k3s/builder.nix new file mode 100644 index 0000000..ece72e5 --- /dev/null +++ b/modules/flake/k3s/builder.nix @@ -0,0 +1,358 @@ +lib: +{ + # git tag + k3sVersion, + # commit hash + k3sCommit, + k3sRepoSha256 ? lib.fakeHash, + k3sVendorHash ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 + k3sRootVersion, + k3sRootSha256 ? lib.fakeHash, + # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/scripts/download#L29-L32 + # see also https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/manifests/traefik.yaml#L8 + chartVersions, + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 + k3sCNIVersion, + k3sCNISha256 ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_CONTAINERD + containerdVersion, + containerdSha256 ? lib.fakeHash, + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + criCtlVersion, + updateScript ? null, +}: + +# builder.nix contains a "builder" expression that, given k3s version and hash +# variables, creates a package for that version. +# Due to variance in k3s's build process, this builder only works for k3s 1.26+ +# currently. +# It is likely we will have to split out additional builders for additional +# versions in the future, or customize this one further. +{ lib +, makeWrapper +, socat +, iptables +, iproute2 +, ipset +, bridge-utils +, btrfs-progs +, conntrack-tools +, buildGoModule +, runc +, rsync +, kmod +, libseccomp +, pkg-config +, ethtool +, util-linux +, fetchFromGitHub +, fetchurl +, fetchzip +, fetchgit +, zstd +, yq-go +, sqlite +, nixosTests +, pkgsBuildBuild +}: + +# k3s is a kinda weird derivation. One of the main points of k3s is the +# simplicity of it being one binary that can perform several tasks. +# However, when you have a good package manager (like nix), that doesn't +# actually make much of a difference; you don't really care if it's one binary +# or 10 since with a good package manager, installing and running it is +# identical. +# Since upstream k3s packages itself as one large binary with several +# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s +# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with +# some exceptions. +# K3s also carries patches to some packages (such as containerd and cni +# plugins), so we intentionally use the k3s versions of those binaries for k3s, +# even if the upstream version of those binaries exist in nixpkgs already. In +# the end, that means we have a thick k3s binary that behaves like the upstream +# one for the most part. +# However, k3s also bundles several pieces of unpatched software, from the +# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. +# Those pieces of software we entirely ignore upstream's handling of, and just +# make sure they're in the path if desired. +let + + baseMeta = with lib; { + description = "A lightweight Kubernetes distribution"; + license = licenses.asl20; + homepage = "https://k3s.io"; + maintainers = with maintainers; [ euank mic92 yajo ]; + platforms = platforms.linux; + + # resolves collisions with other installations of kubectl, crictl, ctr + # prefer non-k3s versions + priority = 5; + }; + + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + + # bundled into the k3s binary + traefikChart = fetchurl chartVersions.traefik; + traefik-crdChart = fetchurl chartVersions.traefik-crd; + + # so, k3s is a complicated thing to package + # This derivation attempts to avoid including any random binaries from the + # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which + # we don't care about doing, we can add those as build or runtime + # dependencies using a real package manager). + # In addition to those binaries, it's also configuration though (right now + # mostly strongswan configuration), and k3s does use those files. + # As such, we download it in order to grab 'etc' and bundle it into the final + # k3s binary. + k3sRoot = fetchzip { + # Note: marked as apache 2.0 license + url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; + sha256 = k3sRootSha256; + stripRoot = false; + }; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; + version = k3sCNIVersion; + vendorHash = null; + + subPackages = [ "." ]; + + src = fetchFromGitHub { + owner = "rancher"; + repo = "plugins"; + rev = "v${version}"; + sha256 = k3sCNISha256; + }; + + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + + meta = baseMeta // { + description = "CNI plugins, as patched by rancher for k3s"; + }; + }; + # Grab this separately from a build because it's used by both stages of the + # k3s build. + k3sRepo = fetchgit { + url = "https://github.com/k3s-io/k3s"; + rev = "v${k3sVersion}"; + sha256 = k3sRepoSha256; + }; + # Stage 1 of the k3s build: + # Let's talk about how k3s is structured. + # One of the ideas of k3s is that there's the single "k3s" binary which can + # do everything you need, from running a k3s server, to being a worker node, + # to running kubectl. + # The way that actually works is that k3s is a single go binary that contains + # a bunch of bindata that it unpacks at runtime into directories (either the + # user's home directory or /var/lib/rancher if run as root). + # This bindata includes both binaries and configuration. + # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. + # First, we build all the binaries that get packed into the thick k3s binary + # (and output them from one derivation so they'll all be suitably patched up). + # Then, we bundle those binaries into our thick k3s binary and use that as + # the final single output. + # This approach was chosen because it ensures the bundled binaries all are + # correctly built to run with nix (we can lean on the existing buildGoModule + # stuff), and we can again lean on that tooling for the final k3s binary too. + # Other alternatives would be to manually run the + # strip/patchelf/remove-references step ourselves in the installPhase of the + # derivation when we've built all the binaries, but haven't bundled them in + # with generated bindata yet. + + k3sServer = buildGoModule { + pname = "k3s-server"; + version = k3sVersion; + + src = k3sRepo; + vendorHash = k3sVendorHash; + + patches = [ ../patches/k3s-nix-snapshotter.patch ]; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libseccomp sqlite.dev ]; + + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + tags = [ "ctrd" "libsqlite3" "linux" ]; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 + ln -s k3s ./bin/containerd + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/k3s-completion + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-token + ln -s k3s ./bin/kubectl + popd + ''; + + meta = baseMeta // { + description = "The various binaries that get packaged into the final k3s binary"; + }; + }; + # Only used for the shim since + # https://github.com/k3s-io/k3s/blob/v1.27.2%2Bk3s1/scripts/build#L153 + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; + version = containerdVersion; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = "v${containerdVersion}"; + sha256 = containerdSha256; + }; + vendorHash = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; + }; +in +buildGoModule rec { + pname = "k3s"; + version = k3sVersion; + + tags = [ "libsqlite3" "linux" "ctrd" ]; + src = k3sRepo; + vendorHash = k3sVendorHash; + + patches = [ ../patches/k3s-nix-snapshotter.patch ]; + + postPatch = '' + # Nix prefers dynamically linked binaries over static binary. + + substituteInPlace scripts/package-cli \ + --replace '"$LDFLAGS $STATIC" -o' \ + '"$LDFLAGS" -o' \ + --replace "STATIC=\"-extldflags \'-static\'\"" \ + "" + + # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': + + substituteInPlace scripts/package-cli \ + --replace '"''${GO}" generate' \ + 'GOFLAGS="" \ + GOOS="${pkgsBuildBuild.go.GOOS}" \ + GOARCH="${pkgsBuildBuild.go.GOARCH}" \ + CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ + "''${GO}" generate' + ''; + + # Important utilities used by the kubelet, see + # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 + # Note the list in that issue is stale and some aren't relevant for k3s. + k3sRuntimeDeps = [ + kmod + socat + iptables + iproute2 + ipset + bridge-utils + ethtool + util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 + conntrack-tools + runc + ]; + + buildInputs = k3sRuntimeDeps; + + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; + + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorHash` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/containerd-shim-runc-v2 ./bin + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + + cp ${traefikChart} ./build/static/charts + cp ${traefik-crdChart} ./build/static/charts + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; + + installPhase = '' + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ + --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ + --prefix PATH : "$out/bin" + ln -s $out/bin/k3s $out/bin/kubectl + ln -s $out/bin/k3s $out/bin/crictl + ln -s $out/bin/k3s $out/bin/ctr + ''; + + doInstallCheck = true; + installCheckPhase = '' + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null + ''; + + passthru.updateScript = updateScript; + + passthru.mkTests = version: + let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); + in { + single-node = nixosTests.k3s.single-node.${k3s_version}; + multi-node = nixosTests.k3s.multi-node.${k3s_version}; + }; + passthru.tests = passthru.mkTests k3sVersion; + + + meta = baseMeta; +} diff --git a/modules/flake/k3s/default.nix b/modules/flake/k3s/default.nix new file mode 100644 index 0000000..c372376 --- /dev/null +++ b/modules/flake/k3s/default.nix @@ -0,0 +1,18 @@ +{ lib, callPackage, ... }@args: + +let + k3s_builder = import ./builder.nix lib; + common = opts: callPackage (k3s_builder opts); + # extraArgs is the extra arguments passed in by the caller to propogate downward. + # This is to allow all-packages.nix to do: + # + # let k3s_1_23 = (callPackage ./path/to/k3s { + # commonK3sArg = .... + # }).k3s_1_23; + extraArgs = builtins.removeAttrs args [ "callPackage" ]; +in +{ + k3s_1_27 = common ((import ./1_27/versions.nix) // { + updateScript = [ ./update-script.sh "27" ]; + }) extraArgs; +} diff --git a/modules/flake/overlays.nix b/modules/flake/overlays.nix index ace44c8..1cb0b82 100644 --- a/modules/flake/overlays.nix +++ b/modules/flake/overlays.nix @@ -1,19 +1,30 @@ { self, inputs, ... }: { # Provide overlay to add `nix-snapshotter`. - flake.overlays.default = self: super: { - nix-snapshotter = self.callPackage ../../package.nix {}; + flake.overlays.default = self: super: + let + nix-snapshotter = self.callPackage ../../package.nix {}; - # Depends on PR merged into main, but not yet in a release tag. - # See: https://github.com/containerd/containerd/pull/9028 - containerd = super.containerd.overrideAttrs(o: { - src = self.fetchFromGitHub { - inherit (o.src) owner repo; - rev = "779875a057ff98e9b754371c193fe3b0c23ae7a2"; - hash = "sha256-sXMDMX0QPbnFvRYrAP+sVFjTI9IqzOmLnmqAo8lE9pg="; - }; - }); - }; + # Depends on PR merged into main, but not yet in a release tag. + # See: https://github.com/containerd/containerd/pull/9028 + containerd = super.containerd.overrideAttrs(o: { + src = self.fetchFromGitHub { + inherit (o.src) owner repo; + rev = "779875a057ff98e9b754371c193fe3b0c23ae7a2"; + hash = "sha256-sXMDMX0QPbnFvRYrAP+sVFjTI9IqzOmLnmqAo8lE9pg="; + }; + }); + + in { + inherit + containerd + nix-snapshotter + ; + + k3s = (self.callPackage ./k3s { + buildGoModule = self.buildGo120Module; + }).k3s_1_27; + }; perSystem = { system, ... }: { _module.args.pkgs = import inputs.nixpkgs { diff --git a/modules/flake/patches/k3s-nix-snapshotter.patch b/modules/flake/patches/k3s-nix-snapshotter.patch new file mode 100644 index 0000000..00b91bc --- /dev/null +++ b/modules/flake/patches/k3s-nix-snapshotter.patch @@ -0,0 +1,350 @@ +diff --git a/go.mod b/go.mod +index 015993bb90..c94e9e5814 100644 +--- a/go.mod ++++ b/go.mod +@@ -6,7 +6,7 @@ replace ( + github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.11.0 + github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.4-k3s1 // k3s/release-1.27 + github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.0.1-0.20230508174102-b42e5faded1c +- github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.6-k3s1.27 ++ github.com/containerd/containerd => github.com/pdtpartners/containerd v1.0.1-stargz.0.20240214083438-8c533e066c1c + github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e + github.com/docker/distribution => github.com/docker/distribution v2.8.2+incompatible + github.com/docker/docker => github.com/docker/docker v24.0.0-rc.2.0.20230801142700-69c9adb7d386+incompatible +@@ -97,7 +97,7 @@ require ( + github.com/containerd/zfs v1.1.0 + github.com/coreos/go-iptables v0.6.0 + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf +- github.com/docker/docker v23.0.3+incompatible ++ github.com/docker/docker v23.0.5+incompatible + github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 + github.com/flannel-io/flannel v0.22.2 + github.com/go-bindata/go-bindata v3.1.2+incompatible +@@ -118,9 +118,10 @@ require ( + github.com/natefinch/lumberjack v2.0.0+incompatible + github.com/onsi/ginkgo/v2 v2.9.4 + github.com/onsi/gomega v1.27.6 +- github.com/opencontainers/runc v1.1.6 ++ github.com/opencontainers/runc v1.1.9 + github.com/opencontainers/selinux v1.11.0 + github.com/otiai10/copy v1.7.0 ++ github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214070632-154924c8d48c + github.com/pkg/errors v0.9.1 + github.com/rancher/dynamiclistener v0.3.6-rc2 + github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc +@@ -128,7 +129,7 @@ require ( + github.com/rancher/wharfie v0.5.3 + github.com/rancher/wrangler v1.1.1 + github.com/robfig/cron/v3 v3.0.1 +- github.com/rootless-containers/rootlesskit v1.0.1 ++ github.com/rootless-containers/rootlesskit v1.1.1 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.4 +@@ -167,6 +168,7 @@ require ( + require ( + cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect ++ dario.cat/mergo v1.0.0 // indirect + github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect + github.com/Azure/azure-sdk-for-go v56.3.0+incompatible // indirect +@@ -201,7 +203,7 @@ require ( + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect + github.com/cilium/ebpf v0.9.1 // indirect +- github.com/container-orchestrated-devices/container-device-interface v0.5.4 // indirect ++ github.com/container-orchestrated-devices/container-device-interface v0.6.0 // indirect + github.com/container-storage-interface/spec v1.7.0 // indirect + github.com/containerd/btrfs/v2 v2.0.0 // indirect + github.com/containerd/cgroups/v3 v3.0.2 // indirect +@@ -331,10 +333,11 @@ require ( + github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect +- github.com/opencontainers/image-spec v1.1.0-rc3 // indirect ++ github.com/opencontainers/image-spec v1.1.0-rc4 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect ++ github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect +@@ -355,7 +358,7 @@ require ( + github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect + github.com/tchap/go-patricia/v2 v2.3.1 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect +- github.com/urfave/cli/v2 v2.23.5 // indirect ++ github.com/urfave/cli/v2 v2.25.7 // indirect + github.com/vbatts/tar-split v0.11.5 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/vmware/govmomi v0.30.0 // indirect +@@ -382,18 +385,18 @@ require ( + go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.9.0 // indirect +- golang.org/x/mod v0.10.0 // indirect ++ golang.org/x/mod v0.12.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + golang.org/x/time v0.3.0 // indirect +- golang.org/x/tools v0.8.0 // indirect ++ golang.org/x/tools v0.11.0 // indirect + golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect + golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect + google.golang.org/api v0.108.0 // indirect + google.golang.org/appengine v1.6.7 // indirect +- google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect +- google.golang.org/protobuf v1.30.0 // indirect ++ google.golang.org/genproto v0.0.0-20230720185612-659f7aaaa771 // indirect ++ google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/gcfg.v1 v1.2.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +diff --git a/go.sum b/go.sum +index 7ac62863e5..b8758ea05a 100644 +--- a/go.sum ++++ b/go.sum +@@ -57,6 +57,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 + cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= + cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= + cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= ++dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= ++dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= + dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= + github.com/AdaLogics/go-fuzz-headers v0.0.0-20221206110420-d395f97c4830/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= + github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= +@@ -200,8 +202,9 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH + github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= + github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= + github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +-github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= + github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= ++github.com/container-orchestrated-devices/container-device-interface v0.6.0 h1:aWwcz/Ep0Fd7ZuBjQGjU/jdPloM7ydhMW13h85jZNvk= ++github.com/container-orchestrated-devices/container-device-interface v0.6.0/go.mod h1:OQlgtJtDrOxSQ1BWODC8OZK1tzi9W69wek+Jy17ndzo= + github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= + github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= + github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY= +@@ -629,8 +632,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X + github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= + github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= + github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +-github.com/k3s-io/containerd v1.7.6-k3s1.27 h1:BGoNTvkH/rxURf5UCvE4hIDY2emZ3o1DPNlTAmCoJpI= +-github.com/k3s-io/containerd v1.7.6-k3s1.27/go.mod h1:dWUW/BzVXrFhxzfRZ1Jmr/yLlRvjryZlb1ns2SCHsgs= + github.com/k3s-io/cri-dockerd v0.3.4-k3s1 h1:eCeVCeXzf10fyanv1gniSwidBjdO83/akv+M72uEnZc= + github.com/k3s-io/cri-dockerd v0.3.4-k3s1/go.mod h1:0KDOU8lLjp+ETJFFCcVBRQbJ8puRoDxaHBDj8C87Fk4= + github.com/k3s-io/cri-tools v1.26.0-rc.0-k3s1 h1:yWVy9pS0T1BWBMZBPRy2Q29gaLmaGknQHSnx+HStrVM= +@@ -934,8 +935,8 @@ github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zM + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= + github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= + github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +-github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +-github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= ++github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= ++github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= + github.com/opencontainers/runc v1.1.8 h1:zICRlc+C1XzivLc3nzE+cbJV4LIi8tib6YG0MqC6OqA= + github.com/opencontainers/runc v1.1.8/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= + github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 h1:R5M2qXZiK/mWPMT4VldCOiSL9HIAMuxQZWdG0CSM5+4= +@@ -954,10 +955,16 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 + github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= + github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= + github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= ++github.com/pdtpartners/containerd v1.0.1-stargz.0.20240214083438-8c533e066c1c h1:OWE2iwUICpISBOLVu9OpelITX4sIZwOabjbF4Vcjm8I= ++github.com/pdtpartners/containerd v1.0.1-stargz.0.20240214083438-8c533e066c1c/go.mod h1:dWUW/BzVXrFhxzfRZ1Jmr/yLlRvjryZlb1ns2SCHsgs= ++github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214070632-154924c8d48c h1:FdjMmoAD5rdlg2kNIGFLbfwkzV8eO32lDdB6MNAO2BM= ++github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214070632-154924c8d48c/go.mod h1:iMHTz4uBpoXqj+xq+uCGwBwpe2Or4CqF3R40daLzlqc= + github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= + github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= + github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= + github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= ++github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= ++github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= + github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= + github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= + github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +@@ -1037,8 +1044,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE + github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= + github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= + github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +-github.com/rootless-containers/rootlesskit v1.0.1 h1:jepqW1txFSowKSMAEkVhWH3Oa1TCY9S400MVYe/6Iro= +-github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc= ++github.com/rootless-containers/rootlesskit v1.1.1 h1:F5psKWoWY9/VjZ3ifVcaosjvFZJOagX85U22M0/EQZE= ++github.com/rootless-containers/rootlesskit v1.1.1/go.mod h1:UD5GoA3dqKCJrnvnhVgQQnweMF2qZnf9KLw8EewcMZI= + github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= + github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= + github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= +@@ -1139,8 +1146,8 @@ github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX + github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= + github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= + github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +-github.com/urfave/cli/v2 v2.23.5 h1:xbrU7tAYviSpqeR3X4nEFWUdB/uDZ6DE+HxmRU7Xtyw= +-github.com/urfave/cli/v2 v2.23.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= ++github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= ++github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= + github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= + github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= + github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= +@@ -1285,8 +1292,9 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= + golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= + golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= + golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= + golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= ++golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= ++golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= + golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= + golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +@@ -1409,8 +1417,9 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= + golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= + golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= + golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= + golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= ++golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= ++golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= + golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +@@ -1447,8 +1456,9 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw + google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= + google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= + google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= + google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= ++google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= ++google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= + gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= + gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +@@ -1492,8 +1502,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= + gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= + gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= + gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= + gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= ++gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= + honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= + honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= + honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go +index ee02becf87..0539c049f3 100644 +--- a/pkg/agent/config/config.go ++++ b/pkg/agent/config/config.go +@@ -546,6 +546,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N + nodeConfig.Containerd.Root) + } + nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" ++ case "nix": ++ if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil { ++ return nil, errors.Wrapf(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", ++ nodeConfig.Containerd.Root) ++ } ++ nodeConfig.AgentConfig.ImageServiceSocket = "/run/k3s/nix-snapshotter/nix-snapshotter.sock" + } + } + nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd") +diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go +index 559d966dec..02c19281e3 100644 +--- a/pkg/agent/containerd/config_linux.go ++++ b/pkg/agent/containerd/config_linux.go +@@ -18,6 +18,7 @@ import ( + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/version" + "github.com/opencontainers/runc/libcontainer/userns" ++ "github.com/pdtpartners/nix-snapshotter/pkg/nix" + "github.com/pkg/errors" + "github.com/rancher/wharfie/pkg/registries" + "github.com/sirupsen/logrus" +@@ -120,3 +121,7 @@ func FuseoverlayfsSupported(root string) error { + func StargzSupported(root string) error { + return stargz.Supported(root) + } ++ ++func NixSupported(root string) error { ++ return nix.Supported(root) ++} +diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go +index 6efbb7a148..2f48418f9a 100644 +--- a/pkg/agent/containerd/config_windows.go ++++ b/pkg/agent/containerd/config_windows.go +@@ -85,3 +85,7 @@ func FuseoverlayfsSupported(root string) error { + func StargzSupported(root string) error { + return errors.Wrapf(util3.ErrUnsupportedPlatform, "stargz is not supported") + } ++ ++func NixSupported(root string) error { ++ return errors.Wrapf(util2.ErrUnsupportedPlatform, "nix is not supported") ++} +diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go +index 4d0c52fc42..83cdb4909c 100644 +--- a/pkg/agent/templates/templates_linux.go ++++ b/pkg/agent/templates/templates_linux.go +@@ -35,7 +35,7 @@ version = 2 + {{- if .NodeConfig.AgentConfig.Snapshotter }} + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" +- disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} ++ disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}} + {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} + {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} + [plugins."io.containerd.snapshotter.v1.stargz"] +@@ -74,6 +74,14 @@ enable_keychain = true + {{end}} + {{end}} + {{end}} ++{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }} ++[[plugins."io.containerd.transfer.v1.local".unpack_config]] ++platform = "linux/amd64" ++snapshotter = "nix" ++[plugins."io.containerd.snapshotter.v1.nix"] ++address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" ++image_service.enable = true ++{{end}} + {{end}} + + {{- if not .NodeConfig.NoFlannel }} +diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go +index a0ea4dc496..98c443625d 100644 +--- a/pkg/containerd/builtins_linux.go ++++ b/pkg/containerd/builtins_linux.go +@@ -32,4 +32,5 @@ import ( + _ "github.com/containerd/fuse-overlayfs-snapshotter/plugin" + _ "github.com/containerd/stargz-snapshotter/service/plugin" + _ "github.com/containerd/zfs/plugin" ++ _ "github.com/pdtpartners/nix-snapshotter/pkg/plugin" + ) +diff --git a/pkg/containerd/utility_linux.go b/pkg/containerd/utility_linux.go +index 76ff569b41..b6f2d0cfeb 100644 +--- a/pkg/containerd/utility_linux.go ++++ b/pkg/containerd/utility_linux.go +@@ -6,6 +6,7 @@ import ( + "github.com/containerd/containerd/snapshots/overlay/overlayutils" + fuseoverlayfs "github.com/containerd/fuse-overlayfs-snapshotter" + stargz "github.com/containerd/stargz-snapshotter/service" ++ "github.com/pdtpartners/nix-snapshotter/pkg/nix" + ) + + func OverlaySupported(root string) error { +@@ -19,3 +20,7 @@ func FuseoverlayfsSupported(root string) error { + func StargzSupported(root string) error { + return stargz.Supported(root) + } ++ ++func NixSupported(root string) error { ++ return nix.Supported(root) ++} +diff --git a/pkg/containerd/utility_windows.go b/pkg/containerd/utility_windows.go +index 13a58e55bf..6f6833177f 100644 +--- a/pkg/containerd/utility_windows.go ++++ b/pkg/containerd/utility_windows.go +@@ -19,3 +19,7 @@ func FuseoverlayfsSupported(root string) error { + func StargzSupported(root string) error { + return errors.Wrapf(util2.ErrUnsupportedPlatform, "stargz is not supported") + } ++ ++func NixSupported(root string) error { ++ return errors.Wrapf(util2.ErrUnsupportedPlatform, "nix is not supported") ++} diff --git a/modules/nixos/k3s.nix b/modules/nixos/k3s.nix index 0fec8f3..7bbe2b0 100644 --- a/modules/nixos/k3s.nix +++ b/modules/nixos/k3s.nix @@ -1,28 +1,15 @@ { pkgs, ... }: -let - k3s-cni-plugins = pkgs.buildEnv { - name = "k3s-cni-plugins"; - paths = [ - pkgs.cni-plugins - pkgs.cni-plugin-flannel - ]; - }; - -in { +{ services.k3s = { enable = true; - configPath = pkgs.writeText "k3s_config.yaml" '' - container-runtime-endpoint: "unix:///run/containerd/containerd.sock" - image-service-endpoint: "unix:///run/nix-snapshotter/nix-snapshotter.sock" - ''; + extraFlags = toString [ + "--snapshotter nix" + ]; }; - virtualisation.containerd = { - settings.plugins."io.containerd.grpc.v1.cri".cni = { - bin_dir = "${k3s-cni-plugins}/bin"; - conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; - }; - }; + systemd.services.k3s.path = with pkgs; [ + nix + ]; environment.sessionVariables = { KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; diff --git a/modules/nixos/kubernetes-startup.nix b/modules/nixos/kubernetes-startup.nix deleted file mode 100644 index 2621a1d..0000000 --- a/modules/nixos/kubernetes-startup.nix +++ /dev/null @@ -1,51 +0,0 @@ -{ lib, config, pkgs, ... }: -let - inherit (config.systemd.services) - kube-addon-manager - ; - - inherit (config.services) - certmgr - ; - - cfg = config.services.kubernetes; - - waitFile = filename: toString (pkgs.writeShellScript "wait-file-${filename}" '' - while [ ! -f /var/lib/kubernetes/secrets/${filename} ]; do sleep 1; done - ''); - - waitPort = port: toString (pkgs.writeShellScript "wait-port-${port}" '' - while ! ${pkgs.netcat}/bin/nc -z localhost ${port}; do sleep 1; done - ''); - -in { - # Fix various startup issues related to kubernetes systemd services to avoid - # failures during NixOS VM boot. - config = lib.mkMerge [ - (lib.mkIf (cfg.roles != []) { - systemd.services.etcd.preStart = waitFile "etcd.pem"; - systemd.services.certmgr.preStart = lib.mkForce '' - mkdir -p ${cfg.secretsPath} - ${waitFile "ca.pem"} - ''; - systemd.services.kube-apiserver = { - preStart = waitFile "service-account-key.pem"; - # Wait for its own securePort to be ready since it doesn't support - # systemd notify. - postStart = waitPort (toString cfg.apiserver.securePort); - }; - }) - (lib.mkIf (cfg.addonManager.bootstrapAddons != {}) { - systemd.services.kube-addon-manager.preStart = - let - files = lib.mapAttrsToList (n: v: pkgs.writeText "${n}.json" (builtins.toJSON v)) - cfg.addonManager.bootstrapAddons; - - in lib.mkForce '' - ${waitFile "cluster-admin.pem"} - export KUBECONFIG="/etc/${cfg.pki.etcClusterAdminKubeconfig}" - ${cfg.package}/bin/kubectl apply -f ${lib.concatStringsSep " \\\n -f " files} - ''; - }) - ]; -} diff --git a/modules/nixos/kubernetes.nix b/modules/nixos/kubernetes.nix deleted file mode 100644 index a04e5af..0000000 --- a/modules/nixos/kubernetes.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib, config, pkgs, ... }: -{ - # Smooths out upstream service startup issues. - imports = [ ./kubernetes-startup.nix ]; - - # Provision single node kubernetes listening on localhost. - services.kubernetes = { - roles = ["master" "node"]; - masterAddress = "localhost"; - kubelet.extraOpts = "--image-service-endpoint unix:///run/nix-snapshotter/nix-snapshotter.sock"; - }; - - # Do not take over cni/net.d as nerdctl wants it writeable as well. - environment.etc = lib.mkMerge [ - { "cni/net.d".enable = false; } - ( - lib.listToAttrs - (lib.imap - (i: entry: - let name = "cni/net.d/${toString (10+i)}-${entry.type}.conf"; - in { - inherit name; - value = { source = pkgs.writeText name (builtins.toJSON entry); }; - }) - config.services.kubernetes.kubelet.cni.config - ) - ) - ]; - - # Allow non-root "admin" user to just use `kubectl`. - services.certmgr.specs.clusterAdmin.private_key.owner = "rootless"; - environment.sessionVariables = { - KUBECONFIG = "/etc/${config.services.kubernetes.pki.etcClusterAdminKubeconfig}"; - }; -} diff --git a/modules/nixos/tests/kubernetes.nix b/modules/nixos/tests/kubernetes.nix index 72e58dd..6c4e739 100644 --- a/modules/nixos/tests/kubernetes.nix +++ b/modules/nixos/tests/kubernetes.nix @@ -1,102 +1,64 @@ { pkgs, ... }: -let - registryHost = "127.0.0.1"; - - registryPort = 5000; - - redisImageName = "${registryHost}:${toString registryPort}/redis"; - - redisNodePort = 30000; - - redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Pod"; - metadata = { - name = "redis"; - labels.name = "redis"; - }; - spec.containers = [{ - name = "redis"; - image = redisImageName; - args = ["--protected-mode" "no"]; - ports = [{ - name = "client"; - containerPort = 6379; - }]; - }]; - }); - - redisService = pkgs.writeText "redis-service.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Service"; - metadata.name = "redis-service"; - spec = { - type = "NodePort"; - selector.name = "redis"; - ports = [{ - name = "client"; - port = 6379; - nodePort = redisNodePort; - }]; - }; - }); - -in { - nodes.machine = { config, ... }: - let - cfg = config.services.kubernetes; - - wrapKubectl = pkgs.runCommand "wrap-kubectl" { - nativeBuildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out/bin - makeWrapper ${pkgs.kubernetes}/bin/kubectl \ - $out/bin/kubectl \ - --set KUBECONFIG "/etc/${cfg.pki.etcClusterAdminKubeconfig}" - ''; - - redisImage = pkgs.nix-snapshotter.buildImage { - name = redisImageName; - tag = "latest"; - config.entrypoint = ["${pkgs.redis}/bin/redis-server"]; - }; - - in { - imports = [ ../nix-snapshotter.nix ]; - - services.nix-snapshotter.enable = true; - - services.kubernetes = { - roles = ["master" "node"]; - masterAddress = "localhost"; +{ + nodes = { + kubernetes = { config, ... }: + let + cfg = config.services.kubernetes; + + wrapKubectl = pkgs.runCommand "wrap-kubectl" { + nativeBuildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + makeWrapper ${pkgs.kubernetes}/bin/kubectl \ + $out/bin/kubectl \ + --set KUBECONFIG "/etc/${cfg.pki.etcClusterAdminKubeconfig}" + ''; + + in { + imports = [ + ../nix-snapshotter.nix + ../redis-spec.nix + ]; + + services.nix-snapshotter.enable = true; + + services.kubernetes = { + roles = ["master" "node"]; + masterAddress = "localhost"; + kubelet.extraOpts = "--image-service-endpoint unix:///run/nix-snapshotter/nix-snapshotter.sock"; + }; + + environment.systemPackages = [ + pkgs.redis + wrapKubectl + ]; }; - services.dockerRegistry = { - enable = true; - listenAddress = registryHost; - port = registryPort; - }; + k3s = { ... }: { + imports = [ + ../k3s.nix + ../redis-spec.nix + ]; environment.systemPackages = [ - (redisImage.copyToRegistry { plainHTTP = true; }) pkgs.redis - wrapKubectl ]; }; + }; testScript = '' start_all() - machine.wait_for_unit("docker-registry.service") - machine.wait_for_open_port(${toString registryPort}) - machine.succeed("copy-to-registry") + def test_redis_service(machine): + machine.wait_until_succeeds("kubectl get node $(hostname) | grep -w Ready") - machine.wait_until_succeeds("kubectl get node machine | grep -w Ready") + machine.wait_until_succeeds("kubectl apply -f /etc/kubernetes/redis/") - machine.wait_until_succeeds("kubectl create -f ${redisPod}") - machine.wait_until_succeeds("kubectl create -f ${redisService}") + machine.wait_until_succeeds("kubectl get pod redis | grep Running") + out = machine.wait_until_succeeds("redis-cli -p 30000 ping") + assert "PONG" in out - machine.wait_until_succeeds("kubectl get pod redis | grep Running") - machine.wait_until_succeeds("redis-cli -p ${toString redisNodePort} ping") + # test_redis_service(kubernetes) + test_redis_service(k3s) ''; } diff --git a/modules/nixos/vm.nix b/modules/nixos/vm.nix index 46a179f..d7fc9b0 100644 --- a/modules/nixos/vm.nix +++ b/modules/nixos/vm.nix @@ -1,4 +1,4 @@ -{ lib, config, pkgs, modulesPath, examples, ... }: +{ lib, pkgs, modulesPath, examples, ... }: let preloadContainerdImages = lib.attrValues examples; @@ -8,8 +8,7 @@ in { # is only intended to be used as a VM. Using vmVariant will emit assertion # errors regarding `fileSystems."/"` and `boot.loader.grub.device`. (modulesPath + "/virtualisation/qemu-vm.nix") - ./kubernetes.nix - # ./k3s.nix + ./k3s.nix ./redis-spec.nix ]; diff --git a/package.nix b/package.nix index e2ca147..c15556b 100644 --- a/package.nix +++ b/package.nix @@ -18,7 +18,7 @@ let ".tar" ]; }; - vendorHash = "sha256-tTAp3bwbtJP+3Jr9N2ULEuaC+P8tXUeiiHXCtL54BGc="; + vendorHash = "sha256-QBLePOnfsr6I19ddyZNSFDih6mCaZ/NV2Qz1B1pSHxs="; passthru = { inherit buildImage; }; };