diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3b6d15..8d1b7b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,9 @@ jobs: test: - snapshotter - kubernetes + - k3s + - k3s-external + - k3s-rootless runs-on: nix-snapshotter-runner needs: [lint, build] if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') diff --git a/README.md b/README.md index 4bb0107..a49bac0 100644 --- a/README.md +++ b/README.md @@ -71,15 +71,25 @@ redis-cli -p 30000 ping Or you can try running in rootless mode: ```sh -nix run ".#vm" +nix run "github:pdtpartners/nix-snapshotter#vm-rootless" nixos login: rootless # (Ctrl-a then x to quit) Password: rootless -# Running pkgs.hello image with nix-snapshotter -nerdctl run ghcr.io/pdtpartners/hello +# `nerdctl run` with rootless k3s containerd currently not supported yet +# See: https://github.com/containerd/nerdctl/issues/2831 +# +# If rootless kubernetes not needed, `nerdctl run` does work with rootless +# containerd + nix-snapshotter. + +# Running `pkgs.redis` image with kubernetes & nix-snapshotter +kubectl apply -f /etc/kubernetes/redis/ + +# Wait a few seconds... +watch kubectl get pods -# Rootless kubernetes not supported yet. -# See: https://github.com/k3s-io/k3s/pull/8279 +# And a kubernetes service will be ready to forward port 30000 to the redis +# pod, so you can test it out with a `ping` command +redis-cli -p 30000 ping ``` ## Installation @@ -128,15 +138,18 @@ easy installation. } ({ pkgs, ... }: { # (1) Import home-manager module. - imports = [ nix-snapshotter.homeModules.nix-snapshotter-rootless ]; + imports = [ nix-snapshotter.homeModules.default ]; # (2) Add overlay. nixpkgs.overlays = [ nix-snapshotter.overlays.default ]; # (3) Enable service. + virtualisation.containerd.rootless = { + enable = true; + nixSnapshotterIntegration = true; + }; services.nix-snapshotter.rootless = { enable = true; - setContainerdSnapshotter = true; }; # (4) Add a containerd CLI like nerdctl. @@ -161,22 +174,24 @@ easy installation. in { imports = [ - ./hardware-configuration.nix # (1) Import home-manager module. - nix-snapshotter.nixosModules.default + nix-snapshotter.homeModules.default ]; - # (2) Add overlay. + // # (2) Add overlay. nixpkgs.overlays = [ nix-snapshotter.overlays.default ]; # (3) Enable service. - services.nix-snapshotter = { + virtualisation.containerd.rootless = { + enable = true; + nixSnapshotterIntegration = true; + }; + services.nix-snapshotter.rootless = { enable = true; - setContainerdSnapshotter = true; }; # (4) Add a containerd CLI like nerdctl. - environment.systemPackages = [ pkgs.nerdctl ]; + home.packages = [ pkgs.nerdctl ]; } ``` @@ -209,9 +224,12 @@ easy installation. nixpkgs.overlays = [ nix-snapshotter.overlays.default ]; # (3) Enable service. + virtualisation.containerd = { + enable = true; + nixSnapshotterIntegration = true; + }; services.nix-snapshotter = { enable = true; - setContainerdSnapshotter = true; }; # (4) Add a containerd CLI like nerdctl. @@ -245,9 +263,12 @@ easy installation. nixpkgs.overlays = [ nix-snapshotter.overlays.default ]; # (3) Enable service. + virtualisation.containerd = { + enable = true; + nixSnapshotterIntegration = true; + }; services.nix-snapshotter = { enable = true; - setContainerdSnapshotter = true; }; # (4) Add a containerd CLI like nerdctl. diff --git a/modules/common/containerd-rootless.nix b/modules/common/containerd-rootless.nix index 808e9ef..3400120 100644 --- a/modules/common/containerd-rootless.nix +++ b/modules/common/containerd-rootless.nix @@ -8,6 +8,8 @@ let cfg = config.virtualisation.containerd.rootless; + ctrd-lib = config.virtualisation.containerd.lib; + settingsFormat = pkgs.formats.toml {}; configFile = settingsFormat.generate "containerd.toml" cfg.settings; @@ -40,8 +42,10 @@ let exec nsenter \ --no-fork \ --preserve-credentials \ - -m -n -U \ - -t "$pid" \ + --mount \ + --net \ + --user \ + --target "$pid" \ -- "$@" ''; }; @@ -54,7 +58,8 @@ let mkRootlessContainerdService = cfg: let - containerdArgs = lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args); + containerdArgs = + lib.concatStringsSep " " (lib.cli.toGNUCommandLine {} cfg.args); containerd-rootless = makeProg { name = "containerd-rootless"; @@ -148,7 +153,18 @@ let }; in { + imports = [ + ./containerd.nix + ]; + options.virtualisation.containerd.rootless = { + inherit (ctrd-lib.options) + nixSnapshotterIntegration + setAddress + setNamespace + setSnapshotter + ; + settings = lib.mkOption { type = settingsFormat.type; default = {}; @@ -175,15 +191,6 @@ in { package = mkPackageOptionMD pkgs "containerd" { }; - setSocketVariable = mkOption { - type = types.bool; - default = false; - description = lib.mdDoc '' - Point {command}`CONTAINERD_ADDRESS` to rootless containerd for normal - users by default. - ''; - }; - bindMounts = lib.mkOption { type = types.attrsOf (types.submodule bindMountOpts); example = lib.literalExpression '' @@ -208,36 +215,52 @@ in { }; lib = mkOption { + type = types.attrs; description = lib.mdDoc "Common functions for the containerd modules."; default = { - inherit mkRootlessContainerdService; + inherit + mkRootlessContainerdService + ; }; - type = types.attrs; internal = true; }; }; - config = lib.mkIf cfg.enable { - virtualisation.containerd.rootless = { - inherit nsenter; + config = lib.mkIf cfg.enable (lib.mkMerge [ + { + virtualisation.containerd.rootless = { + inherit nsenter; - args = { - config = toString containerdConfigChecked; - }; + args.config = toString containerdConfigChecked; + + setAddress = lib.mkDefault "$XDG_RUNTIME_DIR/containerd/containerd.sock"; + + settings = { + version = 2; + plugins."io.containerd.grpc.v1.cri" = { + cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin"; + }; + }; - settings = { - version = 2; - plugins."io.containerd.grpc.v1.cri" = { - cni.bin_dir = lib.mkOptionDefault "${pkgs.cni-plugins}/bin"; + bindMounts = { + "$XDG_RUNTIME_DIR/containerd".mountPoint = "/run/containerd"; + "$XDG_DATA_HOME/containerd".mountPoint = "/var/lib/containerd"; + "$XDG_DATA_HOME/cni".mountPoint = "/var/lib/cni"; + "$XDG_CONFIG_HOME/cni".mountPoint = "/etc/cni"; }; }; + } + (lib.mkIf cfg.nixSnapshotterIntegration { + virtualisation.containerd.rootless = { + setSnapshotter = lib.mkDefault "nix"; + + settings = ctrd-lib.mkNixSnapshotterSettings; - bindMounts = { - "$XDG_RUNTIME_DIR/containerd".mountPoint = "/run/containerd"; - "$XDG_DATA_HOME/containerd".mountPoint = "/var/lib/containerd"; - "$XDG_DATA_HOME/cni".mountPoint = "/var/lib/cni"; - "$XDG_CONFIG_HOME/cni".mountPoint = "/etc/cni"; + bindMounts = { + "$XDG_RUNTIME_DIR/nix-snapshotter".mountPoint = "/run/nix-snapshotter"; + "$XDG_DATA_HOME/nix-snapshotter".mountPoint = "/var/lib/containerd/io.containerd.snapshotter.v1.nix"; + }; }; - }; - }; + }) + ]); } diff --git a/modules/common/containerd.nix b/modules/common/containerd.nix new file mode 100644 index 0000000..36eb4b4 --- /dev/null +++ b/modules/common/containerd.nix @@ -0,0 +1,94 @@ +{ config, pkgs, lib, ... }: +let + inherit (lib) + mkEnableOption + mkOption + types + ; + + inherit (pkgs.go) + GOOS + GOARCH + ; + + cfg = config.virtualisation.containerd; + + options = { + k3sIntegration = mkEnableOption "K3s integration"; + + nixSnapshotterIntegration = mkEnableOption "Nix snapshotter integration"; + + setAddress = mkOption { + type = types.str; + default = "/run/containerd/containerd.sock"; + description = lib.mdDoc '' + Set the default containerd address via environment variable + `CONTAINERD_ADDRESS`. + ''; + }; + + setNamespace = mkOption { + type = types.str; + default = "default"; + description = lib.mdDoc '' + Set the default containerd namespace via environment variable + `CONTAINERD_NAMESPACE`. + ''; + }; + + setSnapshotter = mkOption { + type = types.str; + default = ""; + description = lib.mdDoc '' + Set the default containerd snapshotter via environment variable + `CONTAINERD_SNAPSHOTTER`. + ''; + }; + }; + + mkNixSnapshotterSettings = { + plugins."io.containerd.grpc.v1.cri".containerd = { + snapshotter = "nix"; + }; + + plugins."io.containerd.transfer.v1.local".unpack_config = [{ + platform = "${GOOS}/${GOARCH}"; + snapshotter = "nix"; + }]; + + proxy_plugins.nix = { + type = "snapshot"; + address = "/run/nix-snapshotter/nix-snapshotter.sock"; + }; + }; + +in { + options.virtualisation.containerd = { + inherit (options) + k3sIntegration + nixSnapshotterIntegration + setAddress + setNamespace + setSnapshotter + ; + + lib = mkOption { + type = types.attrs; + description = lib.mdDoc "Common functions for containerd."; + default = { + inherit + options + mkNixSnapshotterSettings + ; + }; + internal = true; + }; + }; + + config = lib.mkIf cfg.enable { + virtualisation.containerd = lib.mkIf cfg.nixSnapshotterIntegration { + setSnapshotter = lib.mkDefault "nix"; + settings = mkNixSnapshotterSettings; + }; + }; +} diff --git a/modules/common/k3s-rootless.nix b/modules/common/k3s-rootless.nix new file mode 100644 index 0000000..250b2f5 --- /dev/null +++ b/modules/common/k3s-rootless.nix @@ -0,0 +1,126 @@ +{ config, pkgs, lib, ... }: +let + inherit (lib) + mkEnableOption + mkOption + mkPackageOption + types + ; + + cfg = config.services.k3s.rootless; + + k3s-lib = config.services.k3s.lib; + + mkRootlessK3sService = cfg: { + Unit = { + Description = "k3s - lightweight kubernetes (Rootless)"; + StartLimitBurst = "3"; + StartLimitInterval = "120s"; + }; + + Install = { + WantedBy = [ "default.target" ]; + }; + + Service = { + Type = "simple"; + Delegate = "yes"; + Restart = "always"; + RestartSec = "2"; + Environment = "PATH=${lib.makeBinPath cfg.path}"; + EnvironmentFile = cfg.environmentFile; + ExecStart = lib.concatStringsSep " \\\n " ( + [ + "${pkgs.k3s}/bin/k3s server --rootless" + ] + ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}") + ++ cfg.extraFlags + ); + + ExecReload = "${pkgs.procps}/bin/kill -s HUP $MAINPID"; + + KillMode = "mixed"; + + LimitNOFILE = "infinity"; + LimitNPROC = "infinity"; + LimitCORE = "infinity"; + TasksMax = "infinity"; + }; + }; + +in { + imports = [ + ./k3s.nix + ]; + + options.services.k3s.rootless = { + inherit (k3s-lib.options) + setEmbeddedContainerd + setKubeConfig + snapshotter + ; + + enable = mkEnableOption (lib.mdDoc "k3s"); + + package = mkPackageOption pkgs "k3s" { }; + + extraFlags = mkOption { + type = types.listOf types.str; + description = lib.mdDoc "Extra flags to pass to the k3s command."; + default = []; + example = [ "--no-deploy traefik" "--cluster-cidr 10.24.0.0/16" ]; + }; + + path = mkOption { + type = types.listOf types.path; + description = lib.mdDoc '' + Packages to be included in the PATH for k3s. + ''; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + description = lib.mdDoc '' + File path containing environment variables for configuring the k3s + service in the format of an EnvironmentFile. See systemd.exec(5). + ''; + default = null; + }; + + configPath = mkOption { + type = types.nullOr types.path; + default = null; + description = lib.mdDoc '' + File path containing the k3s YAML config. This is useful when the config is + generated (for example on boot). + ''; + }; + + lib = mkOption { + type = types.attrs; + description = lib.mdDoc "Common functions for the k3s modules."; + default = { + inherit mkRootlessK3sService; + }; + internal = true; + }; + }; + + config = lib.mkIf cfg.enable { + services.k3s.rootless = lib.mkMerge [ + { + path = with pkgs; [ + nerdctl + slirp4netns + # Need access to newuidmap from "/run/wrappers" + "/run/wrappers" + ]; + + extraFlags = [ "--snapshotter ${cfg.snapshotter}" ]; + } + (lib.mkIf (cfg.snapshotter == "nix") { + path = [ pkgs.nix ]; + }) + ]; + }; +} diff --git a/modules/common/k3s.nix b/modules/common/k3s.nix new file mode 100644 index 0000000..52cd065 --- /dev/null +++ b/modules/common/k3s.nix @@ -0,0 +1,58 @@ +{ lib, ... }: +let + inherit (lib) + mkOption + types + ; + + options = { + setEmbeddedContainerd = mkOption { + type = types.bool; + description = lib.mdDoc '' + Configures CONTAINERD_ADDRESS, CONTAINERD_NAMESPACE, + CONTAINERD_SNAPSHOTTER to target k3s' embedded containerd. + ''; + default = false; + }; + + setKubeConfig = mkOption { + type = types.bool; + description = lib.mdDoc '' + Configures KUBECONFIG environment variable to default kubectl to point + to k3s. + ''; + default = false; + }; + + snapshotter = mkOption { + type = types.enum [ + "overlayfs" + "fuse-overlayfs" + "stargz" + "nix" + ]; + description = lib.mdDoc '' + Specifies the containerd snapshotter for k3s' embedded containerd. + ''; + default = "overlayfs"; + }; + }; + +in { + options.services.k3s = { + inherit (options) + setEmbeddedContainerd + setKubeConfig + snapshotter + ; + + lib = mkOption { + type = types.attrs; + description = lib.mdDoc "Common functions for k3s."; + default = { + inherit options; + }; + internal = true; + }; + }; +} diff --git a/modules/common/nix-snapshotter-lib.nix b/modules/common/nix-snapshotter-lib.nix deleted file mode 100644 index 0005db3..0000000 --- a/modules/common/nix-snapshotter-lib.nix +++ /dev/null @@ -1,199 +0,0 @@ -{ config, pkgs, lib, ... }: -let - inherit (lib) - mkOption - mkPackageOptionMD - types - ; - - inherit (pkgs.go) - GOOS - GOARCH - ; - - inherit (config.virtualisation.containerd.rootless) - nsenter - ; - - options = { - configFile = mkOption { - type = types.nullOr types.path; - description = lib.mdDoc '' - Path to nix-snapshotter config file. - Setting this option will override any configuration applied by the - settings option. - ''; - }; - - package = mkPackageOptionMD pkgs "nix-snapshotter" { }; - - path = mkOption { - type = types.listOf types.package; - default = [ pkgs.nix ]; - description = lib.mdDoc '' - Set the path of the nix-snapshotter service, if it requires access to - alternative nix binaries. - ''; - }; - - preloadContainerdImages = mkOption { - type = types.listOf types.package; - default = []; - description = lib.mdDoc '' - Specify image tar archives that should be preloaded into containerd. - ''; - }; - - setContainerdSnapshotter = mkOption { - type = types.bool; - default = false; - description = lib.mdDoc '' - Set the nix snapshotter to be the default containerd snapshotter - by setting the env var CONTAINERD_SNAPSHOTTER="nix". - ''; - }; - - setContainerdNamespace = mkOption { - type = types.str; - default = "default"; - description = lib.mdDoc '' - Set the default containerd namespace by setting the env var - CONTAINERD_NAMESPACE. - ''; - }; - - settings = mkOption { - type = settingsFormat.type; - default = {}; - description = lib.mdDoc '' - Verbatim lines to add to config.toml - ''; - }; - }; - - settingsFormat = pkgs.formats.toml {}; - - mkContainerdSettings = { - plugins."io.containerd.grpc.v1.cri" = { - containerd.snapshotter = "nix"; - }; - - plugins."io.containerd.transfer.v1.local".unpack_config = [{ - platform = "${GOOS}/${GOARCH}"; - snapshotter = "nix"; - }]; - - proxy_plugins.nix = { - type = "snapshot"; - address = "/run/nix-snapshotter/nix-snapshotter.sock"; - }; - }; - - # SYSTEMD USER SERVICES - # --------------------- - # - # When writing systemd user services targetting both NixOS modules and - # home-manager modules, the easiest transformation is home-manager -> NixOS, - # which is why the services are written as such. - - # Converts a home-manager systemd user service to a NixOS systemd user - # service. Since home-manager style services map closer to raw systemd - # service specification, it's easier to transform in this direction. - convertServiceToNixOS = unit: - { - serviceConfig = lib.optionalAttrs (unit?Service) unit.Service; - unitConfig = lib.optionalAttrs (unit?Unit) unit.Unit; - } // (lib.optionalAttrs (unit?Install.WantedBy) { - # Only `WantedBy` is supported by NixOS as [Install] fields are not - # supported, due to its stateful nature. - wantedBy = unit.Install.WantedBy; - }); - - mkNixSnapshotterService = { - Service = { - Type = "notify"; - Delegate = "yes"; - KillMode = "mixed"; - Restart = "always"; - RestartSec = "2"; - - StateDirectory = "nix-snapshotter"; - RuntimeDirectory = "nix-snapshotter"; - RuntimeDirectoryPreserve = "yes"; - }; - }; - - mkRootlessNixSnapshotterService = cfg: lib.recursiveUpdate - mkNixSnapshotterService - { - Unit = { - Description = "nix-snapshotter - containerd snapshotter that understands nix store paths natively (Rootless)"; - After = [ "containerd.service" ]; - PartOf = [ "containerd.service" ]; - }; - - Install = { - WantedBy = [ "default.target" ]; - }; - - Service.ExecStart = "${nsenter}/bin/containerd-nsenter ${cfg.package}/bin/nix-snapshotter --config ${cfg.configFile}"; - }; - - mkPreloadContainerdImageService = cfg: - let - namespace = cfg.setContainerdNamespace; - - archives = cfg.preloadContainerdImages; - - preload = pkgs.writeShellScriptBin "preload" ( - lib.concatStringsSep "\n" - (builtins.map - (archive: ''${pkgs.nix-snapshotter}/bin/nix2container -n "${namespace}" load ${archive}'' ) - archives - ) - ); - - in { - Unit = { - Wants = [ "containerd.service" "nix-snapshotter.service" ]; - After = [ "containerd.service" "nix-snapshotter.service" ]; - }; - - Service = { - Type = "oneshot"; - ExecStart = "${preload}/bin/preload"; - RemainAfterExit = true; - }; - }; - - mkRootlessPreloadContainerdImageService = cfg: - lib.recursiveUpdate - (mkPreloadContainerdImageService cfg) - { - Unit = { - Description = "Preload images to containerd (Rootless)"; - }; - Install = { - WantedBy = [ "default.target" ]; - }; - }; - -in { - options.services.nix-snapshotter = { - lib = mkOption { - description = lib.mdDoc "Common functions for the nix-snapshotter modules."; - default = { - inherit options; - inherit settingsFormat; - inherit convertServiceToNixOS; - inherit mkContainerdSettings; - inherit mkNixSnapshotterService; - inherit mkRootlessNixSnapshotterService; - inherit mkPreloadContainerdImageService; - inherit mkRootlessPreloadContainerdImageService; - }; - type = types.attrs; - internal = true; - }; - }; -} diff --git a/modules/common/nix-snapshotter-rootless.nix b/modules/common/nix-snapshotter-rootless.nix index 2af1405..0d01e67 100644 --- a/modules/common/nix-snapshotter-rootless.nix +++ b/modules/common/nix-snapshotter-rootless.nix @@ -9,17 +9,18 @@ let ns-lib = config.services.nix-snapshotter.lib; + settingsFormat = pkgs.formats.toml {}; + in { - imports = [ ./nix-snapshotter-lib.nix ]; + imports = [ + ./nix-snapshotter.nix + ]; options.services.nix-snapshotter.rootless = { inherit (ns-lib.options) configFile package path - preloadContainerdImages - setContainerdNamespace - setContainerdSnapshotter settings ; @@ -38,19 +39,7 @@ in { services.nix-snapshotter.rootless = { configFile = lib.mkOptionDefault - (ns-lib.settingsFormat.generate "config.toml" cfg.settings); - }; - - virtualisation.containerd.rootless = { - enable = true; - - # Configure containerd with nix-snapshotter. - settings = ns-lib.mkContainerdSettings; - - bindMounts = { - "$XDG_RUNTIME_DIR/nix-snapshotter".mountPoint = "/run/nix-snapshotter"; - "$XDG_DATA_HOME/nix-snapshotter".mountPoint = "/var/lib/containerd/io.containerd.snapshotter.v1.nix"; - }; + (settingsFormat.generate "config.toml" cfg.settings); }; }; } diff --git a/modules/common/nix-snapshotter.nix b/modules/common/nix-snapshotter.nix new file mode 100644 index 0000000..038c40a --- /dev/null +++ b/modules/common/nix-snapshotter.nix @@ -0,0 +1,115 @@ +{ config, pkgs, lib, ... }: +let + inherit (lib) + mkOption + mkPackageOptionMD + types + ; + + inherit (config.virtualisation.containerd.rootless) + nsenter + ; + + settingsFormat = pkgs.formats.toml {}; + + options = { + configFile = mkOption { + type = types.nullOr types.path; + description = lib.mdDoc '' + Path to nix-snapshotter config file. + Setting this option will override any configuration applied by the + settings option. + ''; + }; + + package = mkPackageOptionMD pkgs "nix-snapshotter" { }; + + path = mkOption { + type = types.listOf types.package; + default = [ pkgs.nix ]; + description = lib.mdDoc '' + Set the path of the nix-snapshotter service, if it requires access to + alternative nix binaries. + ''; + }; + + settings = mkOption { + type = settingsFormat.type; + default = {}; + description = lib.mdDoc '' + Verbatim lines to add to config.toml + ''; + }; + }; + + # SYSTEMD USER SERVICES + # --------------------- + # + # When writing systemd user services targetting both NixOS modules and + # home-manager modules, the easiest transformation is home-manager -> NixOS, + # which is why the services are written as such. + + # Converts a home-manager systemd user service to a NixOS systemd user + # service. Since home-manager style services map closer to raw systemd + # service specification, it's easier to transform in this direction. + convertServiceToNixOS = unit: lib.mkMerge [ + (lib.mkIf (unit ? Service) { + serviceConfig = unit.Service; + }) + (lib.mkIf (unit ? Unit) { + unitConfig = unit.Unit; + }) + (lib.mkIf (unit ? Install.WantedBy) { + # Only `WantedBy` is supported by NixOS as [Install] fields are not + # supported, due to its stateful nature. + wantedBy = unit.Install.WantedBy; + }) + ]; + + mkNixSnapshotterService = { + Service = { + Type = "notify"; + Delegate = "yes"; + KillMode = "mixed"; + Restart = "always"; + RestartSec = "2"; + + StateDirectory = "nix-snapshotter"; + RuntimeDirectory = "nix-snapshotter"; + RuntimeDirectoryPreserve = "yes"; + }; + }; + + mkRootlessNixSnapshotterService = cfg: lib.recursiveUpdate + mkNixSnapshotterService + { + Unit = { + Description = "nix-snapshotter - containerd snapshotter that understands nix store paths natively (Rootless)"; + After = [ "containerd.service" ]; + PartOf = [ "containerd.service" ]; + }; + + Install = { + WantedBy = [ "default.target" ]; + }; + + Service.ExecStart = "${nsenter}/bin/containerd-nsenter ${cfg.package}/bin/nix-snapshotter --config ${cfg.configFile}"; + }; + +in { + options.services.nix-snapshotter = { + lib = mkOption { + type = types.attrs; + description = lib.mdDoc "Common functions for the nix-snapshotter modules."; + default = { + inherit + options + convertServiceToNixOS + mkNixSnapshotterService + mkRootlessNixSnapshotterService + ; + }; + internal = true; + }; + }; +} diff --git a/modules/common/preload-containerd-rootless.nix b/modules/common/preload-containerd-rootless.nix new file mode 100644 index 0000000..b56ba67 --- /dev/null +++ b/modules/common/preload-containerd-rootless.nix @@ -0,0 +1,16 @@ +{ config, ... }: +let + preload-lib = config.services.preload-containerd.lib; + +in { + imports = [ + ./preload-containerd.nix + ]; + + options.services.preload-containerd.rootless = { + inherit (preload-lib.options) + enable + targets + ; + }; +} diff --git a/modules/common/preload-containerd.nix b/modules/common/preload-containerd.nix new file mode 100644 index 0000000..7399bc6 --- /dev/null +++ b/modules/common/preload-containerd.nix @@ -0,0 +1,115 @@ +{ pkgs, lib, ... }: +let + inherit (lib) + mkEnableOption + mkOption + types + ; + + options = { + enable = mkEnableOption "preload-containerd"; + + targets = mkOption { + type = types.listOf targetType; + default = []; + description = lib.mdDoc '' + Specify a list of containerd targets to preload image tar archives. + Each target can specify a different address and namespace. + ''; + }; + }; + + targetType = types.submodule { + options = { + archives = mkOption { + type = types.listOf types.package; + default = []; + description = lib.mdDoc '' + Specify image tar archives to be preloaded to this containerd target. + ''; + }; + + address = mkOption { + type = types.str; + default = "/run/containerd/containerd.sock"; + description = lib.mdDoc '' + Set the containerd address for preloading. + ''; + }; + + namespace = mkOption { + type = types.str; + default = "default"; + description = lib.mdDoc '' + Set the containerd namespace for preloading. + ''; + }; + }; + }; + + mkPreloadContainerdService = cfg: + let + preload = pkgs.writeShellScriptBin "preload" ( + lib.concatStringsSep "\n" + (lib.concatMap + (target: + builtins.map + (archive: '' + ${pkgs.nix-snapshotter}/bin/nix2container \ + -a "${target.address}" \ + -n "${target.namespace}" \ + load ${archive} + '') + target.archives + ) + cfg.targets + ) + ); + + in { + Unit = { + Description = "Preload images to containerd"; + Wants = [ "containerd.service" "nix-snapshotter.service" ]; + After = [ "containerd.service" "nix-snapshotter.service" ]; + }; + + Service = { + Type = "oneshot"; + ExecStart = "${preload}/bin/preload"; + RemainAfterExit = true; + }; + }; + + mkRootlessPreloadContainerdService = cfg: lib.recursiveUpdate + (mkPreloadContainerdService cfg) + { + Unit = { + Description = "Preload images to containerd (Rootless)"; + }; + + Install = { + WantedBy = [ "default.target" ]; + }; + }; + +in { + options.services.preload-containerd = { + inherit (options) + enable + targets + ; + + lib = mkOption { + type = types.attrs; + description = lib.mdDoc "Common functions for preload-containerd."; + default = { + inherit + options + mkPreloadContainerdService + mkRootlessPreloadContainerdService + ; + }; + internal = true; + }; + }; +} diff --git a/modules/flake/default.nix b/modules/flake/default.nix index 7d2fb44..cafe8e0 100644 --- a/modules/flake/default.nix +++ b/modules/flake/default.nix @@ -1,9 +1,11 @@ { imports = [ ./examples.nix + ./k8sResources.nix ./linters.nix ./nixosTests.nix ./overlays.nix ./packages.nix + ./resources.nix ]; } diff --git a/modules/flake/k3s/1_27/chart-versions.nix b/modules/flake/k3s/1_27/chart-versions.nix index d3ff993..1acca4d 100644 --- a/modules/flake/k3s/1_27/chart-versions.nix +++ b/modules/flake/k3s/1_27/chart-versions.nix @@ -1,10 +1,10 @@ { traefik-crd = { - url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-21.2.1+up21.2.0.tgz"; - sha256 = "05j3vyikb7g2z2i07rij9h4ki5lb2hb2rynpiqfd4l1y5qm0qhw9"; + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; + sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; }; traefik = { - url = "https://k3s.io/k3s-charts/assets/traefik/traefik-21.2.1+up21.2.0.tgz"; - sha256 = "0gvz0yzph2893scd0q10b938yc7f36b3zqs57pkjgqqpl1d0nwhg"; + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; + sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; }; } diff --git a/modules/flake/k3s/1_27/versions.nix b/modules/flake/k3s/1_27/versions.nix index d08c3bb..c6e1ab5 100644 --- a/modules/flake/k3s/1_27/versions.nix +++ b/modules/flake/k3s/1_27/versions.nix @@ -1,16 +1,16 @@ { - k3sVersion = "1.27.6+k3s1"; - k3sCommit = "bd04941a294793ec92e8703d5e5da14107902e88"; - k3sRepoSha256 = "04chr8gp0yprihigy1yzhvi2baby053fav384gq0sjq6bkp3fzd8"; - # k3sVendorHash = "sha256-LH9OsBK0Pq/NGEHprbIgYKQsslYdR3i4LYVvo5P0K+8="; + k3sVersion = "1.27.9+k3s1"; + k3sCommit = "2c249a39358bd36438ab53aedef5487d950fd558"; + k3sRepoSha256 = "16zcp1ih34zpz6115ivbcs49n5yikgj8mpiv177jvvb2vakmkgv6"; + # k3sVendorHash = "sha256-zvoBN1mErSXovv/xVzjntHyZjVyCfPzsOdlcTSIwKus="; # Modified hash for ../../patches/k3s-nix-snapshotter patch - k3sVendorHash = "sha256-ryEnRrmV7xkpPIj55N10tz0+eMPHW62IRfmBIh9IbBw="; + k3sVendorHash = "sha256-YwyAFNzTUtW43AdFfpSGbmx/drjKeMkCFFgx1KDZrQs="; chartVersions = import ./chart-versions.nix; k3sRootVersion = "0.12.2"; k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; k3sCNIVersion = "1.3.0-k3s1"; k3sCNISha256 = "0zma9g4wvdnhs9igs03xlx15bk2nq56j73zns9xgqmfiixd9c9av"; - containerdVersion = "1.7.6-k3s1.27"; - containerdSha256 = "1kzjqw56pcdpsqdkw2k5a3pnpf8n93dh4jc2yybgqz3nyj4fw0a8"; + containerdVersion = "1.7.11-k3s2.27"; + containerdSha256 = "0xjxc5dgh3drk2glvcabd885damjffp9r4cs0cm1zgnrrbhlipra"; criCtlVersion = "1.26.0-rc.0-k3s1"; } diff --git a/modules/flake/k3s/builder.nix b/modules/flake/k3s/builder.nix index ece72e5..51a5846 100644 --- a/modules/flake/k3s/builder.nix +++ b/modules/flake/k3s/builder.nix @@ -184,7 +184,14 @@ let src = k3sRepo; vendorHash = k3sVendorHash; - patches = [ ../patches/k3s-nix-snapshotter.patch ]; + patches = [ + # See: https://github.com/k3s-io/k3s/pull/9309 + ../patches/k3s-rootless-containerd-sock.patch + # See: https://github.com/k3s-io/k3s/pull/9308 + ../patches/k3s-rootless-state-dir.patch + # See: https://github.com/k3s-io/k3s/pull/9319 + ../patches/k3s-nix-snapshotter.patch + ]; nativeBuildInputs = [ pkg-config ]; buildInputs = [ libseccomp sqlite.dev ]; @@ -242,7 +249,14 @@ buildGoModule rec { src = k3sRepo; vendorHash = k3sVendorHash; - patches = [ ../patches/k3s-nix-snapshotter.patch ]; + patches = [ + # See: https://github.com/k3s-io/k3s/pull/9309 + ../patches/k3s-rootless-containerd-sock.patch + # See: https://github.com/k3s-io/k3s/pull/9308 + ../patches/k3s-rootless-state-dir.patch + # See: https://github.com/k3s-io/k3s/pull/9319 + ../patches/k3s-nix-snapshotter.patch + ]; postPatch = '' # Nix prefers dynamically linked binaries over static binary. diff --git a/modules/flake/k8sResources.nix b/modules/flake/k8sResources.nix new file mode 100644 index 0000000..8f32465 --- /dev/null +++ b/modules/flake/k8sResources.nix @@ -0,0 +1,24 @@ +{ lib, flake-parts-lib, ... }: +let + inherit (lib) + mkOption + types + ; + + inherit (flake-parts-lib) + mkTransposedPerSystemModule + ; + +in mkTransposedPerSystemModule { + name = "k8sResources"; + + option = mkOption { + type = types.lazyAttrsOf (types.attrsOf types.raw); + default = { }; + description = '' + An attribute set of kubernetes resources. + ''; + }; + + file = ./k8sResources.nix; +} diff --git a/modules/flake/nixosTests.nix b/modules/flake/nixosTests.nix index 37236e0..a60b166 100644 --- a/modules/flake/nixosTests.nix +++ b/modules/flake/nixosTests.nix @@ -19,7 +19,7 @@ in { }; }; - config.perSystem = { config, pkgs, ... }: + config.perSystem = { config, pkgs, k8sResources, ... }: let evalTest = name: module: (lib.nixos.evalTest { @@ -28,7 +28,10 @@ in { module ]; hostPkgs = pkgs; - node.pkgs = pkgs; + node = { + inherit pkgs; + specialArgs = { inherit k8sResources; }; + }; }).config.result; testRigs = lib.mapAttrs (name: module: evalTest name module) config.nixosTests; diff --git a/modules/flake/patches/k3s-nix-snapshotter.patch b/modules/flake/patches/k3s-nix-snapshotter.patch index 00b91bc..cfa4df6 100644 --- a/modules/flake/patches/k3s-nix-snapshotter.patch +++ b/modules/flake/patches/k3s-nix-snapshotter.patch @@ -1,38 +1,35 @@ +commit 0484bf68aaa9cdccb9444affbafaf3f8f6fc0e2a +Author: Edgar Lee +Date: Wed Feb 14 05:27:28 2024 -0500 + + Add support for nix-snapshotter + diff --git a/go.mod b/go.mod -index 015993bb90..c94e9e5814 100644 +index 1b55d6cd07..18bae154b3 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace ( github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.11.0 github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.4-k3s1 // k3s/release-1.27 - github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.0.1-0.20230508174102-b42e5faded1c -- github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.6-k3s1.27 + github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.0.0-20230925161250-364f994b140b +- github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.11-k3s2.27 + github.com/containerd/containerd => github.com/pdtpartners/containerd v1.0.1-stargz.0.20240214083438-8c533e066c1c github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e github.com/docker/distribution => github.com/docker/distribution v2.8.2+incompatible github.com/docker/docker => github.com/docker/docker v24.0.0-rc.2.0.20230801142700-69c9adb7d386+incompatible -@@ -97,7 +97,7 @@ require ( - github.com/containerd/zfs v1.1.0 - github.com/coreos/go-iptables v0.6.0 - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf -- github.com/docker/docker v23.0.3+incompatible -+ github.com/docker/docker v23.0.5+incompatible - github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 - github.com/flannel-io/flannel v0.22.2 - github.com/go-bindata/go-bindata v3.1.2+incompatible -@@ -118,9 +118,10 @@ require ( +@@ -120,9 +120,10 @@ require ( github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/onsi/ginkgo/v2 v2.9.4 - github.com/onsi/gomega v1.27.6 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.10 - github.com/opencontainers/runc v1.1.6 + github.com/opencontainers/runc v1.1.9 github.com/opencontainers/selinux v1.11.0 github.com/otiai10/copy v1.7.0 -+ github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214070632-154924c8d48c ++ github.com/pdtpartners/nix-snapshotter v0.1.2 github.com/pkg/errors v0.9.1 - github.com/rancher/dynamiclistener v0.3.6-rc2 - github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc -@@ -128,7 +129,7 @@ require ( + github.com/rancher/dynamiclistener v0.3.6 + github.com/rancher/lasso v0.0.0-20230830164424-d684fdeb6f29 +@@ -130,7 +131,7 @@ require ( github.com/rancher/wharfie v0.5.3 github.com/rancher/wrangler v1.1.1 github.com/robfig/cron/v3 v3.0.1 @@ -41,15 +38,15 @@ index 015993bb90..c94e9e5814 100644 github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 -@@ -167,6 +168,7 @@ require ( +@@ -169,6 +170,7 @@ require ( require ( - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.21.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect + dario.cat/mergo v1.0.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/azure-sdk-for-go v56.3.0+incompatible // indirect -@@ -201,7 +203,7 @@ require ( +@@ -202,7 +204,7 @@ require ( github.com/chai2010/gettext-go v1.0.2 // indirect github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect github.com/cilium/ebpf v0.9.1 // indirect @@ -57,9 +54,17 @@ index 015993bb90..c94e9e5814 100644 + github.com/container-orchestrated-devices/container-device-interface v0.6.0 // indirect github.com/container-storage-interface/spec v1.7.0 // indirect github.com/containerd/btrfs/v2 v2.0.0 // indirect - github.com/containerd/cgroups/v3 v3.0.2 // indirect -@@ -331,10 +333,11 @@ require ( - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect +@@ -212,7 +214,6 @@ require ( + github.com/containerd/go-cni v1.1.9 // indirect + github.com/containerd/go-runc v1.0.0 // indirect + github.com/containerd/imgcrypt v1.1.7 // indirect +- github.com/containerd/log v0.1.0 // indirect + github.com/containerd/nri v0.4.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/containerd/ttrpc v1.2.2 // indirect +@@ -332,10 +333,11 @@ require ( + github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc3 // indirect @@ -71,53 +76,39 @@ index 015993bb90..c94e9e5814 100644 github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect -@@ -355,7 +358,7 @@ require ( - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect +@@ -357,7 +359,7 @@ require ( github.com/tchap/go-patricia/v2 v2.3.1 // indirect + github.com/tidwall/btree v1.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 // indirect - github.com/urfave/cli/v2 v2.23.5 // indirect + github.com/urfave/cli/v2 v2.25.7 // indirect github.com/vbatts/tar-split v0.11.5 // indirect github.com/vishvananda/netns v0.0.4 // indirect - github.com/vmware/govmomi v0.30.0 // indirect -@@ -382,18 +385,18 @@ require ( - go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect -- golang.org/x/mod v0.10.0 // indirect -+ golang.org/x/mod v0.12.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/time v0.3.0 // indirect -- golang.org/x/tools v0.8.0 // indirect -+ golang.org/x/tools v0.11.0 // indirect - golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect + github.com/vmware/govmomi v0.30.6 // indirect +@@ -394,7 +396,7 @@ require ( golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect - google.golang.org/api v0.108.0 // indirect + google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect -- google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect -- google.golang.org/protobuf v1.30.0 // indirect +- google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto v0.0.0-20230720185612-659f7aaaa771 // indirect -+ google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/gcfg.v1 v1.2.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/gcfg.v1 v1.2.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum -index 7ac62863e5..b8758ea05a 100644 +index 8038f23cb8..9b4af4e651 100644 --- a/go.sum +++ b/go.sum -@@ -57,6 +57,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 - cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= - cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +@@ -71,6 +71,8 @@ cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeL cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= + cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= + cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20221206110420-d395f97c4830/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= -@@ -200,8 +202,9 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH - github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +@@ -216,8 +218,9 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+g + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= @@ -127,16 +118,26 @@ index 7ac62863e5..b8758ea05a 100644 github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY= -@@ -629,8 +632,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X +@@ -250,8 +253,7 @@ github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nN + github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= + github.com/containerd/imgcrypt v1.1.7 h1:WSf9o9EQ0KGHiUx2ESFZ+PKf4nxK9BcvV/nJDX8RkB4= + github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k= +-github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +-github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= ++github.com/containerd/nri v0.3.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI= + github.com/containerd/nri v0.4.0 h1:PjgIBm0RtUiFyEO6JqPBQZRQicbsIz41Fz/5VSC0zgw= + github.com/containerd/nri v0.4.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI= + github.com/containerd/stargz-snapshotter v0.14.4-0.20230913082252-7275d45b185c h1:Qr2w9ZeMciAfruOt2be10s4W13vQiTD3gAEqz3zxUrg= +@@ -661,8 +663,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= --github.com/k3s-io/containerd v1.7.6-k3s1.27 h1:BGoNTvkH/rxURf5UCvE4hIDY2emZ3o1DPNlTAmCoJpI= --github.com/k3s-io/containerd v1.7.6-k3s1.27/go.mod h1:dWUW/BzVXrFhxzfRZ1Jmr/yLlRvjryZlb1ns2SCHsgs= +-github.com/k3s-io/containerd v1.7.11-k3s2.27 h1:p7mKeooDKqLkqiOcNUz1qFjolKOPdwGwZgdzU0GZBmw= +-github.com/k3s-io/containerd v1.7.11-k3s2.27/go.mod h1:bqKO8PJAbtNpsqOM6fl5AICYSahNGKPH2K/YQQpk6Ts= github.com/k3s-io/cri-dockerd v0.3.4-k3s1 h1:eCeVCeXzf10fyanv1gniSwidBjdO83/akv+M72uEnZc= github.com/k3s-io/cri-dockerd v0.3.4-k3s1/go.mod h1:0KDOU8lLjp+ETJFFCcVBRQbJ8puRoDxaHBDj8C87Fk4= github.com/k3s-io/cri-tools v1.26.0-rc.0-k3s1 h1:yWVy9pS0T1BWBMZBPRy2Q29gaLmaGknQHSnx+HStrVM= -@@ -934,8 +935,8 @@ github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zM +@@ -969,8 +969,8 @@ github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zM github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= @@ -144,17 +145,17 @@ index 7ac62863e5..b8758ea05a 100644 -github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= - github.com/opencontainers/runc v1.1.8 h1:zICRlc+C1XzivLc3nzE+cbJV4LIi8tib6YG0MqC6OqA= - github.com/opencontainers/runc v1.1.8/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= + github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQBizE40= + github.com/opencontainers/runc v1.1.10/go.mod h1:+/R6+KmDlh+hOO8NkjmgkG9Qzvypzk0yXxAPYYR65+M= github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 h1:R5M2qXZiK/mWPMT4VldCOiSL9HIAMuxQZWdG0CSM5+4= -@@ -954,10 +955,16 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 +@@ -989,10 +989,16 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pdtpartners/containerd v1.0.1-stargz.0.20240214083438-8c533e066c1c h1:OWE2iwUICpISBOLVu9OpelITX4sIZwOabjbF4Vcjm8I= +github.com/pdtpartners/containerd v1.0.1-stargz.0.20240214083438-8c533e066c1c/go.mod h1:dWUW/BzVXrFhxzfRZ1Jmr/yLlRvjryZlb1ns2SCHsgs= -+github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214070632-154924c8d48c h1:FdjMmoAD5rdlg2kNIGFLbfwkzV8eO32lDdB6MNAO2BM= -+github.com/pdtpartners/nix-snapshotter v0.1.2-0.20240214070632-154924c8d48c/go.mod h1:iMHTz4uBpoXqj+xq+uCGwBwpe2Or4CqF3R40daLzlqc= ++github.com/pdtpartners/nix-snapshotter v0.1.2 h1:rbvHZ5s/L34x8UAuJxN/H6rPgo7Jmg8fk+1hAQMFEr8= ++github.com/pdtpartners/nix-snapshotter v0.1.2/go.mod h1:MKa+V5fH15XmLCDt+s8qRQeIAaadaJ3/4+/oD7f0K0k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= @@ -164,10 +165,10 @@ index 7ac62863e5..b8758ea05a 100644 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -@@ -1037,8 +1044,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE +@@ -1073,8 +1079,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= - github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= - github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= + github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= + github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rootless-containers/rootlesskit v1.0.1 h1:jepqW1txFSowKSMAEkVhWH3Oa1TCY9S400MVYe/6Iro= -github.com/rootless-containers/rootlesskit v1.0.1/go.mod h1:t2UAiYagxrJ+wmpFAUIZPcqsm4k2B7ve6g7lILKbloc= +github.com/rootless-containers/rootlesskit v1.1.1 h1:F5psKWoWY9/VjZ3ifVcaosjvFZJOagX85U22M0/EQZE= @@ -175,7 +176,7 @@ index 7ac62863e5..b8758ea05a 100644 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= -@@ -1139,8 +1146,8 @@ github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX +@@ -1179,8 +1185,8 @@ github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= @@ -186,68 +187,52 @@ index 7ac62863e5..b8758ea05a 100644 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= -@@ -1285,8 +1292,9 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= - golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= - golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= - golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= --golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= - golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -+golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= - golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= - golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= - golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -@@ -1409,8 +1417,9 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= - golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= - golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= - golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= --golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= - golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -+golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -+golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= - golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -@@ -1447,8 +1456,9 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw - google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= - google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= - google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= --google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= - google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= - gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= - gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -@@ -1492,8 +1502,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +@@ -1260,7 +1266,7 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY= + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 h1:Wz7UQn7/eIqZVDJbuNEM6PmqeA71cWXrWcXekP5HZgU= + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0/go.mod h1:OhH1xvgA5jZW2M/S4PcvtDlFE1VULRRBsibBrKuJQGI= +-go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= ++go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= + go.opentelemetry.io/otel/metric v0.32.1 h1:ftff5LSBCIDwL0UkhBuDg8j9NNxx2IusvJ18q9h6RC4= + go.opentelemetry.io/otel/metric v0.32.1/go.mod h1:iLPP7FaKMAD5BIxJ2VX7f2KTuz//0QK2hEUyti5psqQ= + go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM= +@@ -1488,6 +1494,7 @@ google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11K + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= + google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= ++google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= + google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +@@ -1554,8 +1561,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -+gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= ++gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go -index ee02becf87..0539c049f3 100644 +index c3cdf316b8..2c1e6ed25d 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go -@@ -546,6 +546,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N - nodeConfig.Containerd.Root) +@@ -553,6 +553,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N + nodeConfig.Containerd.Root) + } + nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" ++ case "nix": ++ if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil { ++ return nil, errors.Wrapf(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", ++ nodeConfig.Containerd.Root) ++ } ++ nodeConfig.AgentConfig.ImageServiceSocket = "/run/k3s/nix-snapshotter/nix-snapshotter.sock" } - nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" -+ case "nix": -+ if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil { -+ return nil, errors.Wrapf(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"", -+ nodeConfig.Containerd.Root) -+ } -+ nodeConfig.AgentConfig.ImageServiceSocket = "/run/k3s/nix-snapshotter/nix-snapshotter.sock" - } - } - nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd") + } else { + nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go -index 559d966dec..02c19281e3 100644 +index 4a5cd21dfe..ae43adc899 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -18,6 +18,7 @@ import ( @@ -258,7 +243,7 @@ index 559d966dec..02c19281e3 100644 "github.com/pkg/errors" "github.com/rancher/wharfie/pkg/registries" "github.com/sirupsen/logrus" -@@ -120,3 +121,7 @@ func FuseoverlayfsSupported(root string) error { +@@ -127,3 +128,7 @@ func FuseoverlayfsSupported(root string) error { func StargzSupported(root string) error { return stargz.Supported(root) } @@ -267,7 +252,7 @@ index 559d966dec..02c19281e3 100644 + return nix.Supported(root) +} diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go -index 6efbb7a148..2f48418f9a 100644 +index 6efbb7a148..8cd64a52bc 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -85,3 +85,7 @@ func FuseoverlayfsSupported(root string) error { @@ -276,10 +261,10 @@ index 6efbb7a148..2f48418f9a 100644 } + +func NixSupported(root string) error { -+ return errors.Wrapf(util2.ErrUnsupportedPlatform, "nix is not supported") ++ return errors.Wrapf(util3.ErrUnsupportedPlatform, "nix is not supported") +} diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go -index 4d0c52fc42..83cdb4909c 100644 +index 66f8ee1080..263317055a 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -35,7 +35,7 @@ version = 2 @@ -288,20 +273,20 @@ index 4d0c52fc42..83cdb4909c 100644 snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" - disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}} + {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} - [plugins."io.containerd.snapshotter.v1.stargz"] -@@ -74,6 +74,14 @@ enable_keychain = true +@@ -75,6 +75,14 @@ enable_keychain = true {{end}} {{end}} {{end}} +{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }} -+[[plugins."io.containerd.transfer.v1.local".unpack_config]] -+platform = "linux/amd64" -+snapshotter = "nix" +[plugins."io.containerd.snapshotter.v1.nix"] +address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" +image_service.enable = true ++[[plugins."io.containerd.transfer.v1.local".unpack_config]] ++platform = "linux/amd64" ++snapshotter = "nix" +{{end}} {{end}} diff --git a/modules/flake/patches/k3s-rootless-containerd-sock.patch b/modules/flake/patches/k3s-rootless-containerd-sock.patch new file mode 100644 index 0000000..42dcee3 --- /dev/null +++ b/modules/flake/patches/k3s-rootless-containerd-sock.patch @@ -0,0 +1,54 @@ +commit 06cd3dd65431192ea1d7eba691bc2cc032448226 +Author: Edgar Lee +Date: Sat Jan 27 08:51:17 2024 -0500 + + Expose rootless containerd socket directories for external access + + Signed-off-by: Edgar Lee + +diff --git a/pkg/rootless/mounts.go b/pkg/rootless/mounts.go +index 2c19f2343c..f4db388a9e 100644 +--- a/pkg/rootless/mounts.go ++++ b/pkg/rootless/mounts.go +@@ -6,6 +6,7 @@ package rootless + import ( + "fmt" + "os" ++ "os/user" + "path/filepath" + "strings" + +@@ -25,11 +26,17 @@ func setupMounts(stateDir string) error { + _ = os.RemoveAll(f) + } + ++ runDir, err := resolveRunDir() ++ if err != nil { ++ return err ++ } ++ + mountMap := [][]string{ + {"/var/log", filepath.Join(stateDir, "logs")}, + {"/var/lib/cni", filepath.Join(stateDir, "cni")}, + {"/var/lib/kubelet", filepath.Join(stateDir, "kubelet")}, + {"/etc/rancher", filepath.Join(stateDir, "etc", "rancher")}, ++ {"/run/k3s/containerd", filepath.Join(runDir, "k3s", "containerd")}, + } + + for _, v := range mountMap { +@@ -91,3 +98,15 @@ func setupMount(target, dir string) error { + logrus.Debug("Mounting ", dir, target, " none bind") + return unix.Mount(dir, target, "none", unix.MS_BIND, "") + } ++ ++func resolveRunDir() (string, error) { ++ runDir := os.Getenv("XDG_RUNTIME_DIR") ++ if runDir == "" { ++ u, err := user.Lookup(os.Getenv("USER")) ++ if err != nil { ++ return "", err ++ } ++ runDir = filepath.Join("/run/user", u.Uid) ++ } ++ return runDir, nil ++} diff --git a/modules/flake/patches/k3s-rootless-state-dir.patch b/modules/flake/patches/k3s-rootless-state-dir.patch new file mode 100644 index 0000000..2e7befd --- /dev/null +++ b/modules/flake/patches/k3s-rootless-state-dir.patch @@ -0,0 +1,24 @@ +commit 065e2afaf61e9e6996b8c1d909e13840ee5639fa +Author: Edgar Lee +Date: Fri Jan 26 22:49:37 2024 -0500 + + Expose rootless state dir under ~/.rancher/k3s/rootless + + Signed-off-by: Edgar Lee + +diff --git a/pkg/rootless/rootless.go b/pkg/rootless/rootless.go +index 97266d7eaf..f584e7b4a7 100644 +--- a/pkg/rootless/rootless.go ++++ b/pkg/rootless/rootless.go +@@ -132,11 +132,6 @@ func createParentOpt(driver portDriver, stateDir string, enableIPv6 bool) (*pare + return nil, errors.Wrapf(err, "failed to mkdir %s", stateDir) + } + +- stateDir, err := os.MkdirTemp("", "rootless") +- if err != nil { +- return nil, err +- } +- + driver.SetStateDir(stateDir) + + opt := &parent.Opt{ diff --git a/modules/flake/resources.nix b/modules/flake/resources.nix new file mode 100644 index 0000000..d600e47 --- /dev/null +++ b/modules/flake/resources.nix @@ -0,0 +1,54 @@ +{ + perSystem = { pkgs, ... }: + let + redis = pkgs.nix-snapshotter.buildImage { + name = "redis"; + resolvedByNix = true; + config = { + entrypoint = [ "${pkgs.redis}/bin/redis-server" ]; + }; + }; + + k8sResources = { + redisPod = { + apiVersion = "v1"; + kind = "Pod"; + metadata = { + name = "redis"; + labels.name = "redis"; + }; + spec.containers = [{ + inherit (redis) name image; + args = ["--protected-mode" "no"]; + ports = [{ + name = "client"; + containerPort = 6379; + }]; + }]; + }; + + redisService = { + apiVersion = "v1"; + kind = "Service"; + metadata.name = "redis-service"; + spec = { + # In rootless k3s, only LoadBalancer service ports are binded to host. + type = "LoadBalancer"; + selector.name = "redis"; + ports = [{ + name = "client"; + # Exposed as localhost:30000 + port = 30000; + targetPort = 6379; + }]; + }; + }; + }; + + in { + # Load k8s resources into VM. + _module.args = { inherit k8sResources; }; + + inherit k8sResources; + }; +} diff --git a/modules/home/containerd-rootless.nix b/modules/home/containerd-rootless.nix index a9cf01c..58a9d76 100644 --- a/modules/home/containerd-rootless.nix +++ b/modules/home/containerd-rootless.nix @@ -2,17 +2,27 @@ let cfg = config.virtualisation.containerd.rootless; - ns-lib = config.services.nix-snapshotter.lib; - in { - imports = [ ../common/containerd-rootless.nix ]; + imports = [ + ../common/containerd-rootless.nix + ]; config = lib.mkIf cfg.enable { - home.sessionVariablesExtra = lib.optionalString cfg.setSocketVariable '' - if [ -z "$CONTAINERD_ADDRESS" -a -n "$XDG_RUNTIME_DIR" ]; then - export CONTAINERD_ADDRESS="$XDG_RUNTIME_DIR/containerd/containerd.sock" + home.sessionVariablesExtra = '' + if [ -z "$CONTAINERD_ADDRESS" ]; then + export CONTAINERD_ADDRESS="${cfg.setAddress}" + fi + '' + + (lib.optionalString (cfg.setNamespace != "default") '' + if [ -z "$CONTAINERD_NAMESPACE" ]; then + export CONTAINERD_NAMESPACE="${cfg.setNamespace}" + fi + '') + + (lib.optionalString (cfg.setSnapshotter != "") '' + if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then + export CONTAINERD_SNAPSHOTTER="${cfg.setSnapshotter}" fi - ''; + ''); systemd.user.services.containerd = cfg.lib.mkRootlessContainerdService cfg; }; diff --git a/modules/home/default.nix b/modules/home/default.nix index b36aee6..2da769c 100644 --- a/modules/home/default.nix +++ b/modules/home/default.nix @@ -6,8 +6,15 @@ ``` */ flake.homeModules = rec { - default = nix-snapshotter-rootless; + default = { + imports = [ + nix-snapshotter-rootless + containerd-rootless + ]; + }; nix-snapshotter-rootless = ./nix-snapshotter-rootless.nix; containerd-rootless = ./containerd-rootless.nix; + preload-containerd-rootless = ./preload-containerd-rootless.nix; + k3s-rootless = ./k3s-rootless.nix; }; } diff --git a/modules/home/k3s-rootless.nix b/modules/home/k3s-rootless.nix new file mode 100644 index 0000000..cf34ec4 --- /dev/null +++ b/modules/home/k3s-rootless.nix @@ -0,0 +1,31 @@ +{ config, lib, ... }: +let + cfg = config.services.k3s.rootless; + +in { + imports = [ + ../common/k3s-rootless.nix + ]; + + config = lib.mkIf cfg.enable { + systemd.user.services.k3s = cfg.lib.mkRootlessK3sService cfg; + + home.sessionVariablesExtra = + (lib.optionalString cfg.setEmbeddedContainerd '' + if [ -z "$CONTAINERD_ADDRESS" ]; then + export CONTAINERD_ADDRESS="$XDG_RUNTIME_DIR/k3s/containerd/containerd.sock" + fi + if [ -z "$CONTAINERD_NAMESPACE" ]; then + export CONTAINERD_NAMESPACE="k8s.io" + fi + if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then + export CONTAINERD_SNAPSHOTTER="${cfg.snapshotter}" + fi + '') + + (lib.optionalString cfg.setKubeConfig '' + if [ -z "$KUBECONFIG" ]; then + export KUBECONFIG="$HOME/.kube/k3s.yaml" + fi + ''); + }; +} diff --git a/modules/home/nix-snapshotter-rootless.nix b/modules/home/nix-snapshotter-rootless.nix index ccf9b76..e74231a 100644 --- a/modules/home/nix-snapshotter-rootless.nix +++ b/modules/home/nix-snapshotter-rootless.nix @@ -7,31 +7,12 @@ let in { imports = [ ../common/nix-snapshotter-rootless.nix - ./containerd-rootless.nix ]; - config = lib.mkIf cfg.enable (lib.mkMerge [ - { - home.sessionVariablesExtra = - (lib.optionalString cfg.setContainerdSnapshotter '' - if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then - export CONTAINERD_SNAPSHOTTER="nix" - fi - '') + - (lib.optionalString (cfg.setContainerdNamespace != "default") '' - if [ -z "$CONTAINERD_NAMESPACE" ]; then - export CONTAINERD_NAMESPACE="${cfg.setContainerdNamespace}" - fi - ''); - - systemd.user.services.nix-snapshotter = lib.recursiveUpdate - (ns-lib.mkRootlessNixSnapshotterService cfg) - { Service.Environment = "PATH=${lib.makeBinPath cfg.path}"; }; - } - (lib.mkIf (cfg.preloadContainerdImages != []) { - systemd.user.services.preload-containerd-images = lib.recursiveUpdate - (ns-lib.mkRootlessPreloadContainerdImageService cfg) - { Service.Environment = "CONTAINERD_ADDRESS=%t/containerd/containerd.sock"; }; - }) - ]); + config = lib.mkIf cfg.enable { + systemd.user.services.nix-snapshotter = lib.mkMerge [ + (ns-lib.mkRootlessNixSnapshotterService cfg) + { Service.Environment = "PATH=${lib.makeBinPath cfg.path}"; } + ]; + }; } diff --git a/modules/home/preload-containerd-rootless.nix b/modules/home/preload-containerd-rootless.nix new file mode 100644 index 0000000..d00b606 --- /dev/null +++ b/modules/home/preload-containerd-rootless.nix @@ -0,0 +1,18 @@ +{ config, lib, ... }: +let + cfg = config.services.preload-containerd.rootless; + + preload-lib = config.services.preload-containerd.lib; + +in { + imports = [ + ../common/preload-containerd-rootless.nix + ]; + + config = lib.mkIf cfg.enable { + systemd.user.services.preload-containerd = + lib.mkIf + (cfg.targets != []) + (preload-lib.mkRootlessPreloadContainerdService cfg); + }; +} diff --git a/modules/nixos/containerd-rootless.nix b/modules/nixos/containerd-rootless.nix index 8174514..4c4c946 100644 --- a/modules/nixos/containerd-rootless.nix +++ b/modules/nixos/containerd-rootless.nix @@ -7,17 +7,30 @@ let proxyEnv = config.networking.proxy.envVars; in { - imports = [ ../common/containerd-rootless.nix ]; + imports = [ + ../common/containerd-rootless.nix + ]; - config = lib.mkIf cfg.enable { - environment.extraInit = lib.optionalString cfg.setSocketVariable '' - if [ -z "$CONTAINERD_ADDRESS" -a -n "$XDG_RUNTIME_DIR" ]; then - export CONTAINERD_ADDRESS="$XDG_RUNTIME_DIR/containerd/containerd.sock" + config = lib.mkIf cfg.enable { + environment.extraInit = '' + if [ -z "$CONTAINERD_ADDRESS" ]; then + export CONTAINERD_ADDRESS="${cfg.setAddress}" fi - ''; + '' + + (lib.optionalString (cfg.setNamespace != "default") '' + if [ -z "$CONTAINERD_NAMESPACE" ]; then + export CONTAINERD_NAMESPACE="${cfg.setNamespace}" + fi + '') + + (lib.optionalString (cfg.setSnapshotter != "") '' + if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then + export CONTAINERD_SNAPSHOTTER="${cfg.setSnapshotter}" + fi + ''); - systemd.user.services.containerd = lib.recursiveUpdate + systemd.user.services.containerd = lib.mkMerge [ (ns-lib.convertServiceToNixOS (cfg.lib.mkRootlessContainerdService cfg)) - { environment = proxyEnv; }; + { environment = proxyEnv; } + ]; }; } diff --git a/modules/nixos/containerd.nix b/modules/nixos/containerd.nix new file mode 100644 index 0000000..95eb348 --- /dev/null +++ b/modules/nixos/containerd.nix @@ -0,0 +1,74 @@ +{ config, pkgs, lib, ... }: +let + cfg = config.virtualisation.containerd; + + k3s-cni-plugins = pkgs.buildEnv { + name = "k3s-cni-plugins"; + paths = with pkgs; [ + cni-plugins + cni-plugin-flannel + ]; + }; + +in { + imports = [ + ../common/containerd.nix + ./k3s.nix + ]; + + config = lib.mkIf cfg.enable (lib.mkMerge [ + { + environment.extraInit = '' + if [ -z "$CONTAINERD_ADDRESS" ]; then + export CONTAINERD_ADDRESS="${cfg.setAddress}" + fi + '' + + (lib.optionalString (cfg.setNamespace != "default") '' + if [ -z "$CONTAINERD_NAMESPACE" ]; then + export CONTAINERD_NAMESPACE="${cfg.setNamespace}" + fi + '') + + (lib.optionalString (cfg.setSnapshotter != "") '' + if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then + export CONTAINERD_SNAPSHOTTER="${cfg.setSnapshotter}" + fi + ''); + } + (lib.mkIf cfg.k3sIntegration { + services.k3s.moreFlags = [ + "--container-runtime-endpoint unix:///run/containerd/containerd.sock" + ]; + + virtualisation.containerd = { + setNamespace = lib.mkDefault "k8s.io"; + + settings.plugins."io.containerd.grpc.v1.cri" = { + stream_server_address = "127.0.0.1"; + stream_server_port = "10010"; + enable_selinux = false; + enable_unprivileged_ports = true; + enable_unprivileged_icmp = true; + disable_apparmor = true; + disable_cgroup = true; + restrict_oom_score_adj = true; + sandbox_image = "rancher/mirrored-pause:3.6"; + + cni = { + conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; + bin_dir = "${k3s-cni-plugins}/bin"; + }; + + containerd.runtimes.runc = { + runtime_type = "io.containerd.runc.v2"; + options.SystemdCgroup = false; + }; + }; + }; + }) + (lib.mkIf (cfg.k3sIntegration && cfg.nixSnapshotterIntegration) { + services.k3s.moreFlags = [ + "--image-service-endpoint unix:///run/nix-snapshotter/nix-snapshotter.sock" + ]; + }) + ]); +} diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index d88e41e..0c7f3a1 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -2,8 +2,9 @@ let nixosSystemFor = system: module: let - pkgs = withSystem system ({ pkgs, ...}: pkgs); - examples = withSystem system ({ examples, ...}: examples); + pkgs = withSystem system ({ pkgs, ... }: pkgs); + examples = withSystem system ({ examples, ... }: examples); + k8sResources = withSystem system ({ k8sResources, ... }: k8sResources); in lib.nixosSystem { inherit system; @@ -11,7 +12,7 @@ let modules = [ { _module.args = { - inherit examples; + inherit examples k8sResources; pkgs = lib.mkForce pkgs; }; } @@ -20,6 +21,11 @@ let ]; }; + vmApp = name: { + type = "app"; + program = "${self.nixosConfigurations.${name}.config.system.build.vm}/bin/run-nixos-vm"; + }; + in { /* NixOS module to provide nix-snapshotter systemd service. @@ -32,12 +38,23 @@ in { imports = [ nix-snapshotter nix-snapshotter-rootless + containerd + containerd-rootless + preload-containerd + preload-containerd-rootless + k3s + k3s-rootless ]; }; nix-snapshotter = ./nix-snapshotter.nix; nix-snapshotter-rootless = ./nix-snapshotter-rootless.nix; + containerd = ./containerd.nix; containerd-rootless = ./containerd-rootless.nix; + preload-containerd = ./preload-containerd.nix; + preload-containerd-rootless = ./preload-containerd-rootless.nix; + k3s = ./k3s.nix; + k3s-rootless = ./k3s-rootless.nix; }; /* NixOS config for a VM to quickly try out nix-snapshotter. @@ -46,7 +63,10 @@ in { nixos-rebuild build-vm --flake .#vm ``` */ - flake.nixosConfigurations.vm = nixosSystemFor "x86_64-linux" ./vm.nix; + flake.nixosConfigurations = { + vm = nixosSystemFor "x86_64-linux" ./vm.nix; + vm-rootless = nixosSystemFor "x86_64-linux" ./vm-rootless.nix; + }; perSystem = { system, ... }: { /* A convenient `apps` target to run a NixOS VM to quickly try out @@ -56,13 +76,16 @@ in { nix run .#vm ``` */ - apps.vm = { - type = "app"; - program = "${(nixosSystemFor system ./vm.nix).config.system.build.vm}/bin/run-nixos-vm"; + apps = { + vm = vmApp "vm"; + vm-rootless = vmApp "vm-rootless"; }; # NixOS tests for nix-snapshotter. nixosTests.snapshotter = import ./tests/snapshotter.nix; - nixosTests.kubernetes= import ./tests/kubernetes.nix; + nixosTests.kubernetes = import ./tests/kubernetes.nix; + nixosTests.k3s = import ./tests/k3s.nix; + nixosTests.k3s-external = import ./tests/k3s-external.nix; + nixosTests.k3s-rootless = import ./tests/k3s-rootless.nix; }; } diff --git a/modules/nixos/k3s-rootless.nix b/modules/nixos/k3s-rootless.nix new file mode 100644 index 0000000..c4ade62 --- /dev/null +++ b/modules/nixos/k3s-rootless.nix @@ -0,0 +1,43 @@ +{ config, lib, ... }: +let + cfg = config.services.k3s.rootless; + + ns-lib = config.services.nix-snapshotter.lib; + +in { + imports = [ + ../common/nix-snapshotter.nix + ../common/k3s-rootless.nix + ]; + + config = lib.mkIf cfg.enable { + systemd.user.services.k3s = + ns-lib.convertServiceToNixOS (cfg.lib.mkRootlessK3sService cfg); + + environment.extraInit = + (lib.optionalString cfg.setEmbeddedContainerd '' + if [ -z "$CONTAINERD_ADDRESS" ]; then + export CONTAINERD_ADDRESS="$XDG_RUNTIME_DIR/k3s/containerd/containerd.sock" + fi + if [ -z "$CONTAINERD_NAMESPACE" ]; then + export CONTAINERD_NAMESPACE="k8s.io" + fi + if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then + export CONTAINERD_SNAPSHOTTER="${cfg.snapshotter}" + fi + if [ -z "$ROOTLESSKIT_STATE_DIR" ]; then + export ROOTLESSKIT_STATE_DIR="$HOME/.rancher/k3s/rootless" + fi + '') + + (lib.optionalString cfg.setKubeConfig '' + if [ -z "$KUBECONFIG" ]; then + export KUBECONFIG="$HOME/.kube/k3s.yaml" + fi + ''); + + # Crucial to enable k3s rootless mode. + # See: https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cpu-cpuset-and-io-delegation + systemd.services."user@".serviceConfig.Delegate = "memory pids cpu cpuset"; + boot.kernel.sysctl."net.ipv4.ip_forward" = "1"; + }; +} diff --git a/modules/nixos/k3s.nix b/modules/nixos/k3s.nix index 7bbe2b0..5f4b168 100644 --- a/modules/nixos/k3s.nix +++ b/modules/nixos/k3s.nix @@ -1,17 +1,56 @@ -{ pkgs, ... }: -{ - services.k3s = { - enable = true; - extraFlags = toString [ - "--snapshotter nix" - ]; - }; +{ config, lib, pkgs, ... }: +let + inherit (lib) + mkOption + types + ; + + cfg = config.services.k3s; - systemd.services.k3s.path = with pkgs; [ - nix +in { + imports = [ + ../common/k3s.nix ]; - environment.sessionVariables = { - KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; + options = { + services.k3s = { + # This was introduced to provide a mergeable listOf type. + moreFlags = mkOption { + type = types.listOf types.str; + description = lib.mdDoc "Extra flags to pass to the k3s command."; + default = []; + example = [ "--no-deploy traefik" "--cluster-cidr 10.24.0.0/16" ]; + }; + }; + }; + + config = lib.mkIf cfg.enable { + environment.extraInit = + (lib.optionalString cfg.setEmbeddedContainerd '' + if [ -z "$CONTAINERD_ADDRESS" ]; then + export CONTAINERD_ADDRESS="/run/k3s/containerd/containerd.sock" + fi + if [ -z "$CONTAINERD_NAMESPACE" ]; then + export CONTAINERD_NAMESPACE="k8s.io" + fi + if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then + export CONTAINERD_SNAPSHOTTER="${cfg.snapshotter}" + fi + '') + + (lib.optionalString cfg.setKubeConfig '' + if [ -z "$KUBECONFIG" ]; then + export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + fi + ''); + + services.k3s = { + moreFlags = [ "--snapshotter ${cfg.snapshotter}" ]; + + extraFlags = toString cfg.moreFlags; + }; + + systemd.services.k3s.path = lib.mkIf (cfg.snapshotter == "nix") [ + pkgs.nix + ]; }; } diff --git a/modules/nixos/nix-snapshotter-rootless.nix b/modules/nixos/nix-snapshotter-rootless.nix index 411d005..f665dc8 100644 --- a/modules/nixos/nix-snapshotter-rootless.nix +++ b/modules/nixos/nix-snapshotter-rootless.nix @@ -7,31 +7,12 @@ let in { imports = [ ../common/nix-snapshotter-rootless.nix - ./containerd-rootless.nix ]; - config = lib.mkIf cfg.enable (lib.mkMerge [ - { - environment.extraInit = - (lib.optionalString cfg.setContainerdSnapshotter '' - if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then - export CONTAINERD_SNAPSHOTTER="nix" - fi - '') + - (lib.optionalString (cfg.setContainerdNamespace != "default") '' - if [ -z "$CONTAINERD_NAMESPACE" ]; then - export CONTAINERD_NAMESPACE="${cfg.setContainerdNamespace}" - fi - ''); - - systemd.user.services.nix-snapshotter = lib.recursiveUpdate - (ns-lib.convertServiceToNixOS (ns-lib.mkRootlessNixSnapshotterService cfg)) - { inherit (cfg) path; }; - } - { - systemd.user.services.preload-containerd-images = lib.recursiveUpdate - (ns-lib.convertServiceToNixOS (ns-lib.mkRootlessPreloadContainerdImageService cfg)) - { environment.CONTAINERD_ADDRESS = "%t/containerd/containerd.sock"; }; - } - ]); + config = lib.mkIf cfg.enable { + systemd.user.services.nix-snapshotter = lib.mkMerge [ + (ns-lib.convertServiceToNixOS (ns-lib.mkRootlessNixSnapshotterService cfg)) + { inherit (cfg) path; } + ]; + }; } diff --git a/modules/nixos/nix-snapshotter.nix b/modules/nixos/nix-snapshotter.nix index 492d25b..33e883b 100644 --- a/modules/nixos/nix-snapshotter.nix +++ b/modules/nixos/nix-snapshotter.nix @@ -2,74 +2,46 @@ let inherit (lib) mkEnableOption - mkOption - types ; cfg = config.services.nix-snapshotter; ns-lib = cfg.lib; + settingsFormat = pkgs.formats.toml {}; + in { - imports = [ ../common/nix-snapshotter-lib.nix ]; + imports = [ + ../common/nix-snapshotter.nix + ./k3s.nix + ]; options.services.nix-snapshotter = { inherit (ns-lib.options) configFile package path - preloadContainerdImages - setContainerdNamespace - setContainerdSnapshotter settings ; enable = mkEnableOption "nix-snapshotter"; }; - config = lib.mkIf cfg.enable (lib.mkMerge [ - { - services.nix-snapshotter = { - configFile = - lib.mkOptionDefault - (ns-lib.settingsFormat.generate "config.toml" cfg.settings); - }; - - environment.extraInit = - (lib.optionalString cfg.setContainerdSnapshotter '' - if [ -z "$CONTAINERD_SNAPSHOTTER" ]; then - export CONTAINERD_SNAPSHOTTER="nix" - fi - '') + - (lib.optionalString (cfg.setContainerdNamespace != "default") '' - if [ -z "$CONTAINERD_NAMESPACE" ]; then - export CONTAINERD_NAMESPACE="${cfg.setContainerdNamespace}" - fi - ''); - - virtualisation.containerd = { - enable = true; - settings = ns-lib.mkContainerdSettings; - }; - - systemd.services.nix-snapshotter = lib.recursiveUpdate - (ns-lib.convertServiceToNixOS ns-lib.mkNixSnapshotterService) - { - inherit (cfg) path; - description = "nix-snapshotter - containerd snapshotter that understands nix store paths natively"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - partOf = [ "containerd.service" ]; - serviceConfig.ExecStart = "${cfg.package}/bin/nix-snapshotter --config ${cfg.configFile}"; - }; - } - (lib.mkIf (cfg.preloadContainerdImages != []) { - systemd.services.preload-containerd-images = lib.recursiveUpdate - (ns-lib.convertServiceToNixOS (ns-lib.mkPreloadContainerdImageService cfg)) - { - description = "Preload images to containerd"; - wantedBy = [ "multi-user.target" ]; - }; - }) - ]); + config = lib.mkIf cfg.enable { + services.nix-snapshotter.configFile = + lib.mkOptionDefault + (settingsFormat.generate "config.toml" cfg.settings); + + systemd.services.nix-snapshotter = lib.mkMerge [ + (ns-lib.convertServiceToNixOS ns-lib.mkNixSnapshotterService) + { + inherit (cfg) path; + description = "nix-snapshotter - containerd snapshotter that understands nix store paths natively"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + partOf = [ "containerd.service" ]; + serviceConfig.ExecStart = "${cfg.package}/bin/nix-snapshotter --config ${cfg.configFile}"; + } + ]; + }; } diff --git a/modules/nixos/preload-containerd-rootless.nix b/modules/nixos/preload-containerd-rootless.nix new file mode 100644 index 0000000..811937a --- /dev/null +++ b/modules/nixos/preload-containerd-rootless.nix @@ -0,0 +1,20 @@ +{ config, lib, ... }: +let + cfg = config.services.preload-containerd.rootless; + + ns-lib = config.services.nix-snapshotter.lib; + + preload-lib = config.services.preload-containerd.lib; + +in { + imports = [ + ../common/preload-containerd-rootless.nix + ]; + + config = lib.mkIf cfg.enable { + systemd.user.services.preload-containerd = + lib.mkIf + (cfg.targets != []) + (ns-lib.convertServiceToNixOS (preload-lib.mkRootlessPreloadContainerdService cfg)); + }; +} diff --git a/modules/nixos/preload-containerd.nix b/modules/nixos/preload-containerd.nix new file mode 100644 index 0000000..49f8aa8 --- /dev/null +++ b/modules/nixos/preload-containerd.nix @@ -0,0 +1,24 @@ +{ config, lib, ... }: +let + cfg = config.services.preload-containerd; + + ns-lib = config.services.nix-snapshotter.lib; + +in { + imports = [ + ../common/preload-containerd.nix + ]; + + config = lib.mkIf cfg.enable { + systemd.services.preload-containerd = + lib.mkIf + (cfg.targets != []) + (lib.mkMerge [ + (ns-lib.convertServiceToNixOS (cfg.lib.mkPreloadContainerdService cfg)) + { + description = "Preload images to containerd"; + wantedBy = [ "multi-user.target" ]; + } + ]); + }; +} diff --git a/modules/nixos/redis-spec.nix b/modules/nixos/redis-spec.nix index 8275c62..6b049d5 100644 --- a/modules/nixos/redis-spec.nix +++ b/modules/nixos/redis-spec.nix @@ -1,48 +1,13 @@ -{ pkgs, ... }: -let - redis = pkgs.nix-snapshotter.buildImage { - name = "redis"; - resolvedByNix = true; - config = { - entrypoint = [ "${pkgs.redis}/bin/redis-server" ]; - }; - }; - - redisPod = pkgs.writeText "redis-pod.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Pod"; - metadata = { - name = "redis"; - labels.name = "redis"; - }; - spec.containers = [{ - inherit (redis) name image; - args = ["--protected-mode" "no"]; - ports = [{ - name = "client"; - containerPort = 6379; - }]; - }]; - }); - - redisService = pkgs.writeText "redis-service.json" (builtins.toJSON { - apiVersion = "v1"; - kind = "Service"; - metadata.name = "redis-service"; - spec = { - type = "NodePort"; - selector.name = "redis"; - ports = [{ - name = "client"; - port = 6379; - nodePort = 30000; - }]; - }; - }); - -in { +{ pkgs, k8sResources, ... }: { # Provide an example kubernetes config for redis using a nix-snapshotter - # image. - environment.etc."kubernetes/redis/pod.json".source = redisPod; - environment.etc."kubernetes/redis/service.json".source = redisService; + # image to `kubectl apply -f /etc/kubernetes/redis/`. + environment.etc."kubernetes/redis/pod.json".source = + pkgs.writeText + "redis-pod.json" + (builtins.toJSON k8sResources.redisPod); + + environment.etc."kubernetes/redis/service.json".source = + pkgs.writeText + "redis-service.json" + (builtins.toJSON k8sResources.redisService); } diff --git a/modules/nixos/tests/k3s-external.nix b/modules/nixos/tests/k3s-external.nix new file mode 100644 index 0000000..d311a9d --- /dev/null +++ b/modules/nixos/tests/k3s-external.nix @@ -0,0 +1,54 @@ +/* + k3s-external configures k3s to use an external containerd instead of its + embedded containerd. + + This is more flexible as users can leverage the full set of options for the + containerd & nix-snapshotter modules, whereas configuring them for the + embedded containerd is less user friendly. In addition, each service will + be its independent systemd unit. + + Note that rootless k3s cannot use an external containerd because doesn't + provide a way to provision additional processes inside the namespaces + managed by rootlesskit. + +*/ +{ pkgs, ... }: +{ + nodes.machine = { + imports = [ + ../k3s.nix + ../containerd.nix + ../nix-snapshotter.nix + ../redis-spec.nix + ]; + + services.k3s = { + enable = true; + setKubeConfig = true; + }; + + virtualisation.containerd = { + enable = true; + k3sIntegration = true; + nixSnapshotterIntegration = true; + }; + + services.nix-snapshotter = { + enable = true; + }; + + environment.systemPackages = with pkgs; [ + redis + ]; + }; + + testScript = '' + start_all() + + machine.wait_until_succeeds("kubectl get node $(hostname) | grep -w Ready") + machine.wait_until_succeeds("kubectl apply -f /etc/kubernetes/redis/") + machine.wait_until_succeeds("kubectl get pod redis | grep Running") + out = machine.wait_until_succeeds("redis-cli -p 30000 ping") + assert "PONG" in out + ''; +} diff --git a/modules/nixos/tests/k3s-rootless.nix b/modules/nixos/tests/k3s-rootless.nix new file mode 100644 index 0000000..398eb47 --- /dev/null +++ b/modules/nixos/tests/k3s-rootless.nix @@ -0,0 +1,51 @@ +/* + k3s-rootless configures k3s in rootless mode to use its embedded containerd + with nix-snapshotter support. + + Note this is the only possible configuration for rootless mode. See + tests/k3s.nix for more details. + +*/ +{ pkgs, lib, ... }: +{ + nodes.machine = { + imports = [ + ../k3s-rootless.nix + ../redis-spec.nix + ]; + + services.k3s.rootless = { + enable = true; + snapshotter = "nix"; + }; + + environment.systemPackages = with pkgs; [ + kubectl + redis + ]; + + users.users.alice = { + uid = 1000; + isNormalUser = true; + }; + }; + + testScript = { nodes, ... }: + let + sudo_su = lib.concatStringsSep " " [ + "KUBECONFIG=/home/alice/.kube/k3s.yaml" + "sudo" "--preserve-env=KUBECONFIG" "-u" "alice" + ]; + + in '' + start_all() + + machine.succeed("loginctl enable-linger alice") + + machine.wait_until_succeeds("${sudo_su} kubectl get node $(hostname) | grep -w Ready") + machine.wait_until_succeeds("${sudo_su} kubectl apply -f /etc/kubernetes/redis/") + machine.wait_until_succeeds("${sudo_su} kubectl get pod redis | grep Running") + out = machine.wait_until_succeeds("${sudo_su} redis-cli -p 30000 ping") + assert "PONG" in out + ''; +} diff --git a/modules/nixos/tests/k3s.nix b/modules/nixos/tests/k3s.nix new file mode 100644 index 0000000..4311365 --- /dev/null +++ b/modules/nixos/tests/k3s.nix @@ -0,0 +1,37 @@ +/* + k3s configures k3s to use its embedded containerd with nix-snapshotter + support. + + This is the simplest configuration as it's a single systemd unit. However + less flexible than the setup in tests/k3s-external.nix. + +*/ +{ pkgs, ... }: +{ + nodes.machine = { + imports = [ + ../k3s.nix + ../redis-spec.nix + ]; + + services.k3s = { + enable = true; + setKubeConfig = true; + snapshotter = "nix"; + }; + + environment.systemPackages = with pkgs; [ + redis + ]; + }; + + testScript = '' + start_all() + + machine.wait_until_succeeds("kubectl get node $(hostname) | grep -w Ready") + machine.wait_until_succeeds("kubectl apply -f /etc/kubernetes/redis/") + machine.wait_until_succeeds("kubectl get pod redis | grep Running") + out = machine.wait_until_succeeds("redis-cli -p 30000 ping") + assert "PONG" in out + ''; +} diff --git a/modules/nixos/tests/kubernetes.nix b/modules/nixos/tests/kubernetes.nix index 6c4e739..d07e799 100644 --- a/modules/nixos/tests/kubernetes.nix +++ b/modules/nixos/tests/kubernetes.nix @@ -1,64 +1,73 @@ -{ pkgs, ... }: +/* + kubernetes configures Kubernetes with containerd & nix-snapshotter. + +*/ +{ lib, pkgs, ... }: { - nodes = { - kubernetes = { config, ... }: - let - cfg = config.services.kubernetes; + nodes.machine = { config, k8sResources, ... }: + let + # Only k3s has a builtin `LoadBalancer` so the redisService needs to be + # updated to use `NodePort`. + redisService = + lib.recursiveUpdate + k8sResources.redisService + { + spec = { + type = "NodePort"; + ports = [{ + name = "client"; + port = 6379; + nodePort = 30000; + }]; + }; + }; - wrapKubectl = pkgs.runCommand "wrap-kubectl" { - nativeBuildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out/bin - makeWrapper ${pkgs.kubernetes}/bin/kubectl \ - $out/bin/kubectl \ - --set KUBECONFIG "/etc/${cfg.pki.etcClusterAdminKubeconfig}" - ''; + in { + imports = [ + ../containerd.nix + ../nix-snapshotter.nix + ]; - in { - imports = [ - ../nix-snapshotter.nix - ../redis-spec.nix - ]; + virtualisation.containerd = { + enable = true; + nixSnapshotterIntegration = true; + }; - services.nix-snapshotter.enable = true; + services.nix-snapshotter.enable = true; - services.kubernetes = { - roles = ["master" "node"]; - masterAddress = "localhost"; - kubelet.extraOpts = "--image-service-endpoint unix:///run/nix-snapshotter/nix-snapshotter.sock"; - }; + services.kubernetes = { + roles = ["master" "node"]; + masterAddress = "localhost"; + kubelet.extraOpts = "--image-service-endpoint unix:///run/nix-snapshotter/nix-snapshotter.sock"; + }; + + environment.systemPackages = with pkgs; [ + redis + kubectl + ]; - environment.systemPackages = [ - pkgs.redis - wrapKubectl - ]; + environment.sessionVariables = { + KUBECONFIG = "/etc/${config.services.kubernetes.pki.etcClusterAdminKubeconfig}"; }; - k3s = { ... }: { - imports = [ - ../k3s.nix - ../redis-spec.nix - ]; + environment.etc."kubernetes/redis/pod.json".source = + pkgs.writeText + "redis-pod.json" + (builtins.toJSON k8sResources.redisPod); - environment.systemPackages = [ - pkgs.redis - ]; + environment.etc."kubernetes/redis/service.json".source = + pkgs.writeText + "redis-service.json" + (builtins.toJSON redisService); }; - }; testScript = '' start_all() - def test_redis_service(machine): - machine.wait_until_succeeds("kubectl get node $(hostname) | grep -w Ready") - - machine.wait_until_succeeds("kubectl apply -f /etc/kubernetes/redis/") - - machine.wait_until_succeeds("kubectl get pod redis | grep Running") - out = machine.wait_until_succeeds("redis-cli -p 30000 ping") - assert "PONG" in out - - # test_redis_service(kubernetes) - test_redis_service(k3s) + machine.wait_until_succeeds("kubectl get node $(hostname) | grep -w Ready") + machine.wait_until_succeeds("kubectl apply -f /etc/kubernetes/redis/") + machine.wait_until_succeeds("kubectl get pod redis | grep Running") + out = machine.wait_until_succeeds("redis-cli -p 30000 ping") + assert "PONG" in out ''; } diff --git a/modules/nixos/tests/snapshotter.nix b/modules/nixos/tests/snapshotter.nix index 178da53..bd6320e 100644 --- a/modules/nixos/tests/snapshotter.nix +++ b/modules/nixos/tests/snapshotter.nix @@ -27,7 +27,6 @@ let }; in { - # Setup local registry for testing `buildImage` and `copyToRegistry`. services.dockerRegistry = { enable = true; @@ -48,25 +47,34 @@ let rootful = { imports = [ base + ../containerd.nix ../nix-snapshotter.nix ]; + virtualisation.containerd = { + enable = true; + nixSnapshotterIntegration = true; + }; + services.nix-snapshotter = { enable = true; - setContainerdSnapshotter = true; }; }; rootless = { imports = [ base - ../nix-snapshotter.nix + ../containerd-rootless.nix ../nix-snapshotter-rootless.nix ]; + virtualisation.containerd.rootless = { + enable = true; + nixSnapshotterIntegration = true; + }; + services.nix-snapshotter.rootless = { enable = true; - setContainerdSnapshotter = true; }; users.users.alice = { @@ -75,17 +83,12 @@ let }; }; - both = { imports = [ rootful rootless ]; }; - external = { imports = [ - base - ../nix-snapshotter.nix + rootful ]; - services.nix-snapshotter = { - enable = true; - setContainerdSnapshotter = true; + virtualisation.containerd = { settings.external_builder = pkgs.writeScript "external-builder.sh" '' ${pkgs.nix}/bin/nix build --out-link $1 $2 ''; @@ -97,7 +100,6 @@ in { inherit rootful rootless - both external ; }; @@ -175,10 +177,6 @@ in { setup(rootless) test_rootless(rootless) - setup(both) - test_rootful(both, "both") - test_rootless(both, "both") - setup(external) test_rootful(external) ''; diff --git a/modules/nixos/vm-common.nix b/modules/nixos/vm-common.nix new file mode 100644 index 0000000..dd71a1d --- /dev/null +++ b/modules/nixos/vm-common.nix @@ -0,0 +1,39 @@ +{ pkgs, modulesPath, ... }: +{ + imports = [ + # Import qemu-vm directly to avoid using vmVariant since this config + # is only intended to be used as a VM. Using vmVariant will emit assertion + # errors regarding `fileSystems."/"` and `boot.loader.grub.device`. + (modulesPath + "/virtualisation/qemu-vm.nix") + ./redis-spec.nix + ]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + + environment.systemPackages = with pkgs; [ + bat + containerd + cri-tools + git + jq + kubectl + nerdctl + nix-snapshotter + redis + tree + vim + ]; + + virtualisation = { + memorySize = 4096; + cores = 4; + graphics = false; + diskImage = null; + }; + + services.openssh.enable = true; + + networking.firewall.allowedTCPPorts = [ 22 ]; + + system.stateVersion = "23.05"; +} diff --git a/modules/nixos/vm-rootless.nix b/modules/nixos/vm-rootless.nix new file mode 100644 index 0000000..d1c0ce2 --- /dev/null +++ b/modules/nixos/vm-rootless.nix @@ -0,0 +1,34 @@ +{ lib, examples, ... }: +{ + imports = [ + ./vm-common.nix + ]; + + # ROOTLESS + ########## + + users.users.rootless = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + password = "rootless"; + group = "rootless"; + }; + users.groups.rootless = {}; + + # Standalone k3s with embedded containerd and nix-snapshotter. + services.k3s.rootless = { + enable = true; + setKubeConfig = true; + setEmbeddedContainerd = true; + snapshotter = "nix"; + }; + + services.preload-containerd.rootless = { + enable = true; + targets = [{ + archives = lib.attrValues examples; + address = "$XDG_RUNTIME_DIR/k3s/containerd/containerd.sock"; + namespace = "k8s.io"; + }]; + }; +} diff --git a/modules/nixos/vm.nix b/modules/nixos/vm.nix index d7fc9b0..465b632 100644 --- a/modules/nixos/vm.nix +++ b/modules/nixos/vm.nix @@ -1,71 +1,34 @@ -{ lib, pkgs, modulesPath, examples, ... }: -let - preloadContainerdImages = lib.attrValues examples; - -in { +{ lib, examples, ... }: +{ imports = [ - # Import qemu-vm directly to avoid using vmVariant since this config - # is only intended to be used as a VM. Using vmVariant will emit assertion - # errors regarding `fileSystems."/"` and `boot.loader.grub.device`. - (modulesPath + "/virtualisation/qemu-vm.nix") - ./k3s.nix - ./redis-spec.nix + ./vm-common.nix ]; - # Enable rootful & rootless nix-snapshotter. This also starts rootful & - # rootless containerd respectively. - services.nix-snapshotter = { - enable = true; - setContainerdSnapshotter = true; - inherit preloadContainerdImages; + # ROOTFUL + ######### + + users.users.root = { + initialHashedPassword = null; + password = "root"; }; - services.nix-snapshotter.rootless = { + services.k3s = { enable = true; - inherit preloadContainerdImages; + setKubeConfig = true; }; - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - - environment.systemPackages = with pkgs; [ - bat - containerd - cri-tools - git - jq - kubectl - nerdctl - nix-snapshotter - redis - tree - vim - ]; - - users.users = { - root = { - initialHashedPassword = null; - password = "root"; - }; - rootless = { - isNormalUser = true; - extraGroups = [ "wheel" ]; - password = "rootless"; - group = "rootless"; - }; + virtualisation.containerd = { + enable = true; + k3sIntegration = true; + nixSnapshotterIntegration = true; }; - users.groups.rootless = {}; - - virtualisation = { - memorySize = 4096; - cores = 4; - graphics = false; - diskImage = null; + services.nix-snapshotter = { + enable = true; }; - services.openssh.enable = true; - - networking.firewall.allowedTCPPorts = [ 22 ]; - - system.stateVersion = "23.05"; + services.preload-containerd = { + enable = true; + targets = [{ archives = lib.attrValues examples; }]; + }; } diff --git a/package.nix b/package.nix index b8bc48c..eea3463 100644 --- a/package.nix +++ b/package.nix @@ -9,7 +9,7 @@ let nix-snapshotter = buildGoModule { pname = "nix-snapshotter"; - version = "0.1.2"; + version = "0.2.0"; src = lib.cleanSourceWith { src = lib.sourceFilesBySuffices ./. [ ".go" diff --git a/script/k8s/redis.yaml b/script/k8s/redis.yaml deleted file mode 100644 index 48e7418..0000000 --- a/script/k8s/redis.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: redis - labels: - app.kubernetes.io/name: redis -spec: - containers: - - name: redis - image: docker.io/hinshun/redis:nix - # image: ghcr.io/pdtpartners/redis - # imagePullPolicy: Never - args: [ "--protected-mode", "no" ] - ports: - - containerPort: 6379 - name: client ---- -apiVersion: v1 -kind: Service -metadata: - name: redis-service -spec: - type: NodePort - selector: - app.kubernetes.io/name: redis - ports: - - name: client - port: 6379 - nodePort: 30000