nixos/iscsi-root-initiator: support systemd stage 1

nixos/doc/manual/release-notes/rl-2611.section.md

@@ -136,6 +136,8 @@
     and the default changed to a UNIX domain socket.
   - A cookie-cutter nginx vhost can be enabled at [](#opt-services.netbox.nginx.enable).
 
+- [boot.iscsi-initiator](#opt-boot.iscsi-initiator.name) now supports systemd stage 1 and no longer asserts against it. The iSCSI login runs as ordered initrd units instead of a script in `preLVMCommands`, so it no longer races DHCP. Initrd networking is left to [boot.initrd.network.enable](#opt-boot.initrd.network.enable) with networkd, configured from the `ip=` kernel parameter or the `networking.*` options. No options changed, and the scripted initrd path still works.
+
 - `security.run0.enableSudoAlias` now uses the `run0-sudo-shim` instead of a shell-script to improve compatibility.
 
 - With `system.etc.overlay.mutable = false`, NixOS now ships an empty `/etc/machine-id` in the image. Previously the file was absent and systemd logged `System cannot boot: Missing /etc/machine-id and /etc/ is read-only` while `ConditionFirstBoot` fired on every boot. With this change, systemd now overlays a transient ID from `/run/machine-id` for the session, and `systemd-machine-id-commit.service` has `ConditionFirstBoot` so it writes the machine-id through to a persistent backing file when one is bind-mounted over `/etc/machine-id`. To persist the machine-id across reboots, bind-mount a writable file containing `uninitialized` over `/etc/machine-id` from the initrd, or set `systemd.machine_id=` on the kernel command line (use `systemd.machine_id=firmware` to derive a stable ID on hardware that supports it).

nixos/modules/services/networking/iscsi/root-initiator.nix

@@ -7,6 +7,65 @@
 with lib;
 let
   cfg = config.boot.iscsi-initiator;
+
+  useSystemd = config.boot.initrd.systemd.enable;
+
+  # The static portion of the initrd iscsid.conf: the upstream default plus any
+  # `extraConfig`, and `node.startup = automatic` when logging into all targets.
+  # Built as a normal derivation to avoid import-from-derivation. Secrets from
+  # `extraConfigFile` are appended at runtime, not here.
+  iscsidConf = pkgs.runCommand "iscsid-initrd.conf" { } ''
+    cat ${pkgs.openiscsi}/etc/iscsi/iscsid.conf > "$out"
+    cat >> "$out" <<'EOF'
+
+    ${optionalString (cfg.extraConfig != null) cfg.extraConfig}
+    ${optionalString cfg.loginAll "node.startup = automatic"}
+    EOF
+  '';
+
+  # Assemble /etc/iscsi/{iscsid.conf,initiatorname.iscsi} in the (writable)
+  # initrd /etc before iscsid starts. Kept out of the login unit so it also runs
+  # for the systemd path's iscsid ExecStartPre.
+  prepConf = pkgs.writeShellScript "iscsi-initrd-prep" ''
+    set -eu
+    mkdir -p /etc/iscsi /run/lock/iscsi
+    cp ${iscsidConf} /etc/iscsi/iscsid.conf
+    chmod +w /etc/iscsi/iscsid.conf
+    echo "InitiatorName=${cfg.name}" > /etc/iscsi/initiatorname.iscsi
+    ${optionalString (cfg.extraConfigFile != null) ''
+      if [ -f "${cfg.extraConfigFile}" ]; then
+        printf '\n# The following is from ${cfg.extraConfigFile}:\n' >> /etc/iscsi/iscsid.conf
+        cat "${cfg.extraConfigFile}" >> /etc/iscsi/iscsid.conf
+      else
+        echo "Warning: boot.iscsi-initiator.extraConfigFile ${cfg.extraConfigFile} does not exist!" >&2
+      fi
+    ''}
+  '';
+
+  # Discover the portal and log into the root target. Wrapped in a retry because
+  # iscsid may not have opened its socket the instant this unit starts.
+  loginScript = pkgs.writeShellScript "iscsi-root-login" ''
+    set -eu
+    ${pkgs.openiscsi}/bin/iscsiadm -m discovery -t sendtargets \
+      -p ${escapeShellArg cfg.discoverPortal} --debug ${toString cfg.logLevel} || true
+    n=0
+    until ${
+      if cfg.loginAll then
+        "${pkgs.openiscsi}/bin/iscsiadm -m node --loginall all"
+      else
+        "${pkgs.openiscsi}/bin/iscsiadm -m node -T ${escapeShellArg cfg.target} -p ${escapeShellArg cfg.discoverPortal} --login"
+    }; do
+      n=$((n + 1))
+      if [ "$n" -ge 30 ]; then
+        echo "iscsi-root-login: giving up after $n attempts" >&2
+        exit 1
+      fi
+      sleep 1
+      ${pkgs.openiscsi}/bin/iscsiadm -m discovery -t sendtargets \
+        -p ${escapeShellArg cfg.discoverPortal} --debug ${toString cfg.logLevel} || true
+    done
+    ${cfg.extraIscsiCommands}
+  '';
 in
 {
   # If you're booting entirely off another machine you may want to add
@@ -106,104 +165,157 @@ in
     networking.useNetworkd = true;
     networking.useDHCP = false; # Required to set useNetworkd = true
 
-    boot.initrd = {
-      network.enable = true;
-
-      # By default, the stage-1 disables the network and resets the interfaces
-      # on startup. Since our startup disks are on the network, we can't let
-      # the network not work.
-      network.flushBeforeStage2 = false;
-
-      kernelModules = [ "iscsi_tcp" ];
-
-      extraUtilsCommands = ''
-        copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsid
-        copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsiadm
-        ${optionalString (
-          !config.boot.initrd.network.ssh.enable
-        ) "cp -pv ${pkgs.glibc.out}/lib/libnss_files.so.* $out/lib"}
-
-        mkdir -p $out/etc/iscsi
-        cp ${config.environment.etc.hosts.source} $out/etc/hosts
-        cp ${pkgs.openiscsi}/etc/iscsi/iscsid.conf $out/etc/iscsi/iscsid.fragment.conf
-        chmod +w $out/etc/iscsi/iscsid.fragment.conf
-        cat << 'EOF' >> $out/etc/iscsi/iscsid.fragment.conf
-        ${optionalString (cfg.extraConfig != null) cfg.extraConfig}
-        EOF
-      '';
-
-      extraUtilsCommandsTest = ''
-        $out/bin/iscsiadm --version
-      '';
-
-      preLVMCommands =
-        let
-          extraCfgDumper = optionalString (cfg.extraConfigFile != null) ''
-            if [ -f "${cfg.extraConfigFile}" ]; then
-              printf "\n# The following is from ${cfg.extraConfigFile}:\n"
-              cat "${cfg.extraConfigFile}"
-            else
-              echo "Warning: boot.iscsi-initiator.extraConfigFile ${cfg.extraConfigFile} does not exist!" >&2
-            fi
-          '';
-        in
-        ''
-          ${optionalString (!config.boot.initrd.network.ssh.enable) ''
-            # stolen from initrd-ssh.nix
-            echo 'root:x:0:0:root:/root:/bin/ash' > /etc/passwd
-            echo 'passwd: files' > /etc/nsswitch.conf
-          ''}
-
-          cp -f $extraUtils/etc/hosts /etc/hosts
-
-          mkdir -p /etc/iscsi /run/lock/iscsi
-          echo "InitiatorName=${cfg.name}" > /etc/iscsi/initiatorname.iscsi
-
-          (
-            cat "$extraUtils/etc/iscsi/iscsid.fragment.conf"
-            printf "\n"
-            ${optionalString cfg.loginAll ''echo "node.startup = automatic"''}
-            ${extraCfgDumper}
-          ) > /etc/iscsi/iscsid.conf
-
-          iscsid --foreground --no-pid-file --debug ${toString cfg.logLevel} &
-          iscsiadm --mode discoverydb \
-            --type sendtargets \
-            --discover \
-            --portal ${escapeShellArg cfg.discoverPortal} \
-            --debug ${toString cfg.logLevel}
-
-          ${
-            if cfg.loginAll then
-              ''
-                iscsiadm --mode node --loginall all
-              ''
-            else
-              ''
-                iscsiadm --mode node --targetname ${escapeShellArg cfg.target} --login
-              ''
-          }
-
-          ${cfg.extraIscsiCommands}
-
-          pkill -9 iscsid
-        '';
-    };
-
     services.openiscsi = {
       enable = true;
       inherit (cfg) name;
     };
 
+    boot.initrd = mkMerge [
+      {
+        network.enable = true;
+        # By default, the stage-1 disables the network and resets the interfaces
+        # on startup. Since our startup disks are on the network, we can't let
+        # the network not work. (systemd stage 1 already defaults this to false.)
+        network.flushBeforeStage2 = false;
+
+        kernelModules = [ "iscsi_tcp" ];
+      }
+
+      # Scripted (non-systemd) stage 1: bring iscsid up, discover and log in
+      # from preLVMCommands, exactly as this module always has.
+      (mkIf (!useSystemd) {
+        extraUtilsCommands = ''
+          copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsid
+          copy_bin_and_libs ${pkgs.openiscsi}/bin/iscsiadm
+          ${optionalString (
+            !config.boot.initrd.network.ssh.enable
+          ) "cp -pv ${pkgs.glibc.out}/lib/libnss_files.so.* $out/lib"}
+
+          mkdir -p $out/etc/iscsi
+          cp ${config.environment.etc.hosts.source} $out/etc/hosts
+          cp ${pkgs.openiscsi}/etc/iscsi/iscsid.conf $out/etc/iscsi/iscsid.fragment.conf
+          chmod +w $out/etc/iscsi/iscsid.fragment.conf
+          cat << 'EOF' >> $out/etc/iscsi/iscsid.fragment.conf
+          ${optionalString (cfg.extraConfig != null) cfg.extraConfig}
+          EOF
+        '';
+
+        extraUtilsCommandsTest = ''
+          $out/bin/iscsiadm --version
+        '';
+
+        preLVMCommands =
+          let
+            extraCfgDumper = optionalString (cfg.extraConfigFile != null) ''
+              if [ -f "${cfg.extraConfigFile}" ]; then
+                printf "\n# The following is from ${cfg.extraConfigFile}:\n"
+                cat "${cfg.extraConfigFile}"
+              else
+                echo "Warning: boot.iscsi-initiator.extraConfigFile ${cfg.extraConfigFile} does not exist!" >&2
+              fi
+            '';
+          in
+          ''
+            ${optionalString (!config.boot.initrd.network.ssh.enable) ''
+              # stolen from initrd-ssh.nix
+              echo 'root:x:0:0:root:/root:/bin/ash' > /etc/passwd
+              echo 'passwd: files' > /etc/nsswitch.conf
+            ''}
+
+            cp -f $extraUtils/etc/hosts /etc/hosts
+
+            mkdir -p /etc/iscsi /run/lock/iscsi
+            echo "InitiatorName=${cfg.name}" > /etc/iscsi/initiatorname.iscsi
+
+            (
+              cat "$extraUtils/etc/iscsi/iscsid.fragment.conf"
+              printf "\n"
+              ${optionalString cfg.loginAll ''echo "node.startup = automatic"''}
+              ${extraCfgDumper}
+            ) > /etc/iscsi/iscsid.conf
+
+            iscsid --foreground --no-pid-file --debug ${toString cfg.logLevel} &
+            iscsiadm --mode discoverydb \
+              --type sendtargets \
+              --discover \
+              --portal ${escapeShellArg cfg.discoverPortal} \
+              --debug ${toString cfg.logLevel}
+
+            ${
+              if cfg.loginAll then
+                ''
+                  iscsiadm --mode node --loginall all
+                ''
+              else
+                ''
+                  iscsiadm --mode node --targetname ${escapeShellArg cfg.target} --login
+                ''
+            }
+
+            ${cfg.extraIscsiCommands}
+
+            pkill -9 iscsid
+          '';
+      })
+
+      # systemd stage 1: run iscsid and the login as real units, ordered on the
+      # dependency graph (network-online.target -> iscsid -> iscsi-login ->
+      # initrd-root-device.target -> sysroot.mount) instead of a one-shot script.
+      # Networking itself is provided by boot.initrd.network.enable + networkd
+      # (configured from the `ip=` kernel parameter or the networking.* options),
+      # so this module only owns the iSCSI session.
+      (mkIf useSystemd {
+        # makeInitrdNG only follows ELF dependencies, so every store path the
+        # scripts below reference has to be listed explicitly.
+        systemd.storePaths = [
+          pkgs.openiscsi
+          iscsidConf
+          prepConf
+          loginScript
+        ];
+
+        systemd.services.iscsid = {
+          description = "Open-iSCSI daemon (initrd)";
+          wantedBy = [ "initrd.target" ];
+          wants = [ "network-online.target" ];
+          after = [ "network-online.target" ];
+          before = [ "iscsi-login.service" ];
+          unitConfig.DefaultDependencies = false;
+          serviceConfig = {
+            Type = "simple";
+            ExecStartPre = prepConf;
+            ExecStart = "${pkgs.openiscsi}/bin/iscsid --foreground --no-pid-file --debug ${toString cfg.logLevel}";
+          };
+        };
+
+        systemd.services.iscsi-login = {
+          description = "Discover and log into the iSCSI root target (initrd)";
+          wantedBy = [ "initrd-root-device.target" ];
+          requires = [ "iscsid.service" ];
+          wants = [ "network-online.target" ];
+          after = [
+            "iscsid.service"
+            "network-online.target"
+          ];
+          before = [
+            "initrd-root-device.target"
+            "sysroot.mount"
+          ];
+          unitConfig.DefaultDependencies = false;
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStart = loginScript;
+          };
+        };
+      })
+    ];
+
     assertions = [
       {
         assertion = cfg.loginAll -> cfg.target == null;
         message = "iSCSI target name is set while login on all portals is enabled.";
       }
-      {
-        assertion = !config.boot.initrd.systemd.enable;
-        message = "systemd stage 1 does not support iscsi yet.";
-      }
     ];
   };
 }

nixos/tests/all-tests.nix

@@ -857,6 +857,7 @@ in
   irqbalance = runTest ./irqbalance.nix;
   iscsi-multipath-root = runTest ./iscsi-multipath-root.nix;
   iscsi-root = runTest ./iscsi-root.nix;
+  iscsi-root-systemd = runTest ./iscsi-root-systemd.nix;
   isolate = runTest ./isolate.nix;
   isso = runTest ./isso.nix;
   jackett = runTest ./jackett.nix;