Collect test coverage for both unit & NixOS integration tests

pdtpartners · Feb 18, 2024 · f4ae2be · f4ae2be
1 parent bf34e91
commit f4ae2be
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 25 deletions.
diff --git a/.github/actions/setup-nix/action.yml b/.github/actions/setup-nix/action.yml
@@ -13,7 +13,7 @@ runs:
         echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
         sudo udevadm control --reload-rules
         sudo udevadm trigger --name-match=kvm
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install Nix
       uses: DeterminateSystems/nix-installer-action@main
       with:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
         linter:
           - golangci-lint
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix
         uses: ./.github/actions/setup-nix
       - name: Run linter
@@ -30,13 +30,14 @@ jobs:
           - image-redis
           - image-redisWithShell
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix
         uses: ./.github/actions/setup-nix
       - name: Build package
         run: nix build .#${{ matrix.package }}
 
   integration-test:
+    runs-on: nix-snapshotter-runner
     strategy:
       matrix:
         test:
@@ -45,11 +46,10 @@ jobs:
           - k3s
           - k3s-external
           - k3s-rootless
-    runs-on: nix-snapshotter-runner
     needs: [lint, build]
     if: contains(github.event.pull_request.labels.*.name, 'ok-to-test')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup Nix
         uses: ./.github/actions/setup-nix
         with:
@@ -58,3 +58,30 @@ jobs:
         run: nix build
       - name: Run integration tests
         run: nix run -L .#test-${{ matrix.test }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.test }}_build
+          path: build
+          if-no-files-found: ignore
+          retention-days: 1
+
+  coverage-report:
+    runs-on: ubuntu-latest
+    needs: [integration-test]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          path: build
+          merge-multiple: true
+      - uses: actions/setup-go@v4
+        with: { go-version: '1.20' }
+      - name: Collect test coverage
+        run: |
+          mkdir -p "$PWD/build/go-cover/unit"
+          go test -cover ./... -args -test.gocoverdir="$PWD/build/go-cover/unit"
+          go tool covdata textfmt -i=`find ./build/go-cover -mindepth 1 -type d | paste -sd ","` -o ./build/go-cover/profile
+          echo "# Coverage report" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          go tool cover -func ./build/go-cover/profile >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
diff --git a/modules/flake/nixosTests.nix b/modules/flake/nixosTests.nix
@@ -1,4 +1,4 @@
-{ lib, flake-parts-lib, ... }:
+{ self, lib, flake-parts-lib, ... }:
 let
   inherit (lib)
     mkOption
@@ -21,17 +21,51 @@ in {
 
   config.perSystem = { config, pkgs, k8sResources, ... }:
     let
+      pkgs' = pkgs.extend(self: super: {
+        nix-snapshotter = super.nix-snapshotter.overrideAttrs(o: {
+          # Build nix-snapshotter as a cover-instrumented binary.
+          # See: https://go.dev/doc/build-cover
+          preBuild = (o.preBuild or "") + ''
+            buildFlagsArray+=(-cover)
+          '';
+        });
+      });
+
+      defaults = { config, ... }:
+        let
+          collectCoverage = {
+            preStart = "mkdir -p $GOCOVERDIR";
+            environment = { GOCOVERDIR = "/tmp/go-cover"; };
+          };
+
+        in { 
+          imports = [
+            self.nixosModules.default
+          ];
+
+          # Enable emitting coverage data for nix-snapshotter systemd units.
+          config = lib.mkMerge [
+            (lib.mkIf config.services.nix-snapshotter.enable {
+              systemd.services.nix-snapshotter = collectCoverage;
+            })
+            (lib.mkIf config.services.nix-snapshotter.rootless.enable {
+              systemd.user.services.nix-snapshotter = collectCoverage;
+            })
+          ];
+        };
+
       evalTest = name: module:
         (lib.nixos.evalTest {
           imports = [
             { inherit name; }
             module
           ];
-          hostPkgs = pkgs;
+          hostPkgs = pkgs';
           node = {
-            inherit pkgs;
+            pkgs = pkgs';
             specialArgs = { inherit k8sResources; };
           };
+          inherit defaults;
         }).config.result;
 
       testRigs = lib.mapAttrs (name: module: evalTest name module) config.nixosTests;

diff --git a/modules/nixos/tests/k3s-external.nix b/modules/nixos/tests/k3s-external.nix
@@ -12,13 +12,10 @@
   managed by rootlesskit.
 
 */
-{ pkgs, ... }:
+{ config, pkgs, ... }:
 {
   nodes.machine = {
     imports = [
-      ../k3s.nix
-      ../containerd.nix
-      ../nix-snapshotter.nix
       ../redis-spec.nix
     ];
 
@@ -50,5 +47,11 @@
     machine.wait_until_succeeds("kubectl get pod redis | grep Running")
     out = machine.wait_until_succeeds("redis-cli -p 30000 ping")
     assert "PONG" in out
+
+    # Copy out test coverage data.
+    machine.succeed("systemctl stop nix-snapshotter.service")
+    coverfiles = machine.succeed("ls /tmp/go-cover").split()
+    for coverfile in coverfiles:
+      machine.copy_from_vm(f"/tmp/go-cover/{coverfile}", f"build/go-cover/${config.name}-{machine.name}")
   '';
 }
diff --git a/modules/nixos/tests/k3s-rootless.nix b/modules/nixos/tests/k3s-rootless.nix
@@ -10,7 +10,6 @@
 {
   nodes.machine = {
     imports = [
-      ../k3s-rootless.nix
       ../redis-spec.nix
     ];
 

diff --git a/modules/nixos/tests/k3s.nix b/modules/nixos/tests/k3s.nix
@@ -10,7 +10,6 @@
 {
   nodes.machine = {
     imports = [
-      ../k3s.nix
       ../redis-spec.nix
     ];
 

diff --git a/modules/nixos/tests/kubernetes.nix b/modules/nixos/tests/kubernetes.nix
@@ -2,7 +2,7 @@
   kubernetes configures Kubernetes with containerd & nix-snapshotter.
 
 */
-{ lib, pkgs, ... }:
+{ config, lib, pkgs, ... }:
 {
   nodes.machine = { config, k8sResources, ... }:
     let
@@ -23,11 +23,6 @@
           };
 
     in {
-      imports = [
-        ../containerd.nix
-        ../nix-snapshotter.nix
-      ];
-
       virtualisation.containerd = {
         enable = true;
         nixSnapshotterIntegration = true;
@@ -69,5 +64,11 @@
     machine.wait_until_succeeds("kubectl get pod redis | grep Running")
     out = machine.wait_until_succeeds("redis-cli -p 30000 ping")
     assert "PONG" in out
+
+    # Copy out test coverage data.
+    machine.succeed("systemctl stop nix-snapshotter.service")
+    coverfiles = machine.succeed("ls /tmp/go-cover").split()
+    for coverfile in coverfiles:
+      machine.copy_from_vm(f"/tmp/go-cover/{coverfile}", f"build/go-cover/${config.name}-{machine.name}")
   '';
 }
diff --git a/modules/nixos/tests/snapshotter.nix b/modules/nixos/tests/snapshotter.nix
@@ -1,4 +1,4 @@
-{ pkgs, lib, ... }:
+{ config, pkgs, lib, ... }:
 let
   registryHost = "127.0.0.1";
 
@@ -47,8 +47,6 @@ let
   rootful = {
     imports = [
       base
-      ../containerd.nix
-      ../nix-snapshotter.nix
     ];
 
     virtualisation.containerd = {
@@ -64,8 +62,6 @@ let
   rootless = {
     imports = [
       base
-      ../containerd-rootless.nix
-      ../nix-snapshotter-rootless.nix
     ];
 
     virtualisation.containerd.rootless = {
@@ -119,6 +115,19 @@ in {
         machine.wait_for_open_port(${toString registryPort})
         machine.succeed("copy-to-registry")
 
+      def collect_coverage(machine):
+        coverfiles = machine.succeed("ls /tmp/go-cover").split()
+        for coverfile in coverfiles:
+          machine.copy_from_vm(f"/tmp/go-cover/{coverfile}", f"build/go-cover/${config.name}-{machine.name}")
+
+      def teardown_rootful(machine):
+        machine.succeed("systemctl stop nix-snapshotter.service")
+        collect_coverage(machine)
+
+      def teardown_rootless(machine, user = "alice"):
+        machine.succeed(f"systemctl --user --machine={user}@ stop nix-snapshotter.service")
+        collect_coverage(machine)
+
       def wait_for_user_unit(machine, service, user = "alice"):
         machine.wait_until_succeeds(f"systemctl --user --machine={user}@ is-active {service}")
 
@@ -173,11 +182,14 @@ in {
 
       setup(rootful)
       test_rootful(rootful)
+      teardown_rootful(rootful)
 
       setup(rootless)
       test_rootless(rootless)
+      teardown_rootless(rootless)
 
       setup(external)
       test_rootful(external)
+      teardown_rootful(external)
     '';
 }