node: runtime/trace until specified CNs reified

Author: nyiyui

cmd/runner-node/main.go

@@ -94,6 +94,7 @@ func main() {
 	util.ShowCurrent()
 	profile.Profile()
 
+	node.TraceInit()
 	var c config
 	configPath := os.Getenv("CONFIG_PATH")
 	data, err := os.ReadFile(configPath)

node/api.go

@@ -74,6 +74,7 @@ func (n *Node) setupClient(cl *rpc2.Client) {
 		}
 		util.Notify(fmt.Sprintf("STATUS=Synced to new CC (%d CNs)...", len(cc.Networks)))
 		util.Notify("READY=1")
+		n.traceCheck()
 		return nil
 	})
 	go cl.Run()

node/main.go

@@ -22,6 +22,7 @@ type NodeConfig struct {
 	EndpointOverride string
 }
 
+// There must be only one Node instance as a Node can trigger a trace to stop.
 func NewNode(cfg NodeConfig) (*Node, error) {
 	mh, err := newMio(cfg.MioAddr, cfg.MioToken)
 	if err != nil {

node/trace.go

@@ -0,0 +1,72 @@
+package node
+
+import (
+	"encoding/json"
+	"os"
+	"runtime/trace"
+	"sync"
+
+	"github.com/nyiyui/qrystal/util"
+)
+
+var traceUntil []string
+var traceCheckLock sync.Mutex
+var traceFile *os.File
+
+func TraceInit() {
+	outputPath := os.Getenv("QRYSTAL_TRACE_OUTPUT_PATH")
+	if outputPath == "" {
+		return
+	}
+	{
+		traceUntilCNs := os.Getenv("QRYSTAL_TRACE_UNTIL_CNS")
+		traceUntil = make([]string, 0)
+		err := json.Unmarshal([]byte(traceUntilCNs), &traceUntil)
+		if err != nil {
+			util.S.Errorf("trace: failed parse QRYSTAL_TRACE_UNTIL_CNS: %s", err)
+			return
+		}
+	}
+	var err error
+	traceFile, err = os.Create(outputPath)
+	if err != nil {
+		util.S.Errorf("trace: failed to create file %s: %s", outputPath, err)
+		return
+	}
+	err = trace.Start(traceFile)
+	if err != nil {
+		util.S.Errorf("trace: failed to start trace: %s", err)
+		return
+	}
+	util.S.Info("trace: ready.")
+}
+
+func (n *Node) traceCheck() {
+	if traceUntil == nil {
+		// trace not enabled
+		return
+	}
+	traceCheckLock.Lock()
+	defer traceCheckLock.Unlock()
+	matches := 0
+	for key := range n.cc.Networks {
+		for _, key2 := range traceUntil {
+			if key == key2 {
+				matches++
+			}
+		}
+	}
+	if matches != len(traceUntil) {
+		return
+	}
+	if !trace.IsEnabled() {
+		util.S.Info("trace: already stopped.")
+		return
+	}
+	trace.Stop()
+	err := traceFile.Close()
+	if err != nil {
+		util.S.Errorf("trace: failed to close file (while checking): %s", err)
+	}
+	util.S.Info("trace: stopped due to QRYSTAL_TRACE_UNTIL_CNS being satisfied")
+}

result

@@ -0,0 +1 @@
+/nix/store/76nxn1yk05p6307aafvfrjxizg2zim2a-vm-test-run-trace

test.nix

@@ -428,4 +428,75 @@ in {
           break
     '';
   });
+  trace = let
+    networkName = "testnet";
+    eoPath = pkgs.writeShellScript "eo.sh" ''
+      while IFS= read -r line
+      do
+        echo '{"endpoint":"1.2.3.4:5678"}'
+      done
+    '';
+    tracePath = "/etc/qrystal-trace";
+    node = { token }:
+      { pkgs, ... }: {
+        imports = [
+          base
+          self.outputs.nixosModules.${system}.node
+          ({ ... }: { 
+            environment.etc."qrystal-trace" = {
+              text = "not yet";
+              mode = "0666";
+            };
+            systemd.services.qrystal-node.environment = {
+              "QRYSTAL_TRACE_OUTPUT_PATH" = tracePath;
+              "QRYSTAL_TRACE_UNTIL_CNS" = builtins.toJSON [networkName];
+            };
+            environment.systemPackages = with pkgs; [ go ];
+          })
+        ];
+
+        networking.firewall.allowedTCPPorts = [ 39251 ];
+        qrystal.services.node = csConfig [ networkName ] token;
+      };
+  in lib.runTest ({
+    name = "trace";
+    hostPkgs = pkgs;
+    nodes = {
+      node1 = node { token = node1Token; };
+      cs = { pkgs, ... }: {
+        imports = [ base self.outputs.nixosModules.${system}.cs ];
+
+        networking.firewall.allowedTCPPorts = [ 39252 ];
+        qrystal.services.cs = {
+          enable = true;
+          config = {
+            tls = csTls;
+            tokens = [
+              (nodeToken "node1" node1Hash [ networkName ])
+            ];
+            central.networks.${networkName} = networkBase // {
+              peers.node1 = {
+                host = "node1:58120";
+                allowedIPs = [ "10.123.0.1/32" ];
+                canSee.only = [ "node1" ];
+              };
+            };
+          };
+        };
+      };
+    };
+    testScript = { nodes, ... }: ''
+      import time
+
+      cs.start()
+      cs.wait_for_unit("qrystal-cs.service")
+      node1.start()
+      node1.wait_for_unit("qrystal-node.service", timeout=20)
+      start = time.time()
+      node1.wait_until_succeeds("cat ${tracePath}", timeout=20)
+      # verify trace is valid (e.g. if it finished writing correctly)
+      node1.succeed("go tool trace -pprof=net ${tracePath}")
+      # pprof type doesn't matter
+    '';
+  });
 }

trace.md

@@ -0,0 +1,10 @@
+# Execution Trace (Node)
+
+Node can provide an execution trace (using [Go's `runtime/trace`](https://pkg.go.dev/runtime/trace)) from startup until a set of CNs are initially reified (i.e. the WireGuard networks for the CNs are configured and Hokuto DNS is configured for them, if applicable).
+
+## Configuration
+
+Set the `QRYSTAL_TRACE_OUTPUT_PATH` envvar for the output path of the trace. Note that the NixOS modules set `PrivateTmp=yes` on the systemd unit (e.g. if you set `QRYSTAL_TRACE_OUTPUT_PATH=/tmp/qrystal-trace`, then the trace would be in something like `/etc/systemd-private-abc-qrystal-node.service-def/tmp/qrystal-trace`).
+
+Set the `QRYSTAL_TRACE_UNTIL_CNS` envvar to a JSON list of the CNs. Once all of the CNs specified in here are reified, the trace will be stopped (and saved).
+Example: `["examplenet","othernet"]`