From 4c6f58366e61e405e64d54283fe5c959accebfc5 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 25 Feb 2025 15:53:10 -0600
Subject: [PATCH 01/37] nixosNodeSvc: apply with builtins, lib

---
 nix/nixos/cardano-node-service.nix | 76 +++++++++++++++---------------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index 21e19a2a6c1..065aa9ca7a3 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -7,20 +7,20 @@ with lib; with builtins;
 let
   cfg = config.services.cardano-node;
   envConfig = cfg.environments.${cfg.environment};
-  runtimeDir = i : if cfg.runtimeDir i == null then cfg.stateDir i else "${cfg.runDirBase}${lib.removePrefix cfg.runDirBase (cfg.runtimeDir i)}";
+  runtimeDir = i : if cfg.runtimeDir i == null then cfg.stateDir i else "${cfg.runDirBase}${removePrefix cfg.runDirBase (cfg.runtimeDir i)}";
   suffixDir = base: i: "${base}${optionalString (i != 0) "-${toString i}"}";
   nullOrStr = types.nullOr types.str;
   funcToOr = t: types.either t (types.functionTo t);
 
   newTopology = i: {
     localRoots = map (g: {
-      accessPoints = map (e: builtins.removeAttrs e ["valency"]) g.accessPoints;
+      accessPoints = map (e: removeAttrs e ["valency"]) g.accessPoints;
       advertise = g.advertise or false;
       valency = g.valency or (length g.accessPoints);
       trustable = g.trustable or false;
     }) (cfg.producers ++ (cfg.instanceProducers i));
     publicRoots = map (g: {
-      accessPoints = map (e: builtins.removeAttrs e ["valency"]) g.accessPoints;
+      accessPoints = map (e: removeAttrs e ["valency"]) g.accessPoints;
       advertise = g.advertise or false;
     }) (cfg.publicProducers ++ (cfg.instancePublicProducers i));
     bootstrapPeers = cfg.bootstrapPeers;
@@ -113,54 +113,54 @@ let
       else toFile "config-${toString cfg.nodeId}-${toString i}.json" (toJSON instanceConfig);
     consensusParams = {
       RealPBFT = [
-        "${lib.optionalString (cfg.signingKey != null)
+        "${optionalString (cfg.signingKey != null)
           "--signing-key ${cfg.signingKey}"}"
-        "${lib.optionalString (cfg.delegationCertificate != null)
+        "${optionalString (cfg.delegationCertificate != null)
           "--delegation-certificate ${cfg.delegationCertificate}"}"
       ];
       TPraos = [
-        "${lib.optionalString (cfg.vrfKey != null)
+        "${optionalString (cfg.vrfKey != null)
           "--shelley-vrf-key ${cfg.vrfKey}"}"
-        "${lib.optionalString (cfg.kesKey != null)
+        "${optionalString (cfg.kesKey != null)
           "--shelley-kes-key ${cfg.kesKey}"}"
-        "${lib.optionalString (cfg.operationalCertificate != null)
+        "${optionalString (cfg.operationalCertificate != null)
           "--shelley-operational-certificate ${cfg.operationalCertificate}"}"
       ];
       Cardano = [
-        "${lib.optionalString (cfg.signingKey != null)
+        "${optionalString (cfg.signingKey != null)
           "--signing-key ${cfg.signingKey}"}"
-        "${lib.optionalString (cfg.delegationCertificate != null)
+        "${optionalString (cfg.delegationCertificate != null)
           "--delegation-certificate ${cfg.delegationCertificate}"}"
-        "${lib.optionalString (cfg.vrfKey != null)
+        "${optionalString (cfg.vrfKey != null)
           "--shelley-vrf-key ${cfg.vrfKey}"}"
-        "${lib.optionalString (cfg.kesKey != null)
+        "${optionalString (cfg.kesKey != null)
           "--shelley-kes-key ${cfg.kesKey}"}"
-        "${lib.optionalString (cfg.operationalCertificate != null)
+        "${optionalString (cfg.operationalCertificate != null)
           "--shelley-operational-certificate ${cfg.operationalCertificate}"}"
       ];
     };
     instanceDbPath = cfg.databasePath i;
-    cmd = builtins.filter (x: x != "") [
+    cmd = filter (x: x != "") [
       "${cfg.executable} run"
       "--config ${nodeConfigFile}"
       "--database-path ${instanceDbPath}"
       "--topology ${topology i}"
-    ] ++ lib.optionals (!cfg.systemdSocketActivation) ([
+    ] ++ optionals (!cfg.systemdSocketActivation) ([
       "--host-addr ${cfg.hostAddr}"
       "--port ${if (cfg.shareIpv4port || cfg.shareIpv6port) then toString cfg.port else toString (cfg.port + i)}"
       "--socket-path ${cfg.socketPath i}"
-    ] ++ lib.optionals (cfg.ipv6HostAddr i != null) [
+    ] ++ optionals (cfg.ipv6HostAddr i != null) [
       "--host-ipv6-addr ${cfg.ipv6HostAddr i}"
-    ]) ++ lib.optionals (cfg.tracerSocketPathAccept i != null) [
+    ]) ++ optionals (cfg.tracerSocketPathAccept i != null) [
       "--tracer-socket-path-accept ${cfg.tracerSocketPathAccept i}"
-    ] ++ lib.optionals (cfg.tracerSocketPathConnect i != null) [
+    ] ++ optionals (cfg.tracerSocketPathConnect i != null) [
       "--tracer-socket-path-connect ${cfg.tracerSocketPathConnect i}"
     ] ++ consensusParams.${cfg.nodeConfig.Protocol} ++ cfg.extraArgs ++ cfg.rtsArgs;
     in ''
       echo "Starting: ${concatStringsSep "\"\n   echo \"" cmd}"
       echo "..or, once again, in a single line:"
       echo "${toString cmd}"
-      ${lib.optionalString (i > 0) ''
+      ${optionalString (i > 0) ''
       # If exist copy state from existing instance instead of syncing from scratch:
       if [ ! -d ${instanceDbPath} ] && [ -d ${cfg.databasePath 0} ]; then
         echo "Copying existing immutable db from ${cfg.databasePath 0}"
@@ -252,8 +252,8 @@ in {
       };
 
       environment = mkOption {
-        type = types.enum (builtins.attrNames cfg.environments);
         default = "testnet";
+        type = types.enum (attrNames cfg.environments);
         description = ''
           environment node will connect to
         '';
@@ -322,7 +322,7 @@ in {
       ipv6HostAddr = mkOption {
         type = funcToOr nullOrStr;
         default = _: null;
-        apply = ip: if (lib.isFunction ip) then ip else _: ip;
+        apply = ip: if lib.isFunction ip then ip else _: ip;
         description = ''
           The ipv6 host address to bind to. Set to null to disable.
         '';
@@ -347,7 +347,7 @@ in {
       stateDir = mkOption {
         type = funcToOr types.str;
         default = "${cfg.stateDirBase}cardano-node";
-        apply = x : if (lib.isFunction x) then x else i: x;
+        apply = x : if lib.isFunction x then x else i: x;
         description = ''
           Directory to store blockchain data, for each instance.
         '';
@@ -706,7 +706,7 @@ in {
       profilingArgs = mkOption {
         type = types.listOf types.str;
         default = let commonProfilingArgs = ["--machine-readable" "-tcardano-node.stats" "-pocardano-node"]
-          ++ lib.optional (cfg.eventlog) "-l";
+          ++ optional (cfg.eventlog) "-l";
           in if cfg.profiling == "time" then ["-p"] ++ commonProfilingArgs
             else if cfg.profiling == "time-detail" then ["-P"] ++ commonProfilingArgs
             else if cfg.profiling == "space" then ["-h"] ++ commonProfilingArgs
@@ -739,10 +739,10 @@ in {
   };
 
   config = mkIf cfg.enable ( let
-    lmdbPaths = filter (x: x != null) (map (e: cfg.lmdbDatabasePath e) (builtins.genList lib.trivial.id cfg.instances));
+    lmdbPaths = filter (x: x != null) (map (e: cfg.lmdbDatabasePath e) (genList trivial.id cfg.instances));
     genInstanceConf = f: listToAttrs (if cfg.instances > 1
       then genList (i: let n = "cardano-node-${toString i}"; in nameValuePair n (f n i)) cfg.instances
-      else [ (nameValuePair "cardano-node" (f "cardano-node" 0)) ]); in lib.mkMerge [
+      else [ (nameValuePair "cardano-node" (f "cardano-node" 0)) ]); in mkMerge [
     {
       users.groups.cardano-node.gid = 10016;
       users.users.cardano-node = {
@@ -776,19 +776,19 @@ in {
           Group = "cardano-node";
           ExecReload = mkIf (cfg.useSystemdReload && cfg.useNewTopology) "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
           Restart = "always";
-          RuntimeDirectory = lib.mkIf (!cfg.systemdSocketActivation)
-            (lib.removePrefix cfg.runDirBase (runtimeDir i));
+          RuntimeDirectory = mkIf (!cfg.systemdSocketActivation)
+            (removePrefix cfg.runDirBase (runtimeDir i));
           WorkingDirectory = cfg.stateDir i;
           # This assumes cfg.stateDirBase is a prefix of cfg.stateDir.
           # This is checked as an assertion below.
-          StateDirectory =  lib.removePrefix cfg.stateDirBase (cfg.stateDir i);
-          NonBlocking = lib.mkIf cfg.systemdSocketActivation true;
-          # time to sleep before restarting a service
+          StateDirectory =  removePrefix cfg.stateDirBase (cfg.stateDir i);
+          NonBlocking = mkIf cfg.systemdSocketActivation true;
+          # Time to sleep before restarting a service
           RestartSec = 1;
         };
       } (cfg.extraServiceConfig i));
 
-      systemd.sockets = genInstanceConf (n: i: lib.mkIf cfg.systemdSocketActivation (recursiveUpdate {
+      systemd.sockets = genInstanceConf (n: i: mkIf cfg.systemdSocketActivation (recursiveUpdate {
         description = "Socket of the ${n} service.";
         wantedBy = [ "sockets.target" ];
         partOf = [ "${n}.service" ];
@@ -797,7 +797,7 @@ in {
             ++ optional (cfg.ipv6HostAddr i != null) "[${cfg.ipv6HostAddr i}]:${toString (if cfg.shareIpv6port then cfg.port else cfg.port + i)}"
             ++ (cfg.additionalListenStream i)
             ++ [(cfg.socketPath i)];
-          RuntimeDirectory = lib.removePrefix cfg.runDirBase (cfg.runtimeDir i);
+          RuntimeDirectory = removePrefix cfg.runDirBase (cfg.runtimeDir i);
           NoDelay = "yes";
           ReusePort = "yes";
           SocketMode = "0660";
@@ -808,8 +808,8 @@ in {
       } (cfg.extraSocketConfig i)));
     }
     {
-      # oneshot service start allows to easily control all instances at once.
-      systemd.services.cardano-node = lib.mkIf (cfg.instances > 1) {
+      # Oneshot service start allows to easily control all instances at once.
+      systemd.services.cardano-node = mkIf (cfg.instances > 1) {
         description = "Control all ${toString cfg.instances} at once.";
         enable  = true;
         wants = genList (i: "cardano-node-${toString i}.service") cfg.instances;
@@ -820,15 +820,15 @@ in {
           Group = "cardano-node";
           ExecStart = "${pkgs.coreutils}/bin/echo Starting ${toString cfg.instances} cardano-node instances";
           WorkingDirectory = cfg.stateDir i;
-          StateDirectory =  lib.removePrefix cfg.stateDirBase (cfg.stateDir i);
+          StateDirectory =  removePrefix cfg.stateDirBase (cfg.stateDir i);
         };
       };
     }
     {
       assertions = [
         {
-          assertion = builtins.all (i : lib.hasPrefix cfg.stateDirBase (cfg.stateDir i))
-                                   (builtins.genList lib.trivial.id cfg.instances);
+          assertion = all (i : hasPrefix cfg.stateDirBase (cfg.stateDir i))
+                                   (genList trivial.id cfg.instances);
           message = "The option services.cardano-node.stateDir should have ${cfg.stateDirBase}
                      as a prefix, for each instance!";
         }
@@ -841,7 +841,7 @@ in {
           message = "Systemd socket activation cannot be used with p2p topology due to a systemd socket re-use issue.";
         }
         {
-          assertion = (length lmdbPaths) == (length (lib.lists.unique lmdbPaths));
+          assertion = (length lmdbPaths) == (length (lists.unique lmdbPaths));
           message   = "When configuring multiple LMDB enabled nodes on one instance, lmdbDatabasePath must be unique.";
         }
       ];

From 370315d64d1ce5bd9ca283f3cd4ad9aa4c609d86 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 25 Feb 2025 16:03:02 -0600
Subject: [PATCH 02/37] nixosNodeSvc: rm withCardanoTracer, default to preview,
 update opt desc

---
 nix/nixos/cardano-node-service.nix | 173 +++++++++++++++++------------
 1 file changed, 101 insertions(+), 72 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index 065aa9ca7a3..60e71e95df6 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -175,8 +175,8 @@ in {
         type = types.bool;
         default = false;
         description = ''
-          Enable cardano-node, a node implementing ouroboros protocols
-          (the blockchain protocols running cardano).
+          Enable cardano-node, a node implementing ouroboros protocols;
+          the blockchain protocols running cardano.
         '';
       };
 
@@ -184,7 +184,7 @@ in {
         type = types.int;
         default = 1;
         description = ''
-          Number of instance of the service to run.
+          Number of instances of the service to run.
         '';
       };
 
@@ -194,13 +194,28 @@ in {
       };
 
       profiling = mkOption {
-        type = types.enum ["none" "time" "time-detail" "space" "space-cost" "space-module" "space-closure" "space-type" "space-retainer" "space-bio" "space-heap"];
+        type = types.enum [
+          "none"
+          "space"
+          "space-bio"
+          "space-closure"
+          "space-cost"
+          "space-heap"
+          "space-module"
+          "space-retainer"
+          "space-type"
+          "time"
+          "time-detail"
+        ];
         default = "none";
       };
 
       eventlog = mkOption {
         type = types.bool;
         default = false;
+        description = ''
+          Whether to enable eventlog profiling.
+        '';
       };
 
       asserts = mkOption {
@@ -216,9 +231,9 @@ in {
         default = pkgs.cardanoNodePackages or (import ../. { inherit (pkgs) system; }).cardanoNodePackages;
         defaultText = "cardano-node packages";
         description = ''
-          The cardano-node packages and library that should be used.
-          Main usage is sharing optimization:
-          reduce eval time when service is instantiated multiple times.
+          The cardano-node packages and library that should be used. The main
+          usage is sharing optimization which reduces eval time when service is
+          instantiated multiple times.
         '';
       };
 
@@ -230,7 +245,7 @@ in {
           else cfg.cardanoNodePackages.cardano-node;
         defaultText = "cardano-node";
         description = ''
-          The cardano-node package that should be used
+          The cardano-node package that should be used.
         '';
       };
 
@@ -239,7 +254,7 @@ in {
         default = "exec ${cfg.package}/bin/cardano-node";
         defaultText = "cardano-node";
         description = ''
-          The cardano-node executable invocation to use
+          The cardano-node executable invocation to use.
         '';
       };
 
@@ -247,15 +262,15 @@ in {
         type = types.attrs;
         default = cfg.cardanoNodePackages.cardanoLib.environments;
         description = ''
-          environment node will connect to
+          The environments cardano-node will connect to.
         '';
       };
 
       environment = mkOption {
-        default = "testnet";
         type = types.enum (attrNames cfg.environments);
+        default = "preview";
         description = ''
-          environment node will connect to
+          The environment cardano-node will connect to.
         '';
       };
 
@@ -264,7 +279,7 @@ in {
         default = false;
         description = ''
           Whether this node is intended to be a producer.
-          Internal option for inter-module communication.
+          An internal option for inter-module communication.
         '';
       };
 
@@ -274,7 +289,7 @@ in {
         type = types.nullOr (types.either types.str types.path);
         default = null;
         description = ''
-          Signing key
+          The signing key.
         '';
       };
 
@@ -282,7 +297,7 @@ in {
         type = types.nullOr (types.either types.str types.path);
         default = null;
         description = ''
-          Delegation certificate
+          The delegation certificate.
         '';
       };
 
@@ -292,14 +307,14 @@ in {
         type = types.nullOr (types.either types.str types.path);
         default = null;
         description = ''
-          Signing key
+          The KES or key evolving signature key.
         '';
       };
       vrfKey = mkOption {
         type = types.nullOr (types.either types.str types.path);
         default = null;
         description = ''
-          Signing key
+          The VRF or verifable random function key.
         '';
       };
 
@@ -307,7 +322,7 @@ in {
         type = types.nullOr (types.either types.str types.path);
         default = null;
         description = ''
-          Operational certificate
+          The operational certificate.
         '';
       };
 
@@ -315,7 +330,7 @@ in {
         type = types.str;
         default = "127.0.0.1";
         description = ''
-          The host address to bind to
+          The host address to bind to.
         '';
       };
 
@@ -332,7 +347,7 @@ in {
         type = types.functionTo (types.listOf types.str);
         default = _: [];
         description = ''
-          List of additional sockets to listen to. Only available with `systemdSocketActivation`.
+          A List of additional sockets to listen to. Only available with `systemdSocketActivation`.
         '';
       };
 
@@ -340,7 +355,7 @@ in {
         type = types.str;
         default = "/var/lib/";
         description = ''
-          Base directory to store blockchain data, for each instance.
+          The base directory to store blockchain data.
         '';
       };
 
@@ -349,7 +364,7 @@ in {
         default = "${cfg.stateDirBase}cardano-node";
         apply = x : if lib.isFunction x then x else i: x;
         description = ''
-          Directory to store blockchain data, for each instance.
+          The directory to store blockchain data, for each instance.
         '';
       };
 
@@ -357,7 +372,7 @@ in {
         type = types.str;
         default = "/run/";
         description = ''
-          Base runtime directory, for each instance.
+          The base runtime directory.
         '';
       };
 
@@ -366,7 +381,7 @@ in {
         default = i: ''${cfg.runDirBase}${suffixDir "cardano-node" i}'';
         apply = x : if lib.isFunction x then x else if x == null then _: null else "${cfg.runDirBase}${suffixDir "cardano-node" x}";
         description = ''
-          Runtime directory relative to ${cfg.runDirBase}, for each instance
+          The runtime directory relative to ${cfg.runDirBase}, for each instance.
         '';
       };
 
@@ -374,7 +389,7 @@ in {
         type = funcToOr types.str;
         default = i : "${cfg.stateDir i}/${cfg.dbPrefix i}";
         apply = x : if lib.isFunction x then x else _ : x;
-        description = ''Node database path, for each instance.'';
+        description = ''The node database path, for each instance.'';
       };
 
       lmdbDatabasePath = mkOption {
@@ -382,7 +397,7 @@ in {
         default = null;
         apply = x : if lib.isFunction x then x else if x == null then _: null else _: x;
         description = ''
-          Node UTxO-HD LMDB path for performant disk I/O, for each instance.
+          A node UTxO-HD LMDB path for performant disk I/O, for each instance.
           This could point to a direct-access SSD, with a specifically created journal-less file system and optimized mount options.
         '';
       };
@@ -391,7 +406,7 @@ in {
         type = funcToOr types.str;
         default = i : "${runtimeDir i}/node.socket";
         apply = x : if lib.isFunction x then x else _ : x;
-        description = ''Local communication socket path, for each instance.'';
+        description = ''A local communication socket path, for each instance.'';
       };
 
       tracerSocketPathAccept = mkOption {
@@ -399,7 +414,7 @@ in {
         default = null;
         apply = x : if lib.isFunction x then x else _ : x;
         description = ''
-          Listen for incoming cardano-tracer connection on a local socket,
+          Listen for an incoming cardano-tracer connection on a local socket,
           for each instance.
         '';
       };
@@ -409,7 +424,7 @@ in {
         default = null;
         apply = x : if lib.isFunction x then x else _ : x;
         description = ''
-          Connect to cardano-tracer listening on a local socket,
+          Connect to a cardano-tracer listening on a local socket,
           for each instance.
         '';
       };
@@ -418,8 +433,8 @@ in {
         type = types.str;
         default = "cardano-node";
         description = ''
-          systemd socket group owner.
-          Note: only applies to sockets created by systemd
+          The systemd socket group owner.
+          Note: this only applies to sockets created by systemd
           (ie. when `systemdSocketActivation` is turned on).
         '';
       };
@@ -437,7 +452,7 @@ in {
           };
         default = i: {};
         description = ''
-          Extra systemd service config (apply to all instances).
+          Extra systemd service config which applies to all instances.
         '';
       };
 
@@ -448,7 +463,7 @@ in {
           };
         default = i: {};
         description = ''
-          Extra systemd socket config (apply to all instances).
+          Extra systemd socket config which applies to all instances.
         '';
       };
 
@@ -457,8 +472,8 @@ in {
         default = suffixDir "db-${cfg.environment}";
         apply = x : if lib.isFunction x then x else suffixDir x;
         description = ''
-          Prefix of database directories inside `stateDir`.
-          (eg. for "db", there will be db-0, etc.).
+          The prefix of database directories inside `stateDir`.
+          (eg. for "db", there will be db-0, etc.), for each instance.
         '';
       };
 
@@ -466,7 +481,7 @@ in {
         type = types.either types.int types.str;
         default = 3001;
         description = ''
-          The port number
+          The port number to listen on.
         '';
       };
 
@@ -474,9 +489,9 @@ in {
         type = types.bool;
         default = cfg.systemdSocketActivation;
         description = ''
-          Should instances on same machine share ipv4 port.
-          Default: true if systemd activated socket. Otherwise false.
-          If false use port increments starting from `port`.
+          Whether instances on the same machine should share an ipv4 port.
+          Default: true if the socket is systemd activated, otherwise false.
+          If false, use port increments starting from `port`.
         '';
       };
 
@@ -484,9 +499,9 @@ in {
         type = types.bool;
         default = cfg.systemdSocketActivation;
         description = ''
-          Should instances on same machine share ipv6 port.
-          Default: true if systemd activated socket. Otherwise false.
-          If false use port increments starting from `port`.
+          Whether instances on the same machine should share an ipv6 port.
+          Default: true if the socket is systemd activated, otherwise false.
+          If false, use port increments starting from `port`.
         '';
       };
 
@@ -494,7 +509,7 @@ in {
         type = types.int;
         default = 0;
         description = ''
-          The ID for this node
+          The ID for this node.
         '';
       };
 
@@ -508,13 +523,20 @@ in {
           }];
           advertise = false;
         }];
-        description = ''Routes to public peers. Only used if slot < usePeersFromLedgerAfterSlot'';
+        description = ''
+          Routes to public peers. Only used if slot is less than
+          usePeersFromLedgerAfterSlot.
+        '';
       };
 
       instancePublicProducers = mkOption {
         type = types.functionTo (types.listOf types.attrs);
         default = _: [];
-        description = ''Routes to public peers. Only used if slot < usePeersFromLedgerAfterSlot and specific to a given instance (when multiple instances are used).'';
+        description = ''
+          Routes to public peers. Only used if slot is less than
+          usePeersFromLedgerAfterSlot and specific to a given instance when
+          multiple instances are used.
+        '';
       };
 
       producers = mkOption {
@@ -535,7 +557,8 @@ in {
         type = types.functionTo (types.listOf types.attrs);
         default = _: [];
         description = ''
-          Static routes to local peers, specific to a given instance (when multiple instances are used).
+          Static routes to local peers, specific to a given instance when
+          multiple instances are used.
         '';
       };
 
@@ -543,7 +566,7 @@ in {
         type = types.bool;
         default = cfg.nodeConfig.EnableP2P or false;
         description = ''
-          Use new, p2p/ledger peers compatible topology.
+          Use new, peer to peer and ledger peers compatible topology.
         '';
       };
 
@@ -561,8 +584,9 @@ in {
           else envConfig.usePeersFromLedgerAfterSlot or null;
         description = ''
           If set, bootstraps from public roots until it reaches given slot,
-          then it switches to using the ledger as a source of peers. It maintains a connection to its local roots.
-          Default to null for block producers.
+          then it switches to using the ledger as a source of peers. It
+          maintains a connection to its local roots. Defaults to null for block
+          producers.
         '';
       };
 
@@ -570,9 +594,9 @@ in {
         type = types.nullOr (types.listOf types.attrs);
         default = map (e: {address = e.addr; inherit (e) port;}) envConfig.edgeNodes;
         description = ''
-          If set, it will enable bootstrap peers.
-          To disable, set this to null.
-          To enable, set this to a list of attributes of address and port, example: [{ address = "addr"; port = 3001; }]
+          If set, it will enable bootstrap peers. To disable, set this to null.
+          To enable, set this to a list of attributes of address and port,
+          example: [{ address = "addr"; port = 3001; }]
         '';
       };
 
@@ -580,7 +604,8 @@ in {
         type = types.nullOr (types.either types.str types.path);
         default = null;
         description = ''
-          Cluster topology. If not set `producers` array is used to generated topology file.
+          The cluster topology. If not set the `producers` array is used to
+          generate a topology file.
         '';
       };
 
@@ -604,21 +629,21 @@ in {
           merge = loc: foldl' (res: def: recursiveUpdate res def.value) {};
         };
         default = envConfig.nodeConfig;
-        description = ''Internal representation of the config.'';
+        description = ''The internal representation of the config.'';
       };
 
       targetNumberOfRootPeers = mkOption {
         type = types.int;
         default = cfg.nodeConfig.TargetNumberOfRootPeers or 100;
-        description = "Limits the maximum number of root peers the node will know about";
+        description = "The target number of root peers the node will know about.";
       };
 
       targetNumberOfKnownPeers = mkOption {
         type = types.int;
         default = cfg.nodeConfig.TargetNumberOfKnownPeers or cfg.targetNumberOfRootPeers;
         description = ''
-          Target number for known peers (root peers + peers known through gossip).
-          Default to targetNumberOfRootPeers.
+          The target number of known peers, counting root peers and peers known
+          through gossip. Defaults to targetNumberOfRootPeers.
         '';
       };
 
@@ -626,16 +651,20 @@ in {
         type = types.int;
         default = cfg.nodeConfig.TargetNumberOfEstablishedPeers
           or (cfg.targetNumberOfKnownPeers / 2);
-        description = ''Number of peers the node will be connected to, but not necessarily following their chain.
-          Default to half of targetNumberOfKnownPeers.
+        description = ''
+          The target number of peers the node will be connected to, but not
+          necessarily following their chain. Defaults to half of the
+          targetNumberOfKnownPeers.
         '';
       };
 
       targetNumberOfActivePeers = mkOption {
         type = types.int;
         default = cfg.nodeConfig.TargetNumberOfActivePeers or (2 * cfg.targetNumberOfEstablishedPeers / 5);
-        description = ''Number of peers your node is actively downloading headers and blocks from.
-          Default to 2/5 of targetNumberOfEstablishedPeers.
+        description = ''
+          Target number of peers the node is actively downloading headers and
+          blocks from. Defaults to the node config spec or two-fifths of the
+          targetNumberOfEstablishedPeers.
         '';
       };
 
@@ -659,33 +688,33 @@ in {
       nodeConfigFile = mkOption {
         type = nullOrStr;
         default = null;
-        description = ''Actual configuration file (shell expression).'';
+        description = ''The actual configuration file.'';
       };
 
       forceHardForks = mkOption {
         type = types.attrsOf types.int;
         default = {};
         description = ''
-          A developer-oriented dictionary option to force hard forks for given eras at given epochs.  Maps capitalised era names (Shelley, Allegra, Mary, etc.) to hard fork epoch number.
-          '';
-      };
-
-      withCardanoTracer = mkOption {
-        type = types.bool;
-        default = false;
+          A developer-oriented dictionary option to force hard forks for given
+          eras at given epochs.  Maps capitalised era names (Shelley, Allegra,
+          Mary, etc) to hard fork epoch number.
+        '';
       };
 
       withUtxoHdLmdb = mkOption {
         type = funcToOr types.bool;
         default = false;
         apply = x: if lib.isFunction x then x else _: x;
-        description = ''On an UTxO-HD enabled node, the in-memory backend is the default. This activates the on-disk backend (LMDB) instead.'';
+        description = ''
+          On a UTxO-HD enabled node, the in-memory backend is the default.
+          This activates the on-disk backend (LMDB) instead.
+        '';
       };
 
       extraArgs = mkOption {
         type = types.listOf types.str;
         default = [];
-        description = ''Extra CLI args for 'cardano-node'.'';
+        description = ''Extra CLI args for cardano-node.'';
       };
 
       rts_flags_override = mkOption {
@@ -700,7 +729,7 @@ in {
         apply = args: if (args != [] || cfg.profilingArgs != [] || cfg.rts_flags_override != []) then
           ["+RTS"] ++ cfg.profilingArgs ++ args ++ cfg.rts_flags_override ++ ["-RTS"]
           else [];
-        description = ''Extra CLI args for 'cardano-node', to be surrounded by "+RTS"/"-RTS"'';
+        description = ''Extra CLI args for cardano-node, to be surrounded by "+RTS"/"-RTS"'';
       };
 
       profilingArgs = mkOption {

From dc8538cd2b2137e719afb1d228b2516bbc803b17 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 25 Feb 2025 16:12:17 -0600
Subject: [PATCH 03/37] nixosTracerSvc: mv existing tracer module to legacy,
 with desc header

---
 nix/nixos/cardano-tracer-service-legacy.nix | 112 ++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 nix/nixos/cardano-tracer-service-legacy.nix

diff --git a/nix/nixos/cardano-tracer-service-legacy.nix b/nix/nixos/cardano-tracer-service-legacy.nix
new file mode 100644
index 00000000000..b803e72021e
--- /dev/null
+++ b/nix/nixos/cardano-tracer-service-legacy.nix
@@ -0,0 +1,112 @@
+# The legacy cardano-tracer module requires commonLib from the overlay
+# which makes a simple non-flake service module import convoluted
+# and also requires knowledge of the cardano-node commonLib functions to use.
+# This makes this legacy module suitable for internal purposes, but not ideal
+# for a general purpose single import cardano-tracer module.
+#
+# The module at nix/nixos/cardano-tracer-service.nix is the preferred module
+# for general purpose use.
+pkgs:
+let serviceConfigToJSON =
+      cfg:
+      {
+        inherit (cfg) networkMagic resourceFreq metricsHelp;
+        # loRequestNum = 100;
+        network =
+          if        cfg.acceptingSocket != null
+          then {
+            tag      = "AcceptAt";
+            contents = cfg.acceptingSocket;
+          } else if cfg.connectToSocket != null
+          then {
+            tag      = "ConnectTo";
+            contents = cfg.connectToSocket;
+          } else
+            throw "cardano-tracer-service:  either acceptingSocket or connectToSocket must be provided.";
+        logging = [{
+          inherit (cfg) logRoot;
+
+          logMode   = "FileMode";
+          logFormat = "ForMachine";
+        }];
+        rotation = {
+          rpFrequencySecs = 15;
+          rpKeepFilesNum  = 10;
+          rpLogLimitBytes = 1000000000;
+          rpMaxAgeHours   = 24;
+        } // (cfg.rotation or {});
+
+        hasEKG = {
+          epHost  = "127.0.0.1";
+          epPort  = cfg.ekgPortBase;
+        };
+        ekgRequestFreq = 1;
+        hasPrometheus = {
+          epHost    = "127.0.0.1";
+          epPort    = 3200; ## supervisord.portShiftPrometheus
+        } // (cfg.prometheus or {});
+        # Just an example for metrics compatibility mapping.
+        # An entry means the first entry has the second entry as alias.
+        # The Metrics is then avalable, both with the original and the mapped name.
+        # Only one mapping per message is supported.
+        # metricsComp = {
+        #     "Mempool.TxsInMempool" = "Mempool.TxsInMempool.Mapped";
+        #     "ChainDB.SlotNum" = "ChainDB.SlotNum.Mapped";
+        # };
+      } // pkgs.lib.optionalAttrs ((cfg.RTView or {}) != {})
+      {
+        hasRTView = cfg.RTView;
+      };
+in pkgs.commonLib.defServiceModule
+  (lib: with lib;
+    { svcName = "cardano-tracer";
+      svcDesc = "Cardano trace processor";
+
+      svcPackageSelector =
+        pkgs: ## Local:
+              pkgs.cardanoNodePackages.cardano-tracer
+              ## Imported by another repo, that adds an overlay:
+                or pkgs.cardano-tracer;
+              ## TODO:  that's actually a bit ugly and could be improved.
+      ## This exe has to be available in the selected package.
+      exeName = "cardano-tracer";
+
+      extraOptionDecls = {
+        ### You can actually change those!
+        networkMagic    = opt    int 764824073 "Network magic (764824073 for Cardano mainnet).";
+        acceptingSocket = mayOpt str           "Socket path: as acceptor.";
+        connectToSocket = mayOpt str           "Socket path: connect to.";
+        logRoot         = opt    str null      "Log storage root directory.";
+        rotation        = opt    attrs {}      "Log rotation overrides: see cardano-tracer documentation.";
+        RTView          = opt    attrs {}      "RTView config overrides: see cardano-tracer documentation.";
+        ekgPortBase     = opt    int 3100      "EKG port base.";
+        ekgRequestFreq  = opt    int 1         "EKG request frequency";
+        prometheus      = opt    attrs {}      "Prometheus overrides: see cardano-tracer documentation.";
+        resourceFreq    = mayOpt int           "Frequency (1/ms) for tracing resource usage.";
+        metricsHelp     = mayOpt str           "JSON file containing metrics help annotations for Prometheus";
+
+        ### Here be dragons, on the other hand..
+        configFile      = mayOpt str
+          "Config file path override -- only set if you know what you're doing. Shudder. Your 'eminence'..";
+        configJSONfn    = opt (functionTo attrs) serviceConfigToJSON
+          "This is NOT meant to be overridden, at all -- we only expose it so it's externally accessible.";
+      };
+
+      configExeArgsFn = cfg: [
+        "--config" (if cfg.configFile != null then cfg.configFile
+                    else "${pkgs.writeText "cardano-tracer-config.json"
+                                           (__toJSON (serviceConfigToJSON cfg))}")
+        ];
+
+      configSystemdExtraConfig = _: {};
+
+      configSystemdExtraServiceConfig =
+        cfg: with cfg; {
+          Type = "exec";
+          User = "cardano-node";
+          Group = "cardano-node";
+          Restart = "always";
+          # RuntimeDirectory = localNodeConf.runtimeDir;
+          # WorkingDirectory = localNodeConf.stateDir;
+        };
+    })

From 3545a59a9b48303daf9071bf7b428b68ec3affa4 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Thu, 27 Feb 2025 14:45:57 -0600
Subject: [PATCH 04/37] nixosNodeSvc: use topology.json vs yaml for consistency
 w/ other cfgs

---
 nix/nixos/cardano-node-service.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index 60e71e95df6..eb2b0a06032 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -56,11 +56,11 @@ let
   selectTopology = i:
     if cfg.topology != null
     then cfg.topology
-    else toFile "topology.yaml" (toJSON (if (cfg.useNewTopology) then assertNewTopology i else oldTopology i));
+    else toFile "topology.json" (toJSON (if (cfg.useNewTopology) then assertNewTopology i else oldTopology i));
 
   topology = i:
     if cfg.useSystemdReload
-    then "/etc/cardano-node/topology-${toString i}.yaml"
+    then "/etc/cardano-node/topology-${toString i}.json"
     else selectTopology i;
 
   mkScript = cfg:
@@ -617,7 +617,7 @@ in {
           if only the topology file has changed and p2p is in use.
 
           Cardano-node topology files will be stored in /etc as:
-            /etc/cardano-node/topology-''${toString i}.yaml
+            /etc/cardano-node/topology-''${toString i}.json
 
           Enabling this option will also allow direct topology edits for tests when a full
           service re-deployment is not desired.
@@ -782,7 +782,7 @@ in {
       };
 
       environment.etc = mkIf cfg.useSystemdReload (foldl'
-        (acc: i: recursiveUpdate acc {"cardano-node/topology-${toString i}.yaml".source = selectTopology i;}) {}
+        (acc: i: recursiveUpdate acc {"cardano-node/topology-${toString i}.json".source = selectTopology i;}) {}
       (range 0 (cfg.instances - 1)));
 
       ## TODO:  use http://hackage.haskell.org/package/systemd for:

From 9e496953bb907cc562dc049f7aab3a7812efbd78 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Thu, 27 Feb 2025 20:10:20 -0600
Subject: [PATCH 05/37] nixosTracerSvc: add a new downstream consumable tracer
 service

---
 nix/nixos/cardano-tracer-service.nix | 892 ++++++++++++++++++++++++---
 1 file changed, 800 insertions(+), 92 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index dea2dcc6095..62a8f57ed4f 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -1,97 +1,805 @@
-pkgs:
-let serviceConfigToJSON =
-      cfg:
-      {
-        inherit (cfg) networkMagic resourceFreq metricsHelp;
-        # loRequestNum = 100;
-        network =
-          if        cfg.acceptingSocket != null
-          then {
-            tag      = "AcceptAt";
-            contents = cfg.acceptingSocket;
-          } else if cfg.connectToSocket != null
-          then {
-            tag      = "ConnectTo";
-            contents = cfg.connectToSocket;
-          } else
-            throw "cardano-tracer-service:  either acceptingSocket or connectToSocket must be provided.";
-        logging = [{
-          inherit (cfg) logRoot;
-
-          logMode   = "FileMode";
-          logFormat = "ForMachine";
-        }];
-        rotation = {
-          rpFrequencySecs = 15;
-          rpKeepFilesNum  = 10;
-          rpLogLimitBytes = 1000000000;
-          rpMaxAgeHours   = 24;
-        } // (cfg.rotation or {});
-
-        hasEKG = {
-          epHost  = "127.0.0.1"; 
-          epPort  = cfg.ekgPortBase;
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib;
+with builtins; let
+  inherit (types) attrs attrsOf bool either enum ints listOf package path port nullOr str submodule;
+
+  cfg = config.services.cardano-tracer;
+
+  configFile =
+    if !isNull cfg.configFile
+    then cfg.configFile
+    else "/etc/cardano-tracer/config.json";
+
+  tracerConfig =
+    {
+      inherit
+        (cfg)
+        ekgRequestFreq
+        loRequestNum
+        logging
+        metricsComp
+        metricsHelp
+        networkMagic
+        resourceFreq
+        verbosity
+        ;
+
+      network =
+        optionalAttrs (!isNull cfg.acceptingSocket) {
+          tag = "AcceptAt";
+          contents = cfg.acceptingSocket;
+        }
+        // optionalAttrs (!isNull cfg.connectToSocket) {
+          tag = "ConnectTo";
+          contents = cfg.connectToSocket;
         };
-        ekgRequestFreq = 1;
-        hasPrometheus = {
-          epHost    = "127.0.0.1";
-          epPort    = 3200; ## supervisord.portShiftPrometheus
-        } // (cfg.prometheus or {});
-      } // pkgs.lib.optionalAttrs ((cfg.RTView or {}) != {})
-      {
-        hasRTView = cfg.RTView;
-      };
-in pkgs.commonLib.defServiceModule
-  (lib: with lib;
-    { svcName = "cardano-tracer";
-      svcDesc = "Cardano trace processor";
-
-      svcPackageSelector =
-        pkgs: ## Local:
-              pkgs.cardanoNodePackages.cardano-tracer
-              ## Imported by another repo, that adds an overlay:
-                or pkgs.cardano-tracer;
-              ## TODO:  that's actually a bit ugly and could be improved.
-      ## This exe has to be available in the selected package.
-      exeName = "cardano-tracer";
-
-      extraOptionDecls = {
-        ### You can actually change those!
-        networkMagic    = opt    int 764824073 "Network magic (764824073 for Cardano mainnet).";
-        acceptingSocket = mayOpt str           "Socket path: as acceptor.";
-        connectToSocket = mayOpt str           "Socket path: connect to.";
-        logRoot         = opt    str null      "Log storage root directory.";
-        rotation        = opt    attrs {}      "Log rotation overrides: see cardano-tracer documentation.";
-        RTView          = opt    attrs {}      "RTView config overrides: see cardano-tracer documentation.";
-        ekgPortBase     = opt    int 3100      "EKG port base.";
-        ekgRequestFreq  = opt    int 1         "EKG request frequency";
-        prometheus      = opt    attrs {}      "Prometheus overrides: see cardano-tracer documentation.";
-        resourceFreq    = mayOpt int           "Frequency (1/ms) for tracing resource usage.";
-        metricsHelp     = mayOpt str           "JSON file containing metrics help annotations for Prometheus";
-        metricsNoSuffix = mayOpt bool          "Drop suffixes like '_int' in Prometheus exposition, increasing similiarity with legacy system names.";
-
-        ### Here be dragons, on the other hand..
-        configFile      = mayOpt str
-          "Config file path override -- only set if you know what you're doing. Shudder. Your 'eminence'..";
-        configJSONfn    = opt (functionTo attrs) serviceConfigToJSON
-          "This is NOT meant to be overridden, at all -- we only expose it so it's externally accessible.";
-      };
-
-      configExeArgsFn = cfg: [
-        "--config" (if cfg.configFile != null then cfg.configFile
-                    else "${pkgs.writeText "cardano-tracer-config.json"
-                                           (__toJSON (serviceConfigToJSON cfg))}")
+
+      rotation =
+        if isNull cfg.rotation
+        then null
+        else
+          {
+            inherit
+              (cfg.rotation)
+              rpFrequencySecs
+              rpKeepFilesNum
+              rpLogLimitBytes
+              ;
+          }
+          // optionalAttrs (!isNull cfg.rotation.rpMaxAgeHours) {
+            inherit (cfg.rotation) rpMaxAgeHours;
+          }
+          // optionalAttrs (!isNull cfg.rotation.rpMaxAgeMinutes) {
+            inherit (cfg.rotation) rpMaxAgeMinutes;
+          };
+
+      WarnRTViewMissing = cfg.warnRtviewMissing;
+    }
+    // optionalAttrs cfg.ekgEnable {
+      hasEKG = {
+        epHost = cfg.ekgHost;
+        epPort = cfg.ekgPort;
+      };
+    }
+    // optionalAttrs cfg.prometheusEnable {
+      hasPrometheus = {
+        epHost = cfg.prometheusHost;
+        epPort = cfg.prometheusPort;
+      };
+    }
+    // optionalAttrs cfg.rtviewEnable {
+      hasRTView = {
+        epHost = cfg.rtviewHost;
+        epPort = cfg.rtviewPort;
+      };
+    }
+    // cfg.extraConfig;
+
+  prettyConfig =
+    (pkgs.runCommandNoCCLocal "cardano-tracer-config.json" {} ''
+      ${getExe pkgs.jq} --sort-keys \
+        < ${toFile "cardano-tracer-unpretty-config.json" (toJSON tracerConfig)} \
+        > $out
+    '')
+    .out;
+
+  mkScript = let
+    cmd =
+      filter (x: x != "")
+      [
+        "${cfg.executable}"
+        "--config ${configFile}"
+      ]
+      ++ optionals (!isNull cfg.minLogSeverity) [
+        "--min-log-severity ${cfg.minLogSeverity}"
+      ]
+      ++ optionals (!isNull cfg.stateDir) [
+        "--state-dir ${cfg.stateDir}"
+      ]
+      ++ cfg.extraArgs
+      ++ cfg.rtsArgs;
+  in ''
+    echo "Starting: ${concatStringsSep "\"\n   echo \"" cmd}"
+
+    echo "..or, once again, in a single line:"
+    echo "${toString cmd}"
+
+    ${toString cmd}
+  '';
+in {
+  options = {
+    services.cardano-tracer = {
+      enable = mkOption {
+        type = bool;
+        default = false;
+        description = ''
+          Enable cardano-tracer, a service for logging and monitoring of
+          Cardano nodes. After it is connected to the node(s), it periodically
+          asks for different information, receives it, and handles it.
+        '';
+      };
+
+      #####################################
+      #                                   #
+      # Alphabetical nixos module options #
+      #                                   #
+      #####################################
+
+      acceptingSocket = mkOption {
+        type = nullOr (either str path);
+        default = "${cfg.runtimeDir}/cardano-tracer.socket";
+        description = ''
+          If accepting connections from node(s) to a cardano-tracer socket, the
+          path.
+
+          Either this option, or the connectToSocket option must be declared.
+        '';
+      };
+
+      asserts = mkOption {
+        type = bool;
+        default = false;
+        description = ''
+          Whether to use an executable with asserts enabled.
+        '';
+      };
+
+      cardanoNodePackages = mkOption {
+        type = attrs;
+        default = pkgs.cardanoNodePackages or (import ../. {inherit (pkgs) system;}).cardanoNodePackages;
+        defaultText = "cardano-node packages";
+        description = ''
+          The cardano-node packages and library that should be used. The main
+          use case is for a sharing optimization which reduces eval time when
+          cardano node packages are instantiated multiple times.
+        '';
+      };
+
+      configFile = mkOption {
+        type = nullOr (either str path);
+        default = null;
+        description = ''
+          The actual cardano-tracer configuration file. If this option is set
+          to null, a configuration file will be built based on the nixos
+          options and symlinked to `/etc/cardano-tracer/config.json`.
+        '';
+      };
+
+      connectToSocket = mkOption {
+        type = nullOr (either str path);
+        default = null;
+        description = ''
+          If connecting to a cardano-node socket, the path.
+
+          Either this option, or the acceptingSocket option must be declared.
+        '';
+      };
+
+      ekgEnable = mkOption {
+        type = bool;
+        default = true;
+        description = ''
+          Whether to enable an EKG http interface for process monitoring.
+        '';
+      };
+
+      ekgHost = mkOption {
+        type = str;
+        default = "127.0.0.1";
+        description = ''
+          The host to bind if EKG is enabled.
+        '';
+      };
+
+      ekgPort = mkOption {
+        type = port;
+        default = 12788;
+        description = ''
+          The port to listen on if EKG is enabled.
+        '';
+      };
+
+      ekgRequestFreq = mkOption {
+        type = nullOr ints.positive;
+        default = null;
+        description = ''
+          This optional attribute specifies the period of how often EKG metrics
+          will be requested, in seconds. For example, if ekgRequestFreq is 10,
+          cardano-tracer will ask for new EKG metrics every ten seconds. There
+          is no limit as loRequestNum, so every request returns all the metrics
+          the node has in this moment of time.
+
+          If null cardano-tracer will set a default: 1.
+        '';
+      };
+
+      environment = mkOption {
+        type = enum (attrNames cfg.environments);
+        default = "preview";
+        description = ''
+          The environment cardano-tracer will connect to.
+        '';
+      };
+
+      environments = mkOption {
+        type = attrs;
+        default = cfg.cardanoNodePackages.cardanoLib.environments;
+        description = ''
+          The environments cardano-tracer will possibly utilize.
+        '';
+      };
+
+      eventlog = mkOption {
+        type = bool;
+        default = false;
+        description = ''
+          Whether to enable eventlog profiling.
+        '';
+      };
+
+      executable = mkOption {
+        type = str;
+        default = "exec ${cfg.package}/bin/cardano-tracer";
+        defaultText = "cardano-node";
+        description = ''
+          The cardano-tracer executable invocation to use.
+        '';
+      };
+
+      extraArgs = mkOption {
+        type = listOf str;
+        default = [];
+        description = ''
+          Extra CLI args for cardano-tracer.
+        '';
+      };
+
+      extraConfig = mkOption {
+        type = attrs;
+        default = {};
+        description = ''
+          Extra configuration attributes for cardano-tracer which will be
+          merged into default cardano-tracer configuration when option
+          `configFile` is null.
+        '';
+      };
+
+      group = mkOption {
+        type = str;
+        default = "cardano-node";
+        description = ''
+          The default group to run the systemd service as.
+
+          This group is assumed to already exist.
+        '';
+      };
+
+      # Consider adding hasForwarding config once more documentation is
+      # available on the feature. Ex:
+      #
+      # hasForwarding = {...};
+
+      logging = mkOption {
+        type = listOf (submodule {
+          options = {
+            logFormat = mkOption {
+              type = enum ["ForHuman" "ForMachine"];
+              default = "ForHuman";
+              description = ''
+                The logFormat option specifies the format of logs. There are two
+                possible modes: `ForMachine` and `ForHuman`. ForMachine is for JSON
+                format, ForHuman is for human-friendly text format. Since the logging
+                option accepts a list more than one logging section can be declared.
+              '';
+            };
+
+            logMode = mkOption {
+              type = enum ["FileMode" "JournalMode"];
+              default = "JournalMode";
+              description = ''
+                The logMode option specifies logging mode. There are two possible
+                modes: `FileMode` and `JournalMode`. FileMode is for storing logs to
+                the files, JournalMode is for storing them in systemd's journal. If
+                you choose JournalMode, the logRoot option, will be ignored.
+              '';
+            };
+
+            logRoot = mkOption {
+              type = str;
+              default = cfg.stateDir;
+              description = ''
+                The logRoot option specifies the path to the root directory. This
+                directory will contain all the subdirectories with the log files
+                inside. Remember that each subdirectory corresponds to the particular
+                node. If the root directory does not exist, it will be created.
+              '';
+            };
+          };
+        });
+        default = [{}];
+        description = ''
+          The logging option describes the logging operation of cardano-tracer
+          and is a list of a submodule of which each contains a `logFormat`,
+          `logMode` and `logRoot`.
+
+          See the descriptions for each available submodule option.
+        '';
+      };
+
+      loRequestNum = mkOption {
+        type = nullOr ints.positive;
+        default = null;
+        description = ''
+          This optional attribute specifies the number of log items
+          that will be requested from the node. For example, if loRequestNum is
+          10, cardano-tracer will periodically ask 10 log items in one request.
+          This value is useful for fine-tuning network traffic: it is possible
+          to ask 50 log items in one request, or ask them in 50 requests one at
+          a time. loRequestNum is the maximum number of log items. For example,
+          if cardano-tracer requests 50 log items but the node has only 40 at
+          that moment, these 40 items will be returned, the request won't block
+          to wait for an additional 10 items.
+
+          If null cardano-tracer will set a default: 100.
+        '';
+      };
+
+      minLogSeverity = mkOption {
+        type = nullOr (enum ["Debug" "Info" "Notice" "Warning" "Error" "Critical" "Alert" "Emergency"]);
+        default = null;
+        description = ''
+          Setting this will cause cardano-tracer to drop log messages less
+          severe than the level declared.
+        '';
+      };
+
+      metricsComp = mkOption {
+        type = nullOr (attrsOf str);
+        default = null;
+        description = ''
+          Passing metric compatability mapping to cardano-tracer can be done as
+          a an attribute set of strings from metric name to mapped metric names
+          where cardano-tracer's internal metric names have to be used as
+          attribute names.  The metrics are then available with both the
+          original name and mapped name.  Only one mapping per message is
+          supported.
+
+          If such a set is already available as JSON, this also can be imported:
+
+            services.cardano-tracer.metricsComp =
+              builtins.fromJSON (builtins.readFile $PATH);
+
+          Any metrics prefix name declared with `TraceOptionMetricsPrefix` in
+          cardano-node config should not be included in the attribute name.
+          Similarly metric type suffixes, such as `.int` or `.real` should also
+          not be included.
+        '';
+        example = {
+          "Mempool.TxsInMempool" = "Mempool.TxsInMempool.Mapped";
+          "ChainDB.SlotNum" = "ChainDB.SlotNum.Mapped";
+        };
+      };
+
+      metricsHelp = mkOption {
+        type = nullOr (attrsOf str);
+        default = null;
+        description = ''
+          Passing metric help annotations to cardano-tracer can be done as a an
+          attribute set of strings from metric name to help text where
+          cardano-tracer's internal metric names have to be used as attribute
+          names.
+
+          If such a set is already available as JSON, this also can be imported:
+
+            services.cardano-tracer.metricsHelp =
+              builtins.fromJSON (builtins.readFile $PATH);
+
+          Any metrics prefix name declared with `TraceOptionMetricsPrefix` in
+          cardano-node config should not be included in the attribute name.
+          Similarly metric type suffixes, such as `.int` or `.real` should also
+          not be included.
+        '';
+        example = {
+          "Mem.resident" = "Kernel-reported RSS (resident set size)";
+          "RTS.gcMajorNum" = "Major GCs";
+        };
+      };
+
+      networkMagic = mkOption {
+        type = ints.positive;
+        default = (fromJSON (readFile cfg.environments.${cfg.environment}.nodeConfig.ShelleyGenesisFile)).networkMagic;
+        description = ''
+          The network magic of the cardano environment which will be connected
+          with cardano-tracer.
+        '';
+      };
+
+      package = mkOption {
+        type = package;
+        default =
+          if (cfg.profiling != "none")
+          then cfg.cardanoNodePackages.cardano-tracer.passthru.profiled
+          else if cfg.eventlog
+          then cfg.cardanoNodePackages.cardano-tracer.passthru.eventlogged
+          else if cfg.asserts
+          then cfg.cardanoNodePackages.cardano-tracer.passthru.asserted
+          else cfg.cardanoNodePackages.cardano-tracer;
+        defaultText = "cardano-tracer";
+        description = ''
+          The cardano-tracer package that should be used.
+        '';
+      };
+
+      profiling = mkOption {
+        type = enum [
+          "none"
+          "space"
+          "space-bio"
+          "space-closure"
+          "space-cost"
+          "space-heap"
+          "space-module"
+          "space-retainer"
+          "space-type"
+          "time"
+          "time-detail"
         ];
+        default = "none";
+        description = ''
+          Haskell profiling types which are available and will be applied to
+          the cardano-tracer binary if declared.
+        '';
+      };
+
+      profilingArgs = mkOption {
+        type = listOf str;
+        default = let
+          commonProfilingArgs =
+            ["--machine-readable" "-tcardano-tracer.stats" "-pocardano-tracer"]
+            ++ optional (cfg.eventlog) "-l";
+        in
+          if cfg.profiling == "time"
+          then ["-p"] ++ commonProfilingArgs
+          else if cfg.profiling == "time-detail"
+          then ["-P"] ++ commonProfilingArgs
+          else if cfg.profiling == "space"
+          then ["-h"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-cost"
+          then ["-hc"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-module"
+          then ["-hm"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-closure"
+          then ["-hd"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-type"
+          then ["-hy"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-retainer"
+          then ["-hr"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-bio"
+          then ["-hb"] ++ commonProfilingArgs
+          else if cfg.profiling == "space-heap"
+          then ["-hT"] ++ commonProfilingArgs
+          else [];
+        description = ''
+          RTS profiling options.
+        '';
+      };
+
+      prometheusEnable = mkOption {
+        type = bool;
+        default = true;
+        description = ''
+          Whether to enable a prometheus export of EKG metrics.
+        '';
+      };
+
+      prometheusHost = mkOption {
+        type = str;
+        default = "127.0.0.1";
+        description = ''
+          The host to bind if prometheus is enabled.
+        '';
+      };
+
+      prometheusPort = mkOption {
+        type = port;
+        default = 12798;
+        description = ''
+          The port to listen on if prometheus is enabled.
+        '';
+      };
+
+      resourceFreq = mkOption {
+        type = nullOr ints.positive;
+        default = 1000;
+        description = ''
+          The period for tracing cardano-tracer resource usage in milliseconds.
+          The frequency will be 1/resourceFreq times per millisecond.  If null
+          cardano-tracer will not display resource usage.
+        '';
+      };
+
+      rotation = mkOption {
+        type = nullOr (submodule ({config, ...}: {
+          options = {
+            rpFrequencySecs = mkOption {
+              type = ints.positive;
+              description = ''
+                The rpFrequencySecs option specifies rotation period, in
+                seconds.
+              '';
+            };
+
+            rpKeepFilesNum = mkOption {
+              type = ints.positive;
+              description = ''
+                The rpKeepFilesNum option specifies the number of the log
+                files that will be kept.  The last (newest) log files will
+                always be the ones kept, whereas the first (oldest) log files
+                will be purged.
+              '';
+            };
 
-      configSystemdExtraConfig = _: {};
+            rpLogLimitBytes = mkOption {
+              type = ints.positive;
+              description = ''
+                The rpLogLimitBytes option specifies the maximum size of the
+                log file, in bytes. Once the size of the current log file
+                reaches this value, a new log file will be created.
+              '';
+            };
 
-      configSystemdExtraServiceConfig =
-        cfg: with cfg; {
-          Type = "exec";
-          User = "cardano-node";
-          Group = "cardano-node";
-          Restart = "always";
-          # RuntimeDirectory = localNodeConf.runtimeDir;
-          # WorkingDirectory = localNodeConf.stateDir;
+            rpMaxAgeMinutes = mkOption {
+              type = nullOr ints.positive;
+              default = null;
+              description = ''
+                The rpMaxAgeMinutes option specifies the lifetime of the log
+                file in minutes. Once the log file reaches this value, it is
+                treated as outdated and will be deleted.  Please note that N
+                last log files, specified by option rpKeepFilesNum, will be
+                kept even if they are outdated. If the rpMaxAgeHours option is
+                also declared this option takes precedence.
+              '';
+            };
+
+            rpMaxAgeHours = mkOption {
+              type = nullOr ints.positive;
+              default = null;
+              description = ''
+                The rpMaxAgeHours option specifies the lifetime of the log
+                file in hours. Once the log file reaches this value, it is
+                treated as outdated and will be deleted.  Please note that N
+                last log files, specified by option rpKeepFilesNum, will be
+                kept even if they are outdated. If the rpMaxAgeMinutes option
+                is also declared then it takes precedence.
+              '';
+            };
+          };
+        }));
+        apply = rot:
+          if isNull rot
+          then rot
+          else if (isNull rot.rpMaxAgeHours && isNull rot.rpMaxAgeMinutes)
+          then
+            throw ''
+              In services.cardano-tracer.rotation at least one of
+              rpMaxAgeMinutes and rpMaxAgeHours must be declared.
+            ''
+          else if (!isNull rot.rpMaxAgeHours && !isNull rot.rpMaxAgeMinutes)
+          then
+            warn ''
+              In services.cardano-tracer.rotation both rpMaxAgeMinutes and
+              rpMaxAgeHours have been declared.  The latter will be ignored by
+              cardano-tracer.
+            ''
+            rot
+          else rot;
+        default = {
+          # Provide some sane defaults
+          rpFrequencySecs = 60;
+          rpKeepFilesNum = 14;
+          rpMaxAgeHours = 24;
+          rpLogLimitBytes = 10 * 1000 * 1000;
         };
-    })
+        description = ''
+          The rotation option describes the log rotation operation of
+          cardano-tracer and is either null or a submodule with options of
+          `rpFrequencySecs`, `rpKeepFilesNum`, `rpLogLimitBytes`,
+          `rpMaxAgeMinutes`.
+
+          Please note that if you skip this field, all log items will be stored
+          in a single file, and usually that's not what is desired.
+
+          This option will be ignored if all logging has `logMode` configured
+          as `JournalMode`.
+
+          See the descriptions for each available submodule option.
+        '';
+      };
+
+      rts_flags_override = mkOption {
+        type = listOf str;
+        default = [];
+        description = ''
+          RTS flags override from profile content.
+        '';
+      };
+
+      rtsArgs = mkOption {
+        type = listOf str;
+        default = [];
+        apply = args:
+          if (args != [] || cfg.profilingArgs != [] || cfg.rts_flags_override != [])
+          then ["+RTS"] ++ cfg.profilingArgs ++ args ++ cfg.rts_flags_override ++ ["-RTS"]
+          else [];
+        description = ''
+          Extra CLI args for cardano-tracer, to be surrounded by "+RTS"/"-RTS"
+        '';
+      };
+
+      rtviewEnable = mkOption {
+        type = bool;
+        default = false;
+        description = ''
+          Whether to enable an RTView client.
+
+          As of node release 9.1 this option has no effect unless node was
+          built with `-f +rtview`.
+
+          Ref:
+          https://github.com/IntersectMBO/cardano-node/pull/5846
+        '';
+      };
+
+      rtviewHost = mkOption {
+        type = str;
+        default = "127.0.0.1";
+        description = ''
+          The host to bind if RTView is enabled.
+        '';
+      };
+
+      rtviewPort = mkOption {
+        type = port;
+        default = 3300;
+        description = ''
+          The port to listen on if RTView is enabled.
+        '';
+      };
+
+      runtimeDir = mkOption {
+        type = str;
+        default = "${cfg.runDirBase}cardano-tracer";
+        description = ''
+          The directory to store any cardano-tracer runtime related data.
+
+          If creating a cardano-tracer socket, it will default to this
+          location.
+        '';
+      };
+
+      runDirBase = mkOption {
+        type = str;
+        default = "/run/";
+        description = ''
+          The base runtime directory for cardano-tracer.
+        '';
+      };
+
+      stateDir = mkOption {
+        type = str;
+        default = "${cfg.stateDirBase}cardano-tracer";
+        description = ''
+          The directory to store any cardano-tracer process related data.
+
+          RTView if enabled will save its state in this directory.
+        '';
+      };
+
+      stateDirBase = mkOption {
+        type = str;
+        default = "/var/lib/";
+        description = ''
+          The base state directory for cardano-tracer.
+        '';
+      };
+
+      user = mkOption {
+        type = str;
+        default = "cardano-node";
+        description = ''
+          The default user to run the systemd service as.
+
+          This user is assumed to already exist.
+        '';
+      };
+
+      verbosity = mkOption {
+        type = nullOr (enum ["Minimum" "ErrorsOnly" "Maximum"]);
+        default = null;
+        description = ''
+          The optional attribute specifies the verbosity level for the
+          cardano-tracer itself.  There are 3 levels:
+
+            Minimum - cardano-tracer will work as silently as possible.
+            ErrorsOnly - messages about problems will be shown in standard output.
+            Maximum - all the messages will be shown in standard output. Caution: the number of messages can be huge.
+
+          If null cardano-tracer will set a default: ErrorsOnly.
+        '';
+      };
+
+      warnRtviewMissing = mkOption {
+        type = nullOr bool;
+        default = null;
+        description = ''
+          Whether to provide a warning if RTView is requested in config but
+          cardano-tracer was built without it.
+
+          If null cardano-tracer will set a default: true if RTView
+          config is provided but cardano-tracer was built without it, false
+          otherwise.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.etc."cardano-tracer/config.json".source = mkIf (isNull cfg.configFile) prettyConfig;
+
+    systemd.services.cardano-tracer = {
+      description = "cardano-tracer service";
+      wantedBy = ["multi-user.target"];
+
+      # If cardano-tracer implements SIGHUP config reload support in the
+      # future, this can be changed to reloadTriggers.
+      restartTriggers = [
+        (
+          if isNull cfg.configFile
+          then prettyConfig
+          else configFile
+        )
+      ];
+
+      environment.HOME = cfg.stateDir;
+
+      path = [cfg.package];
+
+      # Allow up to 10 failures with 30 second restarts in a 15 minute window
+      # before entering failure state which may trigger alerts if set up.
+      startLimitBurst = 10;
+      startLimitIntervalSec = 900;
+
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+
+        LimitNOFILE = "65535";
+
+        WorkingDirectory = cfg.stateDir;
+        StateDirectory = removePrefix cfg.stateDirBase cfg.stateDir;
+        RuntimeDirectory = removePrefix cfg.runDirBase cfg.runtimeDir;
+
+        # Ensure quick restarts on any condition
+        Restart = "always";
+        RestartSec = 30;
+
+        ExecStart = getExe (pkgs.writeShellApplication {
+          name = "cardano-tracer";
+          text = mkScript;
+        });
+      };
+    };
+
+    assertions = [
+      {
+        assertion = (!isNull cfg.acceptingSocket) != (!isNull cfg.connectToSocket);
+        message = "Exactly one of acceptingSocket or connectToSocket must be declared";
+      }
+    ];
+  };
+}

From ddba6f726fdbe1f5a6f94eca6ce40d25f2aa23d8 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Thu, 27 Feb 2025 20:24:38 -0600
Subject: [PATCH 06/37] nixosNodeSvc: improve descriptions

---
 nix/nixos/cardano-node-service.nix | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index eb2b0a06032..3948e28599b 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -208,6 +208,10 @@ in {
           "time-detail"
         ];
         default = "none";
+        description = ''
+          Haskell profiling types which are available and will be applied to
+          the cardano-node binary if declared.
+        '';
       };
 
       eventlog = mkOption {
@@ -232,8 +236,8 @@ in {
         defaultText = "cardano-node packages";
         description = ''
           The cardano-node packages and library that should be used. The main
-          usage is sharing optimization which reduces eval time when service is
-          instantiated multiple times.
+          use case is for a sharing optimization which reduces eval time when
+          cardano node packages are instantiated multiple times.
         '';
       };
 
@@ -258,14 +262,6 @@ in {
         '';
       };
 
-      environments = mkOption {
-        type = types.attrs;
-        default = cfg.cardanoNodePackages.cardanoLib.environments;
-        description = ''
-          The environments cardano-node will connect to.
-        '';
-      };
-
       environment = mkOption {
         type = types.enum (attrNames cfg.environments);
         default = "preview";
@@ -274,6 +270,14 @@ in {
         '';
       };
 
+      environments = mkOption {
+        type = types.attrs;
+        default = cfg.cardanoNodePackages.cardanoLib.environments;
+        description = ''
+          The environments cardano-node will possibly utilize.
+        '';
+      };
+
       isProducer = mkOption {
         type = types.bool;
         default = false;

From b1a7e9cdb2acc6c06d225b8d65c417d1d2846c3a Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Thu, 27 Feb 2025 20:26:16 -0600
Subject: [PATCH 07/37] nixosTracer: add to cardanoNodePackages and module-list

---
 flake.nix                 | 4 ++++
 nix/nixos/module-list.nix | 1 +
 2 files changed, 5 insertions(+)

diff --git a/flake.nix b/flake.nix
index 848cfdda5c0..d4dd57efda2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -494,6 +494,10 @@
           imports = [./nix/nixos/cardano-submit-api-service.nix];
           services.cardano-submit-api.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.system});
         };
+        cardano-tracer = { pkgs, lib, ... }: {
+          imports = [ ./nix/nixos/cardano-tracer-service.nix ];
+          services.cardano-tracer.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.system});
+        };
       };
     };
 }
diff --git a/nix/nixos/module-list.nix b/nix/nixos/module-list.nix
index 4fd14193844..ff7b385401f 100644
--- a/nix/nixos/module-list.nix
+++ b/nix/nixos/module-list.nix
@@ -1,4 +1,5 @@
 [
  ./cardano-node-service.nix
  ./cardano-submit-api-service.nix
+ ./cardano-tracer-service.nix
 ]

From a698a67649585a6f97f544456dcd4eda2474800f Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 28 Feb 2025 16:54:24 -0600
Subject: [PATCH 08/37] nixosNodeSvc: usePeersFromLedgerAfterSlot ->
 useLedgerAfterSlot

* Update usePeersFromLedgerAfterSlot to useLedgerAfterSlot for
  consistent naming with the cardano-node topology file key.
---
 nix/nixos/cardano-node-service.nix | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index 3948e28599b..0847b6f1742 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -24,8 +24,8 @@ let
       advertise = g.advertise or false;
     }) (cfg.publicProducers ++ (cfg.instancePublicProducers i));
     bootstrapPeers = cfg.bootstrapPeers;
-  } // optionalAttrs (cfg.usePeersFromLedgerAfterSlot != null) {
-    useLedgerAfterSlot = cfg.usePeersFromLedgerAfterSlot;
+  } // optionalAttrs (cfg.useLedgerAfterSlot != null) {
+    useLedgerAfterSlot = cfg.useLedgerAfterSlot;
   } // optionalAttrs (cfg.peerSnapshotFile i != null) {
     peerSnapshotFile = cfg.peerSnapshotFile i;
   };
@@ -169,6 +169,12 @@ let
       ''}
       ${toString cmd}'';
 in {
+  imports = [
+    # Update the option name for consistency with the cardano-node topology file key.
+    (mkRenamedOptionModule
+      [ "services" "cardano-node" "usePeersFromLedgerAfterSlot" ] [ "services" "cardano-node" "useLedgerAfterSlot" ])
+  ];
+
   options = {
     services.cardano-node = {
       enable = mkOption {
@@ -529,7 +535,7 @@ in {
         }];
         description = ''
           Routes to public peers. Only used if slot is less than
-          usePeersFromLedgerAfterSlot.
+          useLedgerAfterSlot.
         '';
       };
 
@@ -538,7 +544,7 @@ in {
         default = _: [];
         description = ''
           Routes to public peers. Only used if slot is less than
-          usePeersFromLedgerAfterSlot and specific to a given instance when
+          useLedgerAfterSlot and specific to a given instance when
           multiple instances are used.
         '';
       };
@@ -582,10 +588,10 @@ in {
         '';
       };
 
-      usePeersFromLedgerAfterSlot = mkOption {
+      useLedgerAfterSlot = mkOption {
         type = types.nullOr types.int;
         default = if cfg.kesKey != null then null
-          else envConfig.usePeersFromLedgerAfterSlot or null;
+          else envConfig.useLedgerAfterSlot or null;
         description = ''
           If set, bootstraps from public roots until it reaches given slot,
           then it switches to using the ledger as a source of peers. It

From 815c7149376f7b671e129c6a0d45c894f0c9a1d4 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 5 Mar 2025 15:30:25 -0600
Subject: [PATCH 09/37] bump: update iohkNix for env cfgs w/ new tracing

---
 flake.lock | 7 ++++---
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index ea0482b1e94..a074a721315 100644
--- a/flake.lock
+++ b/flake.lock
@@ -582,15 +582,16 @@
         "sodium": "sodium"
       },
       "locked": {
-        "lastModified": 1745582862,
-        "narHash": "sha256-dMoJ6yHcRvUcB66nofzfAtmVxnDg8oPW3wiVtimXT/o=",
+        "lastModified": 1745972087,
+        "narHash": "sha256-KM6MyIeb1fQcLUzCgI4uk10H7Cj3AthM6AEM36JUM2g=",
         "owner": "input-output-hk",
         "repo": "iohk-nix",
-        "rev": "5c16fdfc45deda7a4ccf964b6dfa1c3cab72f1f7",
+        "rev": "8319cbe0c308ab30b21d544965240966162f55ca",
         "type": "github"
       },
       "original": {
         "owner": "input-output-hk",
+        "ref": "jl/new-tracing",
         "repo": "iohk-nix",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index d4dd57efda2..504e7a6a1e0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -55,7 +55,7 @@
     incl.url = "github:divnix/incl";
 
     iohkNix = {
-      url = "github:input-output-hk/iohk-nix";
+      url = "github:input-output-hk/iohk-nix/jl/new-tracing";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 

From f6c7e012503ca37d02f1bfc73ee6f67456036698 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 5 Mar 2025 15:33:56 -0600
Subject: [PATCH 10/37] nixosNodeSvc: bring lib.types into scope to reduce code

---
 nix/nixos/cardano-node-service.nix | 118 +++++++++++++++--------------
 1 file changed, 60 insertions(+), 58 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index 0847b6f1742..93a2ff26899 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -5,12 +5,14 @@
 
 with lib; with builtins;
 let
+  inherit (types) attrs attrsOf bool enum functionTo listOf package nullOr str;
+
   cfg = config.services.cardano-node;
   envConfig = cfg.environments.${cfg.environment};
   runtimeDir = i : if cfg.runtimeDir i == null then cfg.stateDir i else "${cfg.runDirBase}${removePrefix cfg.runDirBase (cfg.runtimeDir i)}";
   suffixDir = base: i: "${base}${optionalString (i != 0) "-${toString i}"}";
-  nullOrStr = types.nullOr types.str;
-  funcToOr = t: types.either t (types.functionTo t);
+  nullOrStr = nullOr str;
+  funcToOr = t: either t (functionTo t);
 
   newTopology = i: {
     localRoots = map (g: {
@@ -178,7 +180,7 @@ in {
   options = {
     services.cardano-node = {
       enable = mkOption {
-        type = types.bool;
+        type = bool;
         default = false;
         description = ''
           Enable cardano-node, a node implementing ouroboros protocols;
@@ -187,7 +189,7 @@ in {
       };
 
       instances = mkOption {
-        type = types.int;
+        type = int;
         default = 1;
         description = ''
           Number of instances of the service to run.
@@ -195,12 +197,12 @@ in {
       };
 
       script = mkOption {
-        type = types.str;
+        type = str;
         default = mkScript cfg 0;
       };
 
       profiling = mkOption {
-        type = types.enum [
+        type = enum [
           "none"
           "space"
           "space-bio"
@@ -221,7 +223,7 @@ in {
       };
 
       eventlog = mkOption {
-        type = types.bool;
+        type = bool;
         default = false;
         description = ''
           Whether to enable eventlog profiling.
@@ -229,7 +231,7 @@ in {
       };
 
       asserts = mkOption {
-        type = types.bool;
+        type = bool;
         default = false;
         description = ''
           Whether to use an executable with asserts enabled.
@@ -237,7 +239,7 @@ in {
       };
 
       cardanoNodePackages = mkOption {
-        type = types.attrs;
+        type = attrs;
         default = pkgs.cardanoNodePackages or (import ../. { inherit (pkgs) system; }).cardanoNodePackages;
         defaultText = "cardano-node packages";
         description = ''
@@ -248,7 +250,7 @@ in {
       };
 
       package = mkOption {
-        type = types.package;
+        type = package;
         default = if (cfg.profiling != "none")
           then cfg.cardanoNodePackages.cardano-node.passthru.profiled
           else if cfg.asserts then cfg.cardanoNodePackages.cardano-node.passthru.asserted
@@ -260,7 +262,7 @@ in {
       };
 
       executable = mkOption {
-        type = types.str;
+        type = str;
         default = "exec ${cfg.package}/bin/cardano-node";
         defaultText = "cardano-node";
         description = ''
@@ -269,7 +271,7 @@ in {
       };
 
       environment = mkOption {
-        type = types.enum (attrNames cfg.environments);
+        type = enum (attrNames cfg.environments);
         default = "preview";
         description = ''
           The environment cardano-node will connect to.
@@ -277,7 +279,7 @@ in {
       };
 
       environments = mkOption {
-        type = types.attrs;
+        type = attrs;
         default = cfg.cardanoNodePackages.cardanoLib.environments;
         description = ''
           The environments cardano-node will possibly utilize.
@@ -285,7 +287,7 @@ in {
       };
 
       isProducer = mkOption {
-        type = types.bool;
+        type = bool;
         default = false;
         description = ''
           Whether this node is intended to be a producer.
@@ -296,7 +298,7 @@ in {
       # Byron signing/delegation
 
       signingKey = mkOption {
-        type = types.nullOr (types.either types.str types.path);
+        type = nullOr (either str path);
         default = null;
         description = ''
           The signing key.
@@ -304,7 +306,7 @@ in {
       };
 
       delegationCertificate = mkOption {
-        type = types.nullOr (types.either types.str types.path);
+        type = nullOr (either str path);
         default = null;
         description = ''
           The delegation certificate.
@@ -314,14 +316,14 @@ in {
       # Shelley kes/vrf keys and operation cert
 
       kesKey = mkOption {
-        type = types.nullOr (types.either types.str types.path);
+        type = nullOr (either str path);
         default = null;
         description = ''
           The KES or key evolving signature key.
         '';
       };
       vrfKey = mkOption {
-        type = types.nullOr (types.either types.str types.path);
+        type = nullOr (either str path);
         default = null;
         description = ''
           The VRF or verifable random function key.
@@ -329,7 +331,7 @@ in {
       };
 
       operationalCertificate = mkOption {
-        type = types.nullOr (types.either types.str types.path);
+        type = nullOr (either str path);
         default = null;
         description = ''
           The operational certificate.
@@ -337,7 +339,7 @@ in {
       };
 
       hostAddr = mkOption {
-        type = types.str;
+        type = str;
         default = "127.0.0.1";
         description = ''
           The host address to bind to.
@@ -354,7 +356,7 @@ in {
       };
 
       additionalListenStream = mkOption {
-        type = types.functionTo (types.listOf types.str);
+        type = functionTo (listOf str);
         default = _: [];
         description = ''
           A List of additional sockets to listen to. Only available with `systemdSocketActivation`.
@@ -362,7 +364,7 @@ in {
       };
 
       stateDirBase = mkOption {
-        type = types.str;
+        type = str;
         default = "/var/lib/";
         description = ''
           The base directory to store blockchain data.
@@ -370,7 +372,7 @@ in {
       };
 
       stateDir = mkOption {
-        type = funcToOr types.str;
+        type = funcToOr str;
         default = "${cfg.stateDirBase}cardano-node";
         apply = x : if lib.isFunction x then x else i: x;
         description = ''
@@ -379,7 +381,7 @@ in {
       };
 
       runDirBase = mkOption {
-        type = types.str;
+        type = str;
         default = "/run/";
         description = ''
           The base runtime directory.
@@ -396,7 +398,7 @@ in {
       };
 
       databasePath = mkOption {
-        type = funcToOr types.str;
+        type = funcToOr str;
         default = i : "${cfg.stateDir i}/${cfg.dbPrefix i}";
         apply = x : if lib.isFunction x then x else _ : x;
         description = ''The node database path, for each instance.'';
@@ -413,7 +415,7 @@ in {
       };
 
       socketPath = mkOption {
-        type = funcToOr types.str;
+        type = funcToOr str;
         default = i : "${runtimeDir i}/node.socket";
         apply = x : if lib.isFunction x then x else _ : x;
         description = ''A local communication socket path, for each instance.'';
@@ -440,7 +442,7 @@ in {
       };
 
       socketGroup = mkOption {
-        type = types.str;
+        type = str;
         default = "cardano-node";
         description = ''
           The systemd socket group owner.
@@ -450,13 +452,13 @@ in {
       };
 
       systemdSocketActivation = mkOption {
-        type = types.bool;
+        type = bool;
         default = false;
         description = ''Use systemd socket activation'';
       };
 
       extraServiceConfig = mkOption {
-        type = types.functionTo types.attrs
+        type = functionTo attrs
           // {
             merge = loc: foldl' (res: def: i: recursiveUpdate (res i) (def.value i)) (i: {});
           };
@@ -467,7 +469,7 @@ in {
       };
 
       extraSocketConfig = mkOption {
-        type = types.functionTo types.attrs
+        type = functionTo attrs
           // {
             merge = loc: foldl' (res: def: i: recursiveUpdate (res i) (def.value i)) (i: {});
           };
@@ -478,7 +480,7 @@ in {
       };
 
       dbPrefix = mkOption {
-        type = types.either types.str (types.functionTo types.str);
+        type = either str (functionTo str);
         default = suffixDir "db-${cfg.environment}";
         apply = x : if lib.isFunction x then x else suffixDir x;
         description = ''
@@ -488,7 +490,7 @@ in {
       };
 
       port = mkOption {
-        type = types.either types.int types.str;
+        type = either int str;
         default = 3001;
         description = ''
           The port number to listen on.
@@ -496,7 +498,7 @@ in {
       };
 
       shareIpv4port = mkOption {
-        type = types.bool;
+        type = bool;
         default = cfg.systemdSocketActivation;
         description = ''
           Whether instances on the same machine should share an ipv4 port.
@@ -506,7 +508,7 @@ in {
       };
 
       shareIpv6port = mkOption {
-        type = types.bool;
+        type = bool;
         default = cfg.systemdSocketActivation;
         description = ''
           Whether instances on the same machine should share an ipv6 port.
@@ -516,7 +518,7 @@ in {
       };
 
       nodeId = mkOption {
-        type = types.int;
+        type = int;
         default = 0;
         description = ''
           The ID for this node.
@@ -524,7 +526,7 @@ in {
       };
 
       publicProducers = mkOption {
-        type = types.listOf types.attrs;
+        type = listOf attrs;
         default = [];
         example = [{
           accessPoints = [{
@@ -540,7 +542,7 @@ in {
       };
 
       instancePublicProducers = mkOption {
-        type = types.functionTo (types.listOf types.attrs);
+        type = functionTo (listOf attrs);
         default = _: [];
         description = ''
           Routes to public peers. Only used if slot is less than
@@ -550,7 +552,7 @@ in {
       };
 
       producers = mkOption {
-        type = types.listOf types.attrs;
+        type = listOf attrs;
         default = [];
         example = [{
           accessPoints = [{
@@ -564,7 +566,7 @@ in {
       };
 
       instanceProducers = mkOption {
-        type = types.functionTo (types.listOf types.attrs);
+        type = functionTo (listOf attrs);
         default = _: [];
         description = ''
           Static routes to local peers, specific to a given instance when
@@ -573,7 +575,7 @@ in {
       };
 
       useNewTopology = mkOption {
-        type = types.bool;
+        type = bool;
         default = cfg.nodeConfig.EnableP2P or false;
         description = ''
           Use new, peer to peer and ledger peers compatible topology.
@@ -581,7 +583,7 @@ in {
       };
 
       useLegacyTracing = mkOption {
-        type = types.bool;
+        type = bool;
         default = true;
         description = ''
           Use the legacy tracing, based on iohk-monitoring-framework.
@@ -589,7 +591,7 @@ in {
       };
 
       useLedgerAfterSlot = mkOption {
-        type = types.nullOr types.int;
+        type = nullOr int;
         default = if cfg.kesKey != null then null
           else envConfig.useLedgerAfterSlot or null;
         description = ''
@@ -601,7 +603,7 @@ in {
       };
 
       bootstrapPeers = mkOption {
-        type = types.nullOr (types.listOf types.attrs);
+        type = nullOr (listOf attrs);
         default = map (e: {address = e.addr; inherit (e) port;}) envConfig.edgeNodes;
         description = ''
           If set, it will enable bootstrap peers. To disable, set this to null.
@@ -611,7 +613,7 @@ in {
       };
 
       topology = mkOption {
-        type = types.nullOr (types.either types.str types.path);
+        type = nullOr (either str path);
         default = null;
         description = ''
           The cluster topology. If not set the `producers` array is used to
@@ -620,7 +622,7 @@ in {
       };
 
       useSystemdReload = mkOption {
-        type = types.bool;
+        type = bool;
         default = false;
         description = ''
           If set, systemd will reload cardano-node service units instead of restarting them
@@ -635,7 +637,7 @@ in {
       };
 
       nodeConfig = mkOption {
-        type = types.attrs // {
+        type = attrs // {
           merge = loc: foldl' (res: def: recursiveUpdate res def.value) {};
         };
         default = envConfig.nodeConfig;
@@ -643,13 +645,13 @@ in {
       };
 
       targetNumberOfRootPeers = mkOption {
-        type = types.int;
+        type = int;
         default = cfg.nodeConfig.TargetNumberOfRootPeers or 100;
         description = "The target number of root peers the node will know about.";
       };
 
       targetNumberOfKnownPeers = mkOption {
-        type = types.int;
+        type = int;
         default = cfg.nodeConfig.TargetNumberOfKnownPeers or cfg.targetNumberOfRootPeers;
         description = ''
           The target number of known peers, counting root peers and peers known
@@ -658,7 +660,7 @@ in {
       };
 
       targetNumberOfEstablishedPeers = mkOption {
-        type = types.int;
+        type = int;
         default = cfg.nodeConfig.TargetNumberOfEstablishedPeers
           or (cfg.targetNumberOfKnownPeers / 2);
         description = ''
@@ -669,7 +671,7 @@ in {
       };
 
       targetNumberOfActivePeers = mkOption {
-        type = types.int;
+        type = int;
         default = cfg.nodeConfig.TargetNumberOfActivePeers or (2 * cfg.targetNumberOfEstablishedPeers / 5);
         description = ''
           Target number of peers the node is actively downloading headers and
@@ -679,7 +681,7 @@ in {
       };
 
       extraNodeConfig = mkOption {
-        type = types.attrs // {
+        type = attrs // {
           merge = loc: foldl' (res: def: recursiveUpdate res def.value) {};
         };
         default = {};
@@ -687,7 +689,7 @@ in {
       };
 
       extraNodeInstanceConfig = mkOption {
-        type = types.functionTo types.attrs
+        type = functionTo attrs
           // {
             merge = loc: foldl' (res: def: i: recursiveUpdate (res i) (def.value i)) (i: {});
           };
@@ -702,7 +704,7 @@ in {
       };
 
       forceHardForks = mkOption {
-        type = types.attrsOf types.int;
+        type = attrsOf int;
         default = {};
         description = ''
           A developer-oriented dictionary option to force hard forks for given
@@ -712,7 +714,7 @@ in {
       };
 
       withUtxoHdLmdb = mkOption {
-        type = funcToOr types.bool;
+        type = funcToOr bool;
         default = false;
         apply = x: if lib.isFunction x then x else _: x;
         description = ''
@@ -722,19 +724,19 @@ in {
       };
 
       extraArgs = mkOption {
-        type = types.listOf types.str;
+        type = listOf str;
         default = [];
         description = ''Extra CLI args for cardano-node.'';
       };
 
       rts_flags_override = mkOption {
-        type = types.listOf types.str;
+        type = listOf str;
         default = [];
         description = ''RTS flags override from profile content.'';
       };
 
       rtsArgs = mkOption {
-        type = types.listOf types.str;
+        type = listOf str;
         default = [ "-N2" "-I0" "-A16m" "-qg" "-qb" "--disable-delayed-os-memory-return" ];
         apply = args: if (args != [] || cfg.profilingArgs != [] || cfg.rts_flags_override != []) then
           ["+RTS"] ++ cfg.profilingArgs ++ args ++ cfg.rts_flags_override ++ ["-RTS"]
@@ -743,7 +745,7 @@ in {
       };
 
       profilingArgs = mkOption {
-        type = types.listOf types.str;
+        type = listOf str;
         default = let commonProfilingArgs = ["--machine-readable" "-tcardano-node.stats" "-pocardano-node"]
           ++ optional (cfg.eventlog) "-l";
           in if cfg.profiling == "time" then ["-p"] ++ commonProfilingArgs

From 15fe4c63d6c61f05efb2210c6a852ce8c66d0de0 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 7 Mar 2025 14:54:08 -0600
Subject: [PATCH 11/37] imp: fix hydraJobs eval

---
 nix/nixos/cardano-node-service.nix |  2 +-
 nix/workbench/service/tracer.nix   | 18 ++++++++++--------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index 93a2ff26899..b30e01cb8ec 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -5,7 +5,7 @@
 
 with lib; with builtins;
 let
-  inherit (types) attrs attrsOf bool enum functionTo listOf package nullOr str;
+  inherit (types) attrs attrsOf bool either enum functionTo int listOf package nullOr str;
 
   cfg = config.services.cardano-node;
   envConfig = cfg.environments.${cfg.environment};
diff --git a/nix/workbench/service/tracer.nix b/nix/workbench/service/tracer.nix
index 86c666c92bf..127c20fa853 100644
--- a/nix/workbench/service/tracer.nix
+++ b/nix/workbench/service/tracer.nix
@@ -56,14 +56,16 @@ let
           };
         in evalModules {
           prefix = [];
-          modules =    import ../../nixos/module-list.nix
-                    ++ [
-                         (import ../../nixos/cardano-tracer-service.nix pkgs)
-                           systemdCompat
-                           extra
-                           { config._module.args = { inherit pkgs; }; }
-                       ]
-                    ++ [ backend.service-modules.tracer or {} ]
+         modules = [
+                     (import ../../nixos/cardano-node-service.nix)
+                     (import ../../nixos/cardano-submit-api-service.nix)
+                     # (import ../../nixos/cardano-tracer-service.nix)
+                     (import ../../nixos/cardano-tracer-service-legacy.nix pkgs)
+                     systemdCompat
+                     extra
+                     { config._module.args = { inherit pkgs; }; }
+                   ]
+                   ++ [ backend.service-modules.tracer or {} ]
           ;
           # args = { inherit pkgs; };
         }

From 2dce9124df63c1641273599ab4742935db672672 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 7 Mar 2025 19:42:10 -0600
Subject: [PATCH 12/37] benchTracer: add a flag to toggle legacy and new nixos
 svc use

---
 nix/nixos/cardano-tracer-service.nix | 44 ++++++++++++-----
 nix/workbench/service/tracer.nix     | 73 +++++++++++++++++-----------
 2 files changed, 76 insertions(+), 41 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index 62a8f57ed4f..86c2957d623 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -109,6 +109,7 @@ with builtins; let
     echo "..or, once again, in a single line:"
     echo "${toString cmd}"
 
+    trap 'RC="$?"; echo "Service binary 'cardano-tracer' returned status: $RC" >&2; exit $RC' EXIT
     ${toString cmd}
   '';
 in {
@@ -244,7 +245,7 @@ in {
 
       executable = mkOption {
         type = str;
-        default = "exec ${cfg.package}/bin/cardano-tracer";
+        default = "${cfg.package}/bin/cardano-tracer";
         defaultText = "cardano-node";
         description = ''
           The cardano-tracer executable invocation to use.
@@ -359,7 +360,7 @@ in {
       };
 
       metricsComp = mkOption {
-        type = nullOr (attrsOf str);
+        type = nullOr (either str (attrsOf str));
         default = null;
         description = ''
           Passing metric compatability mapping to cardano-tracer can be done as
@@ -369,10 +370,8 @@ in {
           original name and mapped name.  Only one mapping per message is
           supported.
 
-          If such a set is already available as JSON, this also can be imported:
-
-            services.cardano-tracer.metricsComp =
-              builtins.fromJSON (builtins.readFile $PATH);
+          If such a set is already available as JSON in a file, this option can
+          be declared as a string of the path to such file.
 
           Any metrics prefix name declared with `TraceOptionMetricsPrefix` in
           cardano-node config should not be included in the attribute name.
@@ -386,7 +385,7 @@ in {
       };
 
       metricsHelp = mkOption {
-        type = nullOr (attrsOf str);
+        type = nullOr (either str (attrsOf str));
         default = null;
         description = ''
           Passing metric help annotations to cardano-tracer can be done as a an
@@ -394,10 +393,8 @@ in {
           cardano-tracer's internal metric names have to be used as attribute
           names.
 
-          If such a set is already available as JSON, this also can be imported:
-
-            services.cardano-tracer.metricsHelp =
-              builtins.fromJSON (builtins.readFile $PATH);
+          If such a set is already available as JSON in a file, this option can
+          be declared as a string of the path to such file.
 
           Any metrics prefix name declared with `TraceOptionMetricsPrefix` in
           cardano-node config should not be included in the attribute name.
@@ -690,13 +687,26 @@ in {
         '';
       };
 
-      stateDir = mkOption {
+      script = mkOption {
         type = str;
+        default = mkScript;
+        internal = true;
+        description = ''
+          The default nixos generated shell script used in workbench profile
+          generation.
+        '';
+      };
+
+      stateDir = mkOption {
+        type = nullOr str;
         default = "${cfg.stateDirBase}cardano-tracer";
         description = ''
           The directory to store any cardano-tracer process related data.
 
           RTView if enabled will save its state in this directory.
+
+          For non-systemd use cases, this can be set to null or any other
+          string path.
         '';
       };
 
@@ -708,6 +718,16 @@ in {
         '';
       };
 
+      tracerConfig = mkOption {
+        type = attrs;
+        default = tracerConfig;
+        internal = true;
+        description = ''
+          The default nixos tracerConfig attribute set used in workbench
+          profile generation.
+        '';
+      };
+
       user = mkOption {
         type = str;
         default = "cardano-node";
diff --git a/nix/workbench/service/tracer.nix b/nix/workbench/service/tracer.nix
index 127c20fa853..eb39b17fce8 100644
--- a/nix/workbench/service/tracer.nix
+++ b/nix/workbench/service/tracer.nix
@@ -8,6 +8,9 @@
 with pkgs.lib;
 
 let
+  # For testing legacy to new cardano-tracer-service transition
+  # useLegacyTracerService = true;
+  useLegacyTracerService = false;
 
   ## Given an env config, evaluate it and produce the service.
   ##
@@ -17,17 +20,12 @@ let
     let
       tracerConfig =
         {
+          enable = true;
           ## In both the local and remote scenarios, it's most frequently
           ## convenient to act as an acceptor.
           acceptingSocket = "tracer.socket";
           networkMagic = profile.genesis.network_magic;
-          dsmPassthrough = {
-            # rtsOpts = ["-xc"];
-          } // optionalAttrs (profile.tracer.withresources or false) {
-            rtsOpts = [ "-scardano-tracer.gcstats" ];
-          };
           configFile     = "config.json";
-          logRoot        = ".";
           metricsHelp    = "../../../cardano-tracer/configuration/metrics_help.json";
         } // optionalAttrs backend.useCabalRun {
           executable     = "cardano-tracer";
@@ -38,6 +36,25 @@ let
           };
         } // optionalAttrs (profile.tracer.withresources or false) {
           resourceFreq = 1000;
+        } // optionalAttrs useLegacyTracerService {
+          dsmPassthrough = {
+            # rtsOpts = ["-xc"];
+          } // optionalAttrs (profile.tracer.withresources or false) {
+            rtsOpts = [ "-scardano-tracer.gcstats" ];
+          };
+          logRoot    = ".";
+        } // optionalAttrs (!useLegacyTracerService) {
+          logging = [
+            {
+              logRoot    = ".";
+              logMode    = "FileMode";
+              logFormat  = "ForMachine";
+            }
+          ];
+          rtsArgs =
+            # ["-xc"] ++
+            optionals (profile.tracer.withresources or false) ["-scardano-tracer.gcstats"];
+          stateDir = null;
         }
       ;
       systemdCompat.options = {
@@ -47,29 +64,25 @@ let
         assertions = mkOption {};
         environment = mkOption {};
       };
-      eval =
-        let
-          extra = {
-            services.cardano-tracer = {
-              enable = true;
-            } // tracerConfig;
-          };
-        in evalModules {
-          prefix = [];
-         modules = [
-                     (import ../../nixos/cardano-node-service.nix)
-                     (import ../../nixos/cardano-submit-api-service.nix)
-                     # (import ../../nixos/cardano-tracer-service.nix)
-                     (import ../../nixos/cardano-tracer-service-legacy.nix pkgs)
-                     systemdCompat
-                     extra
-                     { config._module.args = { inherit pkgs; }; }
-                   ]
-                   ++ [ backend.service-modules.tracer or {} ]
+      eval = evalModules {
+        prefix = [];
+
+        modules = [
+          (import ../../nixos/cardano-node-service.nix)
+          (import ../../nixos/cardano-submit-api-service.nix)
+          { config._module.args = { inherit pkgs; }; }
+          { services.cardano-tracer = tracerConfig; }
+          systemdCompat
+        ]
+          ++ [ backend.service-modules.tracer or {} ]
+          ++ optionals useLegacyTracerService
+            [ (import ../../nixos/cardano-tracer-service-legacy.nix pkgs) ]
+          ++ optionals (!useLegacyTracerService)
+            [ (import ../../nixos/cardano-tracer-service.nix) ]
           ;
-          # args = { inherit pkgs; };
-        }
-      ;
+
+        # args = { inherit pkgs; };
+      };
     in
       eval.config.services.cardano-tracer;
 
@@ -80,7 +93,9 @@ let
     (nodeSpecs:
     let
       nixosServiceConfig    = tracerConfigServiceConfig;
-      execConfig            = nixosServiceConfig.configJSONfn nixosServiceConfig;
+      execConfig            = if useWorkbenchTracerService
+                              then nixosServiceConfig.configJSONfn nixosServiceConfig
+                              else nixosServiceConfig.tracerConfig;
     in {
       start =
         ''

From 40234e0b4f343a84c4d06828fa6d42cfe2eca6df Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 7 Mar 2025 19:53:03 -0600
Subject: [PATCH 13/37] ci: switch mn cfg to new tracing

---
 configuration/cardano/mainnet-config.json | 173 +++++++++++-----------
 1 file changed, 90 insertions(+), 83 deletions(-)

diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json
index fe46ac57bb0..233832363b3 100644
--- a/configuration/cardano/mainnet-config.json
+++ b/configuration/cardano/mainnet-config.json
@@ -34,91 +34,98 @@
   "TargetNumberOfEstablishedPeers": 40,
   "TargetNumberOfKnownPeers": 150,
   "TargetNumberOfRootPeers": 60,
-  "TraceAcceptPolicy": true,
-  "TraceBlockFetchClient": false,
-  "TraceBlockFetchDecisions": false,
-  "TraceBlockFetchProtocol": false,
-  "TraceBlockFetchProtocolSerialised": false,
-  "TraceBlockFetchServer": false,
-  "TraceChainDb": true,
-  "TraceChainSyncBlockServer": false,
-  "TraceChainSyncClient": false,
-  "TraceChainSyncHeaderServer": false,
-  "TraceChainSyncProtocol": false,
-  "TraceConnectionManager": true,
-  "TraceDNSResolver": true,
-  "TraceDNSSubscription": true,
-  "TraceDiffusionInitialization": true,
-  "TraceErrorPolicy": true,
-  "TraceForge": true,
-  "TraceHandshake": true,
-  "TraceInboundGovernor": true,
-  "TraceIpSubscription": true,
-  "TraceLedgerPeers": true,
-  "TraceLocalChainSyncProtocol": false,
-  "TraceLocalConnectionManager": true,
-  "TraceLocalErrorPolicy": true,
-  "TraceLocalHandshake": true,
-  "TraceLocalRootPeers": true,
-  "TraceLocalTxSubmissionProtocol": false,
-  "TraceLocalTxSubmissionServer": false,
   "TraceMempool": false,
-  "TraceMux": false,
-  "TracePeerSelection": true,
-  "TracePeerSelectionActions": true,
-  "TracePublicRootPeers": true,
-  "TraceServer": true,
-  "TraceTxInbound": false,
-  "TraceTxOutbound": false,
-  "TraceTxSubmissionProtocol": false,
-  "TracingVerbosity": "NormalVerbosity",
-  "TurnOnLogMetrics": true,
-  "TurnOnLogging": true,
-  "UseTraceDispatcher": false,
-  "defaultBackends": [
-    "KatipBK"
-  ],
-  "defaultScribes": [
-    [
-      "StdoutSK",
-      "stdout"
-    ]
-  ],
-  "hasEKG": 12788,
-  "hasPrometheus": [
-    "127.0.0.1",
-    12798
-  ],
-  "minSeverity": "Info",
-  "options": {
-    "mapBackends": {
-      "cardano.node.metrics": [
-        "EKGViewBK"
+  "TraceOptionForwarder": {
+    "connQueueSize": 64,
+    "disconnQueueSize": 128
+  },
+  "TraceOptionMetricsPrefix": "cardano.node.metrics.",
+  "TraceOptionPeerFrequency": 2000,
+  "TraceOptionResourceFrequency": 1000,
+  "TraceOptions": {
+    "": {
+      "backends": [
+        "Stdout HumanFormatColoured",
+        "EKGBackend",
+        "Forwarder"
       ],
-      "cardano.node.resources": [
-        "EKGViewBK"
-      ]
-    },
-    "mapSubtrace": {
-      "cardano.node.metrics": {
-        "subtrace": "Neutral"
-      }
+      "detail": "DNormal",
+      "severity": "Notice"
+    },
+    "BlockFetch.Client.CompletedBlockFetch": {
+      "maxFrequency": 2
+    },
+    "BlockFetch.Decision": {
+      "severity": "Silence"
+    },
+    "ChainDB": {
+      "severity": "Info"
+    },
+    "ChainDB.AddBlockEvent.AddBlockValidation": {
+      "severity": "Silence"
+    },
+    "ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate": {
+      "maxFrequency": 2
+    },
+    "ChainDB.AddBlockEvent.AddedBlockToQueue": {
+      "maxFrequency": 2
+    },
+    "ChainDB.AddBlockEvent.AddedBlockToVolatileDB": {
+      "maxFrequency": 2
+    },
+    "ChainDB.CopyToImmutableDBEvent.CopiedBlockToImmutableDB": {
+      "maxFrequency": 2
+    },
+    "ChainSync.Client": {
+      "severity": "Warning"
+    },
+    "Forge.Loop": {
+      "severity": "Info"
+    },
+    "Forge.StateInfo": {
+      "severity": "Info"
+    },
+    "Mempool": {
+      "severity": "Info"
+    },
+    "Net.ConnectionManager.Remote": {
+      "severity": "Info"
+    },
+    "Net.ConnectionManager.Remote.ConnectionManagerCounters": {
+      "severity": "Silence"
+    },
+    "Net.ErrorPolicy": {
+      "severity": "Info"
+    },
+    "Net.ErrorPolicy.Local": {
+      "severity": "Info"
+    },
+    "Net.InboundGovernor": {
+      "severity": "Warning"
+    },
+    "Net.InboundGovernor.Remote": {
+      "severity": "Info"
+    },
+    "Net.Mux.Remote": {
+      "severity": "Info"
+    },
+    "Net.PeerSelection": {
+      "severity": "Silence"
+    },
+    "Net.Subscription.DNS": {
+      "severity": "Info"
+    },
+    "Net.Subscription.IP": {
+      "severity": "Info"
+    },
+    "Resources": {
+      "severity": "Silence"
+    },
+    "Startup.DiffusionInit": {
+      "severity": "Info"
     }
   },
-  "rotation": {
-    "rpKeepFilesNum": 10,
-    "rpLogLimitBytes": 5000000,
-    "rpMaxAgeHours": 24
-  },
-  "setupBackends": [
-    "KatipBK"
-  ],
-  "setupScribes": [
-    {
-      "scFormat": "ScText",
-      "scKind": "StdoutSK",
-      "scName": "stdout",
-      "scRotation": null
-    }
-  ]
+  "TurnOnLogMetrics": true,
+  "TurnOnLogging": true,
+  "UseTraceDispatcher": true
 }

From d64c4de20a3a94021a7dc77b38646fac234d1149 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 11 Mar 2025 19:33:34 -0500
Subject: [PATCH 14/37] nixosTracerSvc: add ekgRequestFull module option

---
 nix/nixos/cardano-tracer-service.nix | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index 86c2957d623..fed252286c7 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -20,6 +20,7 @@ with builtins; let
       inherit
         (cfg)
         ekgRequestFreq
+        ekgRequestFull
         loRequestNum
         logging
         metricsComp
@@ -205,6 +206,21 @@ in {
         '';
       };
 
+      ekgRequestFull = mkOption {
+        type = nullOr bool;
+        default = null;
+        description = ''
+          When recieving forwarded metrics, cardano-tracer can request a full
+          set of EKG metrics, or a delta from the previous request.
+
+          If true, a full set of metrics will be sent on each request.
+          Otherwise, on false, only a metrics delta to the previous request
+          will be sent.
+
+          If null cardano-tracer will set a default: false.
+        '';
+      };
+
       ekgRequestFreq = mkOption {
         type = nullOr ints.positive;
         default = null;

From 6cc5abb524fbe61161eb3e548051b7d0a5933b8f Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 12 Mar 2025 14:40:32 -0500
Subject: [PATCH 15/37] nixosTracerSvc: mv legacy -> workbench for preferred
 naming

---
 ...nix => cardano-tracer-service-workbench.nix} | 11 ++++++-----
 nix/workbench/service/tracer.nix                | 17 +++++++++--------
 2 files changed, 15 insertions(+), 13 deletions(-)
 rename nix/nixos/{cardano-tracer-service-legacy.nix => cardano-tracer-service-workbench.nix} (92%)

diff --git a/nix/nixos/cardano-tracer-service-legacy.nix b/nix/nixos/cardano-tracer-service-workbench.nix
similarity index 92%
rename from nix/nixos/cardano-tracer-service-legacy.nix
rename to nix/nixos/cardano-tracer-service-workbench.nix
index b803e72021e..21f0c6c6d75 100644
--- a/nix/nixos/cardano-tracer-service-legacy.nix
+++ b/nix/nixos/cardano-tracer-service-workbench.nix
@@ -1,8 +1,9 @@
-# The legacy cardano-tracer module requires commonLib from the overlay
-# which makes a simple non-flake service module import convoluted
-# and also requires knowledge of the cardano-node commonLib functions to use.
-# This makes this legacy module suitable for internal purposes, but not ideal
-# for a general purpose single import cardano-tracer module.
+# This workbench cardano-tracer service module requires commonLib from the
+# cardano-node flake overlay which prevents a simple non-flake service module
+# import and also requires knowledge of the commonLib functions to use.
+#
+# This makes this workbench service module suitable for internal purposes, but
+# not for a general purpose single import cardano-tracer service module.
 #
 # The module at nix/nixos/cardano-tracer-service.nix is the preferred module
 # for general purpose use.
diff --git a/nix/workbench/service/tracer.nix b/nix/workbench/service/tracer.nix
index eb39b17fce8..8b842109999 100644
--- a/nix/workbench/service/tracer.nix
+++ b/nix/workbench/service/tracer.nix
@@ -8,9 +8,10 @@
 with pkgs.lib;
 
 let
-  # For testing legacy to new cardano-tracer-service transition
-  # useLegacyTracerService = true;
-  useLegacyTracerService = false;
+  # For testing the transition from cardano-tracer-service-workbench to
+  # cardano-tracer-service.
+  # useWorkbenchTracerService = true;
+  useWorkbenchTracerService = false;
 
   ## Given an env config, evaluate it and produce the service.
   ##
@@ -36,14 +37,14 @@ let
           };
         } // optionalAttrs (profile.tracer.withresources or false) {
           resourceFreq = 1000;
-        } // optionalAttrs useLegacyTracerService {
+        } // optionalAttrs useWorkbenchTracerService {
           dsmPassthrough = {
             # rtsOpts = ["-xc"];
           } // optionalAttrs (profile.tracer.withresources or false) {
             rtsOpts = [ "-scardano-tracer.gcstats" ];
           };
           logRoot    = ".";
-        } // optionalAttrs (!useLegacyTracerService) {
+        } // optionalAttrs (!useWorkbenchTracerService) {
           logging = [
             {
               logRoot    = ".";
@@ -75,9 +76,9 @@ let
           systemdCompat
         ]
           ++ [ backend.service-modules.tracer or {} ]
-          ++ optionals useLegacyTracerService
-            [ (import ../../nixos/cardano-tracer-service-legacy.nix pkgs) ]
-          ++ optionals (!useLegacyTracerService)
+          ++ optionals useWorkbenchTracerService
+            [ (import ../../nixos/cardano-tracer-service-workbench.nix pkgs) ]
+          ++ optionals (!useWorkbenchTracerService)
             [ (import ../../nixos/cardano-tracer-service.nix) ]
           ;
 

From 98a2d41cbd112c4c2b0ec03dfe8fc195f1f080cd Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 12 Mar 2025 15:02:11 -0500
Subject: [PATCH 16/37] nixosTracerSvc: rm eol metricsComp; update prom port,
 resourceFreq

---
 nix/nixos/cardano-tracer-service.nix | 41 +++++++---------------------
 1 file changed, 10 insertions(+), 31 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index fed252286c7..6cb2a922656 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -23,7 +23,6 @@ with builtins; let
         ekgRequestFull
         loRequestNum
         logging
-        metricsComp
         metricsHelp
         networkMagic
         resourceFreq
@@ -368,36 +367,11 @@ in {
 
       minLogSeverity = mkOption {
         type = nullOr (enum ["Debug" "Info" "Notice" "Warning" "Error" "Critical" "Alert" "Emergency"]);
-        default = null;
-        description = ''
-          Setting this will cause cardano-tracer to drop log messages less
-          severe than the level declared.
-        '';
-      };
-
-      metricsComp = mkOption {
-        type = nullOr (either str (attrsOf str));
-        default = null;
+        default = "Info";
         description = ''
-          Passing metric compatability mapping to cardano-tracer can be done as
-          a an attribute set of strings from metric name to mapped metric names
-          where cardano-tracer's internal metric names have to be used as
-          attribute names.  The metrics are then available with both the
-          original name and mapped name.  Only one mapping per message is
-          supported.
-
-          If such a set is already available as JSON in a file, this option can
-          be declared as a string of the path to such file.
-
-          Any metrics prefix name declared with `TraceOptionMetricsPrefix` in
-          cardano-node config should not be included in the attribute name.
-          Similarly metric type suffixes, such as `.int` or `.real` should also
-          not be included.
+          Setting this will cause cardano-tracer to drop its own log messages
+          that are less severe than the level declared.
         '';
-        example = {
-          "Mempool.TxsInMempool" = "Mempool.TxsInMempool.Mapped";
-          "ChainDB.SlotNum" = "ChainDB.SlotNum.Mapped";
-        };
       };
 
       metricsHelp = mkOption {
@@ -520,15 +494,20 @@ in {
 
       prometheusPort = mkOption {
         type = port;
-        default = 12798;
+        default = 12808;
         description = ''
           The port to listen on if prometheus is enabled.
+
+          Defaults to the legacy prometheus listening port, 12798, plus 10.
+
+          This avoids a conflict with the cardano-node default metrics port
+          binding when PrometheusSimple backend is in use.
         '';
       };
 
       resourceFreq = mkOption {
         type = nullOr ints.positive;
-        default = 1000;
+        default = null;
         description = ''
           The period for tracing cardano-tracer resource usage in milliseconds.
           The frequency will be 1/resourceFreq times per millisecond.  If null

From 46a490223464a8ae03f59380951ec0866a7cea39 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 18 Mar 2025 14:46:53 -0500
Subject: [PATCH 17/37] nixosTracerSvc: add metricsNoSuffix

---
 nix/nixos/cardano-tracer-service.nix | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index 6cb2a922656..ebcf384807d 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -24,6 +24,7 @@ with builtins; let
         loRequestNum
         logging
         metricsHelp
+        metricsNoSuffix
         networkMagic
         resourceFreq
         verbosity
@@ -367,10 +368,12 @@ in {
 
       minLogSeverity = mkOption {
         type = nullOr (enum ["Debug" "Info" "Notice" "Warning" "Error" "Critical" "Alert" "Emergency"]);
-        default = "Info";
+        default = null;
         description = ''
           Setting this will cause cardano-tracer to drop its own log messages
           that are less severe than the level declared.
+
+          If null cardano-tracer will set a default: Info.
         '';
       };
 
@@ -390,6 +393,9 @@ in {
           cardano-node config should not be included in the attribute name.
           Similarly metric type suffixes, such as `.int` or `.real` should also
           not be included.
+
+          The effect of this option applies to prometheus metrics only, ie: not
+          EKG.
         '';
         example = {
           "Mem.resident" = "Kernel-reported RSS (resident set size)";
@@ -397,6 +403,20 @@ in {
         };
       };
 
+      metricsNoSuffix = mkOption {
+        type = nullOr bool;
+        default = null;
+        description = ''
+          If set true, metrics name suffixes, like "_int", will be dropped,
+          increases similiarity with old system names.
+
+          The effect of this option applies to prometheus metrics only, ie: not
+          EKG.
+
+          If null cardano-tracer will set a default: false.
+        '';
+      };
+
       networkMagic = mkOption {
         type = ints.positive;
         default = (fromJSON (readFile cfg.environments.${cfg.environment}.nodeConfig.ShelleyGenesisFile)).networkMagic;
@@ -500,8 +520,8 @@ in {
 
           Defaults to the legacy prometheus listening port, 12798, plus 10.
 
-          This avoids a conflict with the cardano-node default metrics port
-          binding when PrometheusSimple backend is in use.
+          This avoids a conflict with the cardano-node default metrics port binding
+          when PrometheusSimple backend is in use.
         '';
       };
 

From 1fe7feaadacc3956c02ad0be38c27336e1a53d89 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 18 Mar 2025 16:22:33 -0500
Subject: [PATCH 18/37] ci: update mainnet cfg json w/ iohkNix bump and match
 yaml to json

---
 configuration/cardano/mainnet-config.json |  11 +-
 configuration/cardano/mainnet-config.yaml | 353 +++++++++-------------
 2 files changed, 143 insertions(+), 221 deletions(-)

diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json
index 233832363b3..ae99237ba92 100644
--- a/configuration/cardano/mainnet-config.json
+++ b/configuration/cardano/mainnet-config.json
@@ -34,10 +34,10 @@
   "TargetNumberOfEstablishedPeers": 40,
   "TargetNumberOfKnownPeers": 150,
   "TargetNumberOfRootPeers": 60,
-  "TraceMempool": false,
   "TraceOptionForwarder": {
     "connQueueSize": 64,
-    "disconnQueueSize": 128
+    "disconnQueueSize": 128,
+    "maxReconnectDeplay": 30
   },
   "TraceOptionMetricsPrefix": "cardano.node.metrics.",
   "TraceOptionPeerFrequency": 2000,
@@ -45,9 +45,10 @@
   "TraceOptions": {
     "": {
       "backends": [
-        "Stdout HumanFormatColoured",
         "EKGBackend",
-        "Forwarder"
+        "Forwarder",
+        "PrometheusSimple suffix 127.0.0.1 12798",
+        "Stdout HumanFormatColoured"
       ],
       "detail": "DNormal",
       "severity": "Notice"
@@ -86,7 +87,7 @@
       "severity": "Info"
     },
     "Mempool": {
-      "severity": "Info"
+      "severity": "Silence"
     },
     "Net.ConnectionManager.Remote": {
       "severity": "Info"
diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml
index ee201855fcf..09e9984c212 100644
--- a/configuration/cardano/mainnet-config.yaml
+++ b/configuration/cardano/mainnet-config.yaml
@@ -87,231 +87,152 @@ TurnOnLogging: True
 # be directed to the logs or monitoring backends.
 TurnOnLogMetrics: True
 
-# Use legacy tracing
-UseTraceDispatcher: False
-
-# Global logging severity filter. Messages must have at least this severity to
-# pass. Typical values would be Warning, Notice, Info or Debug.
-minSeverity: Info
-
-# Log items can be rendered with more or less verbose detail.
-# Verbosity ranges from MinimalVerbosity, NormalVerbosity to MaximalVerbosity
-TracingVerbosity: NormalVerbosity
-
-# The system supports a number of backends for logging and monitoring.
-# This setting lists the backends that will be available to use in the
-# configuration below. The logging backend is called Katip.
-setupBackends:
-  - KatipBK
-
-# This specifies the default backends that trace output is sent to if it
-# is not specifically configured to be sent to other backends.
-defaultBackends:
-  - KatipBK
-
-# EKG is a simple metrics monitoring system. Uncomment the following to enable
-# this backend and listen on the given local port and point your web browser to
-# http://localhost:12788/
-hasEKG: 12788
-
-# The Prometheus monitoring system exports EKG metrics. Uncomment the following
-# to listen on the given port. Output is provided on
-# http://localhost:12789/metrics
-# hasPrometheus:
-#   - "127.0.0.1"
-#   - 12789
-
-# To enable the legacy 'TraceForwarder' backend, uncomment the following setting. Log
-# items are then forwarded based on an entry in 'mapBackends' to a separate
-# process running a 'TraceAcceptor'.
-# Example using UNIX pipes:
-# traceForwardTo:
-#   tag: RemotePipe
-#   contents: "logs/pipe"
-#
-# Example using Windows named pipes:
-# traceForwardTo:
-#   tag: RemotePipe
-#   contents: "\\\\.\\pipe\\acceptor"
-#
-# Example using network socket:
-# traceForwardTo:
-#   tag: RemoteSocket
-#   contents:
-#     - "127.0.0.1"
-#     - "2997"
-
-# For the Katip logging backend we must set up outputs (called scribes)
-# The available types of scribe are:
-#   FileSK for files
-#   StdoutSK/StderrSK for stdout/stderr
-#   JournalSK for systemd's journal system
-#   DevNullSK ignores all output
-# The scribe output format can be ScText or ScJson. Log rotation settings can
-# be specified in the defaults below or overridden on a per-scribe basis here.
-setupScribes:
-  - scKind: StdoutSK
-    scName: stdout
-    scFormat: ScText
-    scRotation: null
-
-# For the Katip logging backend this specifies the default scribes that trace
-# output is sent to if it is not configured to be sent to other scribes.
-defaultScribes:
-  - - StdoutSK
-    - stdout
-
-# The default file rotation settings for katip scribes, unless overridden
-# in the setupScribes above for specific scribes.
-rotation:
-  rpLogLimitBytes: 5000000
-  rpKeepFilesNum:  10
-  rpMaxAgeHours:   24
-
-
-##### Coarse grained logging control #####
-
-# Trace output from whole subsystems can be enabled/disabled using the following
-# settings. This provides fairly coarse grained control, but it is relatively
-# efficient at filtering out unwanted trace output.
-
-TraceAcceptPolicy: True
-
-# Trace BlockFetch client.
-TraceBlockFetchClient: False
-
-# Trace BlockFetch decisions made by the BlockFetch client.
-TraceBlockFetchDecisions: False
-
-# Trace BlockFetch protocol messages.
-TraceBlockFetchProtocol: False
-
-# Serialised Trace BlockFetch protocol messages.
-TraceBlockFetchProtocolSerialised: False
-
-# Trace BlockFetch server.
-TraceBlockFetchServer: False
-
-# Trace BlockchainTime.
-# TraceBlockchainTime: False
-
-# Verbose tracer of ChainDB
-TraceChainDb: True
-
-# Trace ChainSync client.
-TraceChainSyncClient: False
-
-# Trace ChainSync server (blocks).
-TraceChainSyncBlockServer: False
-
-# Trace ChainSync server (headers).
-TraceChainSyncHeaderServer: False
-
-# Trace ChainSync protocol messages.
-TraceChainSyncProtocol: False
-
-TraceConnectionManager: True
-
-# Trace DNS Resolver messages.
-TraceDNSResolver: True
-
-# Trace DNS Subscription messages.
-TraceDNSSubscription: True
-
-TraceDiffusionInitialization: True
+# Use the modern tracing system instead of the legacy tracing system.
+UseTraceDispatcher: True
 
-# Trace error policy resolution.
-TraceErrorPolicy: True
+# Match the metrics prefix of the legacy tracing system to minimize breaking
+# changes.
+TraceOptionMetricsPrefix: "cardano.node.metrics."
 
-# Trace local error policy resolution.
-TraceLocalErrorPolicy: True
+# Optional node name.  Defaults to hostname if left unset. Ideally this is
+# set in downstream code where the node's name is known.
+# TraceOptionNodeName:
 
-# Trace block forging.
-TraceForge: True
+# The frequency of peer messages.
+TraceOptionPeerFrequency: 2000
 
-# Trace Handshake protocol messages.
-TraceHandshake: True
+# The frequency of resource messages.
+TraceOptionResourceFrequency: 1000
 
-# Trace IP Subscription messages.
-TraceIpSubscription: True
-
-TraceLedgerPeers: True
-
-TraceLocalRootPeers: True
-
-TraceInboundGovernor: True
-
-TracePeerSelection: True
-
-TracePeerSelectionActions: True
-
-TracePublicRootPeers: True
-
-TraceServer: True
-
-
-# Trace local ChainSync protocol messages.
-TraceLocalChainSyncProtocol: False
-
-# Trace local connection manager messages.
-TraceLocalConnectionManager: True
-
-# Trace local Handshake protocol messages.
-TraceLocalHandshake: True
-
-# Trace local TxSubmission protocol messages.
-TraceLocalTxSubmissionProtocol: False
-
-# Trace local TxSubmission server.
-TraceLocalTxSubmissionServer: False
-
-# Trace mempool.
-TraceMempool: False
-
-# Trace Mux Events.
-TraceMux: False
-
-# Trace TxSubmission server (inbound transactions).
-TraceTxInbound: False
-
-# Trace TxSubmission client (outbound transactions).
-TraceTxOutbound: False
-
-# Trace TxSubmission protocol messages.
-TraceTxSubmissionProtocol: False
-
-##### Fine grained logging control #####
-
-# It is also possible to have more fine grained control over filtering of
-# trace output, and to match and route trace output to particular backends.
-# This is less efficient than the coarse trace filters above but provides
-# much more precise control.
-
-options:
+# Queue size control:
+# In case of a missing forwarding service consumer, trace messages will be
+# buffered. This mitigates short forwarding interruptions, or delays at
+# startup time.
+#
+# The queue capacity should thus correlate to the expected log lines per
+# second given a particular tracing configuration to avoid unnecessarily
+# increasing memory footprint.
+#
+# The maxReconnectDelay config option specifies the maximum delay in seconds
+# between (re-)connection attempts of a forwarder.
+TraceOptionForwarder:
+  connQueueSize: 64
+  disconnQueueSize: 128
+  maxReconnectDeplay: 30
+
+# Tracing options for node
+TraceOptions:
+  # The default tracer configuration
+  '':
+    backends:
+    # None, any combination, or all of the following backends can be
+    # enabled, where `EKGBackend` forwards EKG resource status to
+    # cardano-tracer, `Forwarder` forwards message traces and
+    # `PrometheusSimple` serves cardano-node metrics directly from
+    # cardano-node and defaults to use of the same port as in the legacy
+    # tracing system.
+    - EKGBackend
+    - Forwarder
+
+    # Only one of the following can be enabled, which determines for format
+    # of node logging to stdout.
+    - PrometheusSimple suffix 127.0.0.1 12798
+    - Stdout HumanFormatColoured
+
+    # Each tracer can specify the level of details for printing messages.
+    # Options include `DMinimal`, `DNormal`, `DDetailed`, and `DMaximum`. If
+    # no implementation is given, `DNormal` is chosen.
+    detail: DNormal
+
+    # The severity levels, ranging from the least severe (`Debug`) to the
+    # most severe (`Emergency`), provide a framework for ignoring messages
+    # with severity levels below a globally configured severity cutoff.
+    #
+    # The full list of severities are:
+    # `Debug`, `Info`, `Notice`, `Warning`, `Error`, `Critical`, `Alert` and
+    # `Emergency`.
+    #
+    # To enhance severity filtering, there is also the option of `Silence`
+    # which allows for the unconditional silencing of a specific trace,
+    # essentially representing the deactivation of tracers -- a semantic
+    # continuation of the functionality in the legacy system.
+    severity: Notice
+
+  # The following tracer configurations are configured to closely match the
+  # default logging seen in the legacy cardano-node tracing system.
+  BlockFetch.Client.CompletedBlockFetch:
+    # A frequency limit for the number of messages per second may also be
+    # provided for any tracer.
+    maxFrequency: 2
+
+  BlockFetch.Decision:
+    severity: Silence
+
+  ChainDB:
+    severity: Info
+
+  ChainDB.AddBlockEvent.AddBlockValidation:
+    severity: Silence
+
+  ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate:
+    maxFrequency: 2
+
+  ChainDB.AddBlockEvent.AddedBlockToQueue:
+    maxFrequency: 2
+
+  ChainDB.AddBlockEvent.AddedBlockToVolatileDB:
+    maxFrequency: 2
+
+  ChainDB.CopyToImmutableDBEvent.CopiedBlockToImmutableDB:
+    maxFrequency: 2
+
+  ChainSync.Client:
+    severity: Warning
+
+  Forge.Loop:
+    severity: Info
+
+  Forge.StateInfo:
+    severity: Info
+
+  Mempool:
+    severity: Silence
+
+  Net.ConnectionManager.Remote:
+    severity: Info
+
+  Net.ConnectionManager.Remote.ConnectionManagerCounters:
+    severity: Silence
+
+  Net.ErrorPolicy:
+    severity: Info
+
+  Net.ErrorPolicy.Local:
+    severity: Info
+
+  Net.InboundGovernor:
+    severity: Warning
+
+  Net.InboundGovernor.Remote:
+    severity: Info
+
+  Net.Mux.Remote:
+    severity: Info
 
-  # This routes metrics matching specific names to particular backends.
-  # This overrides the 'defaultBackends' listed above. And note that it is
-  # an override and not an extension so anything matched here will not
-  # go to the default backend, only to the explicitly listed backends.
-  mapBackends:
-    cardano.node.metrics:
-      - EKGViewBK
+  Net.PeerSelection:
+    severity: Silence
+
+  Net.Subscription.DNS:
+    severity: Info
 
-    cardano.node.resources:
-      - EKGViewBK
+  Net.Subscription.IP:
+    severity: Info
 
-  # # redirects traced values to a specific scribe which is identified by its
-  # # type and its name, separated by "::":
-  # mapScribes:
-  #   cardano.node.metrics:
-  #     - "FileSK::logs/mainnet.log"
-  mapSubtrace:
-    cardano.node.metrics:
-      subtrace: Neutral
+  Resources:
+    severity: Silence
 
-hasPrometheus:
-  - 127.0.0.1
-  - 12798
+  Startup.DiffusionInit:
+    severity: Info
 
 # Set or unset the mempool capacity override in number of bytes.
 #

From 3358cd7aefd6058d8d172ed9bb1d1f41896712d7 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 18 Mar 2025 18:36:40 -0500
Subject: [PATCH 19/37] ci: add legacy, make updates for new tracing in ci
 tests

---
 Makefile                                      |   4 +-
 .../Cardano/Tracing/NewTracing/Consistency.hs |   2 +-
 .../cardano/mainnet-config-legacy.json        | 110 +++++++++++
 configuration/cardano/mainnet-config.json     |   3 +-
 configuration/cardano/mainnet-config.yaml     |   4 +
 .../cardano/testnet-template-config.json      | 180 ++++++++++--------
 configuration/cardano/update-config-files.sh  |   4 +-
 nix/haskell.nix                               |   2 +-
 ...w-tracing.sh => mainnet-legacy-tracing.sh} |   3 +-
 scripts/lite/mainnet.sh                       |   1 +
 10 files changed, 227 insertions(+), 86 deletions(-)
 create mode 100644 configuration/cardano/mainnet-config-legacy.json
 rename scripts/lite/{mainnet-new-tracing.sh => mainnet-legacy-tracing.sh} (83%)

diff --git a/Makefile b/Makefile
index 7e7b31bc9d0..a2cb3546c6e 100644
--- a/Makefile
+++ b/Makefile
@@ -36,7 +36,7 @@ cli node:
 	cabal --ghc-options="+RTS -qn8 -A32M -RTS" build cardano-$@
 
 trace-documentation:
-	cabal run -- exe:cardano-node trace-documentation --config 'configuration/cardano/mainnet-config-new-tracing.yaml' --output-file 'doc/new-tracing/tracers_doc_generated.md'
+	cabal run -- exe:cardano-node trace-documentation --config 'configuration/cardano/mainnet-config.yaml' --output-file 'doc/new-tracing/tracers_doc_generated.md'
 
 ###
 ### Workbench
@@ -83,7 +83,7 @@ $(eval $(call define_profile_targets,           $(LOCAL_PROFILES)))
 $(eval $(call define_profile_targets_nomadcloud,$(CLOUD_PROFILES)))
 
 # Dynamic local/supervisor profile targets.
-playground-%: 
+playground-%:
 	nix-shell -A 'workbench-shell' --max-jobs 8 --cores 0 --show-trace --argstr profileName $*-${ERA} --argstr backendName supervisor
 
 ###
diff --git a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs
index 7ff1d998edd..9ce11415100 100644
--- a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs
+++ b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs
@@ -25,7 +25,7 @@ tests = do
             [ ( []
               -- This file name should reference the current standard config with new tracing
               , configSubdir
-              , "mainnet-config-new-tracing.json"
+              , "mainnet-config.json"
               )
               ,
               (  []
diff --git a/configuration/cardano/mainnet-config-legacy.json b/configuration/cardano/mainnet-config-legacy.json
new file mode 100644
index 00000000000..27b2e6218b8
--- /dev/null
+++ b/configuration/cardano/mainnet-config-legacy.json
@@ -0,0 +1,110 @@
+{
+  "AlonzoGenesisFile": "mainnet-alonzo-genesis.json",
+  "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874",
+  "ByronGenesisFile": "mainnet-byron-genesis.json",
+  "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb",
+  "ConwayGenesisFile": "mainnet-conway-genesis.json",
+  "ConwayGenesisHash": "15a199f895e461ec0ffc6dd4e4028af28a492ab4e806d39cb674c88f7643ef62",
+  "EnableP2P": true,
+  "LastKnownBlockVersion-Alt": 0,
+  "LastKnownBlockVersion-Major": 3,
+  "LastKnownBlockVersion-Minor": 0,
+  "MaxKnownMajorProtocolVersion": 2,
+  "MinNodeVersion": "10.1.4",
+  "PeerSharing": true,
+  "Protocol": "Cardano",
+  "RequiresNetworkMagic": "RequiresNoMagic",
+  "ShelleyGenesisFile": "mainnet-shelley-genesis.json",
+  "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81",
+  "TargetNumberOfActivePeers": 20,
+  "TargetNumberOfEstablishedPeers": 40,
+  "TargetNumberOfKnownPeers": 150,
+  "TargetNumberOfRootPeers": 60,
+  "TraceAcceptPolicy": true,
+  "TraceBlockFetchClient": false,
+  "TraceBlockFetchDecisions": false,
+  "TraceBlockFetchProtocol": false,
+  "TraceBlockFetchProtocolSerialised": false,
+  "TraceBlockFetchServer": false,
+  "TraceChainDb": true,
+  "TraceChainSyncBlockServer": false,
+  "TraceChainSyncClient": false,
+  "TraceChainSyncHeaderServer": false,
+  "TraceChainSyncProtocol": false,
+  "TraceConnectionManager": true,
+  "TraceDNSResolver": true,
+  "TraceDNSSubscription": true,
+  "TraceDiffusionInitialization": true,
+  "TraceErrorPolicy": true,
+  "TraceForge": true,
+  "TraceHandshake": true,
+  "TraceInboundGovernor": true,
+  "TraceIpSubscription": true,
+  "TraceLedgerPeers": true,
+  "TraceLocalChainSyncProtocol": false,
+  "TraceLocalConnectionManager": true,
+  "TraceLocalErrorPolicy": true,
+  "TraceLocalHandshake": true,
+  "TraceLocalRootPeers": true,
+  "TraceLocalTxSubmissionProtocol": false,
+  "TraceLocalTxSubmissionServer": false,
+  "TraceMempool": false,
+  "TraceMux": false,
+  "TracePeerSelection": true,
+  "TracePeerSelectionActions": true,
+  "TracePublicRootPeers": true,
+  "TraceServer": true,
+  "TraceTxInbound": false,
+  "TraceTxOutbound": false,
+  "TraceTxSubmissionProtocol": false,
+  "TracingVerbosity": "NormalVerbosity",
+  "TurnOnLogMetrics": true,
+  "TurnOnLogging": true,
+  "UseTraceDispatcher": false,
+  "defaultBackends": [
+    "KatipBK"
+  ],
+  "defaultScribes": [
+    [
+      "StdoutSK",
+      "stdout"
+    ]
+  ],
+  "hasEKG": 12788,
+  "hasPrometheus": [
+    "127.0.0.1",
+    12798
+  ],
+  "minSeverity": "Info",
+  "options": {
+    "mapBackends": {
+      "cardano.node.metrics": [
+        "EKGViewBK"
+      ],
+      "cardano.node.resources": [
+        "EKGViewBK"
+      ]
+    },
+    "mapSubtrace": {
+      "cardano.node.metrics": {
+        "subtrace": "Neutral"
+      }
+    }
+  },
+  "rotation": {
+    "rpKeepFilesNum": 10,
+    "rpLogLimitBytes": 5000000,
+    "rpMaxAgeHours": 24
+  },
+  "setupBackends": [
+    "KatipBK"
+  ],
+  "setupScribes": [
+    {
+      "scFormat": "ScText",
+      "scKind": "StdoutSK",
+      "scName": "stdout",
+      "scRotation": null
+    }
+  ]
+}
diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json
index ae99237ba92..e752a96556b 100644
--- a/configuration/cardano/mainnet-config.json
+++ b/configuration/cardano/mainnet-config.json
@@ -124,7 +124,8 @@
     },
     "Startup.DiffusionInit": {
       "severity": "Info"
-    }
+    },
+    "minSeverity": "Critical"
   },
   "TurnOnLogMetrics": true,
   "TurnOnLogging": true,
diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml
index 09e9984c212..f42ff9332ac 100644
--- a/configuration/cardano/mainnet-config.yaml
+++ b/configuration/cardano/mainnet-config.yaml
@@ -234,6 +234,10 @@ TraceOptions:
   Startup.DiffusionInit:
     severity: Info
 
+# Required by the legacy tracing system, this key is still required for
+# cardano-node to start.
+minSeverity = "Critical";
+
 # Set or unset the mempool capacity override in number of bytes.
 #
 # This is intended for testing, and for low-resource machines to run with a smaller mempool.
diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json
index fe7cf6ab0e7..70a2462474f 100644
--- a/configuration/cardano/testnet-template-config.json
+++ b/configuration/cardano/testnet-template-config.json
@@ -1,10 +1,12 @@
 {
-  "ByronGenesisFile": "byron-genesis.json",
-  "ShelleyGenesisFile": "shelley-genesis.json",
   "AlonzoGenesisFile": "alonzo-genesis.json",
-  "ConwayGenesisFile": "conway-genesis.json",
   "ApplicationName": "cardano-sl",
   "ApplicationVersion": 0,
+  "ByronGenesisFile": "byron-genesis.json",
+  "ConwayGenesisFile": "conway-genesis.json",
+  "EnableP2P": true,
+  "ExperimentalHardForksEnabled": true,
+  "ExperimentalProtocolsEnabled": true,
   "LastKnownBlockVersion-Alt": 0,
   "LastKnownBlockVersion-Major": 3,
   "LastKnownBlockVersion-Minor": 1,
@@ -19,83 +21,109 @@
   "PBftSignatureThreshold": 1.1,
   "Protocol": "Cardano",
   "RequiresNetworkMagic": "RequiresMagic",
-  "TestShelleyHardForkAtEpoch": 0,
-  "TestAllegraHardForkAtEpoch": 0,
-  "TestMaryHardForkAtEpoch": 0,
-  "TestAlonzoHardForkAtEpoch": 0,
-  "EnableP2P": true,
-  "ExperimentalProtocolsEnabled": true,
-  "ExperimentalHardForksEnabled": true,
+  "ShelleyGenesisFile": "shelley-genesis.json",
   "TargetNumberOfActivePeers": 20,
   "TargetNumberOfEstablishedPeers": 40,
   "TargetNumberOfKnownPeers": 100,
   "TargetNumberOfRootPeers": 100,
-  "TraceAcceptPolicy": true,
-  "TraceBlockFetchClient": false,
-  "TraceBlockFetchDecisions": false,
-  "TraceBlockFetchProtocol": false,
-  "TraceBlockFetchProtocolSerialised": false,
-  "TraceBlockFetchServer": false,
-  "TraceChainDb": true,
-  "TraceChainSyncBlockServer": false,
-  "TraceChainSyncClient": false,
-  "TraceChainSyncHeaderServer": false,
-  "TraceChainSyncProtocol": false,
-  "TraceConnectionManager": true,
-  "TraceDNSResolver": true,
-  "TraceDNSSubscription": true,
-  "TraceDiffusionInitialization": true,
-  "TraceErrorPolicy": true,
-  "TraceForge": true,
-  "TraceHandshake": false,
-  "TraceInboundGovernor": true,
-  "TraceIpSubscription": true,
-  "TraceLedgerPeers": true,
-  "TraceLocalChainSyncProtocol": false,
-  "TraceLocalErrorPolicy": true,
-  "TraceLocalHandshake": false,
-  "TraceLocalRootPeers": true,
-  "TraceLocalTxSubmissionProtocol": false,
-  "TraceLocalTxSubmissionServer": false,
-  "TraceMempool": false,
-  "TraceMux": false,
-  "TracePeerSelection": true,
-  "TracePeerSelectionActions": true,
-  "TracePublicRootPeers": true,
-  "TraceServer": true,
-  "TraceTxInbound": false,
-  "TraceTxOutbound": false,
-  "TraceTxSubmissionProtocol": false,
-  "TracingVerbosity": "NormalVerbosity",
-  "TurnOnLogMetrics": true,
-  "TurnOnLogging": true,
-  "defaultBackends": ["KatipBK"],
-  "defaultScribes": [["StdoutSK", "cardano"]],
-  "hasEKG": 12788,
-  "hasPrometheus": ["0.0.0.0", 12798],
-  "minSeverity": "Debug",
-  "options": {
-    "mapBackends": {
-      "cardano.node.metrics": ["EKGViewBK"],
-      "cardano.node.resources": ["EKGViewBK"]
-    },
-    "mapSubtrace": {
-      "cardano.node.metrics": {
-        "subtrace": "Neutral"
-      }
-    }
-  },
-  "rotation": {
-    "rpKeepFilesNum": 10,
-    "rpLogLimitBytes": 5000000,
-    "rpMaxAgeHours": 24
+  "TestAllegraHardForkAtEpoch": 0,
+  "TestAlonzoHardForkAtEpoch": 0,
+  "TestMaryHardForkAtEpoch": 0,
+  "TestShelleyHardForkAtEpoch": 0,
+  "TraceOptionForwarder": {
+    "connQueueSize": 64,
+    "disconnQueueSize": 128,
+    "maxReconnectDeplay": 30
   },
-  "setupBackends": ["KatipBK"],
-  "setupScribes": [
-    {
-      "scFormat": "ScText",
-      "scKind": "StdoutSK",
-      "scName": "cardano"
+  "TraceOptionMetricsPrefix": "cardano.node.metrics.",
+  "TraceOptionPeerFrequency": 2000,
+  "TraceOptionResourceFrequency": 1000,
+  "TraceOptions": {
+    "": {
+      "backends": [
+        "EKGBackend",
+        "Forwarder",
+        "PrometheusSimple suffix 127.0.0.1 12798",
+        "Stdout HumanFormatColoured"
+      ],
+      "detail": "DNormal",
+      "severity": "Notice"
+    },
+    "BlockFetch.Client.CompletedBlockFetch": {
+      "maxFrequency": 2.0
+    },
+    "BlockFetch.Decision": {
+      "severity": "Silence"
+    },
+    "ChainDB": {
+      "severity": "Info"
+    },
+    "ChainDB.AddBlockEvent.AddBlockValidation": {
+      "severity": "Silence"
+    },
+    "ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate": {
+      "maxFrequency": 2.0
+    },
+    "ChainDB.AddBlockEvent.AddedBlockToQueue": {
+      "maxFrequency": 2.0
+    },
+    "ChainDB.AddBlockEvent.AddedBlockToVolatileDB": {
+      "maxFrequency": 2.0
+    },
+    "ChainDB.CopyToImmutableDBEvent.CopiedBlockToImmutableDB": {
+      "maxFrequency": 2.0
+    },
+    "ChainSync.Client": {
+      "severity": "Warning"
+    },
+    "Forge.Loop": {
+      "severity": "Info"
+    },
+    "Forge.StateInfo": {
+      "severity": "Info"
+    },
+    "Mempool": {
+      "severity": "Info"
+    },
+    "Net.ConnectionManager.Remote": {
+      "severity": "Info"
+    },
+    "Net.ConnectionManager.Remote.ConnectionManagerCounters": {
+      "severity": "Silence"
+    },
+    "Net.ErrorPolicy": {
+      "severity": "Info"
+    },
+    "Net.ErrorPolicy.Local": {
+      "severity": "Info"
+    },
+    "Net.InboundGovernor": {
+      "severity": "Warning"
+    },
+    "Net.InboundGovernor.Remote": {
+      "severity": "Info"
+    },
+    "Net.Mux.Remote": {
+      "severity": "Info"
+    },
+    "Net.PeerSelection": {
+      "severity": "Silence"
+    },
+    "Net.Subscription.DNS": {
+      "severity": "Info"
+    },
+    "Net.Subscription.IP": {
+      "severity": "Info"
+    },
+    "Resources": {
+      "severity": "Silence"
+    },
+    "Startup.DiffusionInit": {
+      "severity": "Info"
     }
-  ]
+  },
+  "TurnOnLogMetrics": true,
+  "TurnOnLogging": true,
+  "UseTraceDispatcher": true,
+  "minSeverity": "Critical"
 }
diff --git a/configuration/cardano/update-config-files.sh b/configuration/cardano/update-config-files.sh
index a4eb94cc05a..bdbd8ccf08b 100755
--- a/configuration/cardano/update-config-files.sh
+++ b/configuration/cardano/update-config-files.sh
@@ -31,15 +31,13 @@ copyCfg "mainnet-byron-genesis.json"
 copyCfg "mainnet-checkpoints.json"
 copyCfg "mainnet-config.json"
 copyCfg "mainnet-config-bp.json"
+copyCfg "mainnet-config-legacy.json"
 copyCfg "mainnet-conway-genesis.json"
 copyCfg "mainnet-peer-snapshot.json"
 copyCfg "mainnet-shelley-genesis.json"
 copyCfg "mainnet-topology.json"
 copyCfg "mainnet-topology.json"
 
-# IohkNix new tracing config placeholder
-# copyCfg "mainnet-config-new-tracing.json"
-
 # Testnet-template
 copyTmplCfg "alonzo.json"
 copyTmplCfg "byron.json"
diff --git a/nix/haskell.nix b/nix/haskell.nix
index cd58b3485cb..9a481b75d1c 100644
--- a/nix/haskell.nix
+++ b/nix/haskell.nix
@@ -194,7 +194,7 @@ let
               mainnetConfigFiles = [
                 "configuration/cardano/mainnet-config.yaml"
                 "configuration/cardano/mainnet-config.json"
-                "configuration/cardano/mainnet-config-new-tracing.json"
+                "configuration/cardano/mainnet-config-legacy.json"
                 "configuration/cardano/mainnet-byron-genesis.json"
                 "configuration/cardano/mainnet-shelley-genesis.json"
                 "configuration/cardano/mainnet-alonzo-genesis.json"
diff --git a/scripts/lite/mainnet-new-tracing.sh b/scripts/lite/mainnet-legacy-tracing.sh
similarity index 83%
rename from scripts/lite/mainnet-new-tracing.sh
rename to scripts/lite/mainnet-legacy-tracing.sh
index edf95dcb429..3fbe4b3d2ae 100755
--- a/scripts/lite/mainnet-new-tracing.sh
+++ b/scripts/lite/mainnet-legacy-tracing.sh
@@ -14,11 +14,10 @@ mkdir -p "${socket_dir}"
 
 # Launch a node
 cabal run exe:cardano-node -- run \
-  --config "${configuration}/mainnet-config-new-tracing.json" \
+  --config "${configuration}/mainnet-config-legacy.json" \
   --topology "${configuration}/mainnet-topology.json" \
   --database-path "${db_dir}" \
   --socket-path "${socket_dir}/node-1-socket" \
-  --tracer-socket-path-connect "${socket_dir}/tracer.socket" \
   --host-addr "0.0.0.0" \
   --port "3001"
 
diff --git a/scripts/lite/mainnet.sh b/scripts/lite/mainnet.sh
index ef93f734b55..f7c1af884ae 100755
--- a/scripts/lite/mainnet.sh
+++ b/scripts/lite/mainnet.sh
@@ -18,6 +18,7 @@ cabal run exe:cardano-node -- run \
   --topology "${configuration}/mainnet-topology.json" \
   --database-path "${db_dir}" \
   --socket-path "${socket_dir}/node-1-socket" \
+  --tracer-socket-path-connect "${socket_dir}/tracer.socket" \
   --host-addr "0.0.0.0" \
   --port "3001"
 

From 3249bc9c2b185872025198cf3f7a0e3e263a9224 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 19 Mar 2025 11:59:17 -0500
Subject: [PATCH 20/37] nixosNodeSvc: use PrometheusSimple only for 10.3+

---
 configuration/cardano/mainnet-config.json          | 7 +++----
 configuration/cardano/mainnet-config.yaml          | 8 ++++++--
 configuration/cardano/testnet-template-config.json | 1 -
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json
index e752a96556b..c6d4c3b8e4a 100644
--- a/configuration/cardano/mainnet-config.json
+++ b/configuration/cardano/mainnet-config.json
@@ -47,7 +47,6 @@
       "backends": [
         "EKGBackend",
         "Forwarder",
-        "PrometheusSimple suffix 127.0.0.1 12798",
         "Stdout HumanFormatColoured"
       ],
       "detail": "DNormal",
@@ -124,10 +123,10 @@
     },
     "Startup.DiffusionInit": {
       "severity": "Info"
-    },
-    "minSeverity": "Critical"
+    }
   },
   "TurnOnLogMetrics": true,
   "TurnOnLogging": true,
-  "UseTraceDispatcher": true
+  "UseTraceDispatcher": true,
+  "minSeverity": "Critical"
 }
diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml
index f42ff9332ac..d1a7d087ee6 100644
--- a/configuration/cardano/mainnet-config.yaml
+++ b/configuration/cardano/mainnet-config.yaml
@@ -134,10 +134,14 @@ TraceOptions:
     - EKGBackend
     - Forwarder
 
+    # Include for the 10.3 release
+    # - PrometheusSimple suffix 127.0.0.1 12798
+
     # Only one of the following can be enabled, which determines for format
     # of node logging to stdout.
-    - PrometheusSimple suffix 127.0.0.1 12798
     - Stdout HumanFormatColoured
+    # - Stdout HumanFormatUncoloured
+    # - Stdout MachineFormat
 
     # Each tracer can specify the level of details for printing messages.
     # Options include `DMinimal`, `DNormal`, `DDetailed`, and `DMaximum`. If
@@ -236,7 +240,7 @@ TraceOptions:
 
 # Required by the legacy tracing system, this key is still required for
 # cardano-node to start.
-minSeverity = "Critical";
+minSeverity: Critical
 
 # Set or unset the mempool capacity override in number of bytes.
 #
diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json
index 70a2462474f..3c4cc75b074 100644
--- a/configuration/cardano/testnet-template-config.json
+++ b/configuration/cardano/testnet-template-config.json
@@ -43,7 +43,6 @@
       "backends": [
         "EKGBackend",
         "Forwarder",
-        "PrometheusSimple suffix 127.0.0.1 12798",
         "Stdout HumanFormatColoured"
       ],
       "detail": "DNormal",

From c23e3503c25eef6d4e4aa8d7e44e14cb772d3d52 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 19 Mar 2025 16:54:47 -0500
Subject: [PATCH 21/37] nixosNodeSvc: add legacy cfg still required in new
 tracing cfg for tests

---
 configuration/cardano/mainnet-config.json          | 7 ++++++-
 configuration/cardano/mainnet-config.yaml          | 7 +++++++
 configuration/cardano/testnet-template-config.json | 7 ++++++-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json
index c6d4c3b8e4a..ee0adb6ad9e 100644
--- a/configuration/cardano/mainnet-config.json
+++ b/configuration/cardano/mainnet-config.json
@@ -128,5 +128,10 @@
   "TurnOnLogMetrics": true,
   "TurnOnLogging": true,
   "UseTraceDispatcher": true,
-  "minSeverity": "Critical"
+  "defaultBackends": [],
+  "defaultScribes": [],
+  "minSeverity": "Critical",
+  "options": {},
+  "setupBackends": [],
+  "setupScribes": []
 }
diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml
index d1a7d087ee6..a8e2f1eb137 100644
--- a/configuration/cardano/mainnet-config.yaml
+++ b/configuration/cardano/mainnet-config.yaml
@@ -242,6 +242,13 @@ TraceOptions:
 # cardano-node to start.
 minSeverity: Critical
 
+# Required by some legacy tests which may otherwise fail to start.
+defaultBackends: []
+defaultScribes: []
+options: {}
+setupBackends: []
+setupScribes: []
+
 # Set or unset the mempool capacity override in number of bytes.
 #
 # This is intended for testing, and for low-resource machines to run with a smaller mempool.
diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json
index 3c4cc75b074..ae24abea77b 100644
--- a/configuration/cardano/testnet-template-config.json
+++ b/configuration/cardano/testnet-template-config.json
@@ -124,5 +124,10 @@
   "TurnOnLogMetrics": true,
   "TurnOnLogging": true,
   "UseTraceDispatcher": true,
-  "minSeverity": "Critical"
+  "defaultBackends": [],
+  "defaultScribes": [],
+  "minSeverity": "Critical",
+  "options": {},
+  "setupBackends": [],
+  "setupScribes": []
 }

From b2d2778cd56cfd5a92ebbed37092888eef581572 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 19 Mar 2025 17:37:05 -0500
Subject: [PATCH 22/37] nixosTests: update the cardano-node-edge test for new
 tracing

---
 nix/nixos/tests/cardano-node-edge.nix | 87 ++++++++++++++-------------
 1 file changed, 44 insertions(+), 43 deletions(-)

diff --git a/nix/nixos/tests/cardano-node-edge.nix b/nix/nixos/tests/cardano-node-edge.nix
index 8202dad1c99..f07ba9952f9 100644
--- a/nix/nixos/tests/cardano-node-edge.nix
+++ b/nix/nixos/tests/cardano-node-edge.nix
@@ -1,65 +1,66 @@
-{ pkgs, ... }:
-with pkgs;
-{
+{pkgs, ...}:
+with pkgs; let
+  environment = "mainnet";
+in {
   name = "cardano-node-edge-test";
   nodes = {
-    machine = { config, ... }: {
+    machine = {config, ...}: {
       nixpkgs.pkgs = pkgs;
       imports = [
         ../.
       ];
-      services.cardano-node = {
-        enable = true;
-        port = 3001;
-        hostAddr = "127.0.0.1";
-        environment = "mainnet";
-        topology = commonLib.mkEdgeTopologyP2P {
-          edgeNodes = [
-            {
-              addr = "127.0.0.1";
-              port = 3001;
-            }
-          ];
+
+      services = {
+        cardano-node = {
+          inherit environment;
+
+          enable = true;
+          port = 3001;
+          hostAddr = "127.0.0.1";
+          topology = commonLib.mkEdgeTopologyP2P {
+            edgeNodes = [
+              {
+                addr = "127.0.0.1";
+                port = 3001;
+              }
+            ];
+          };
+
+          # Default tracing system logging is to stdout and default prometheus
+          # metrics are exported to localhost on port 12798.
+          nodeConfig = config.services.cardano-node.environments.${environment}.nodeConfig;
         };
-        nodeConfig = config.services.cardano-node.environments.${config.services.cardano-node.environment}.nodeConfig // {
-          hasPrometheus = [ config.services.cardano-node.hostAddr 12798 ];
-          # Use Journald output:
-          setupScribes = [{
-            scKind = "JournalSK";
-            scName = "cardano";
-            scFormat = "ScText";
-          }];
-          defaultScribes = [
-            [
-              "JournalSK"
-              "cardano"
-            ]
-          ];
+
+        cardano-submit-api = {
+          enable = true;
+          port = 8101;
+          network = environment;
+          socketPath = config.services.cardano-node.socketPath 0;
         };
       };
-      systemd.services.cardano-node.serviceConfig.Restart = lib.mkForce "no";
-      services.cardano-submit-api = {
-        enable = true;
-        port = 8101;
-        network = "mainnet";
-        socketPath = config.services.cardano-node.socketPath 0;
+
+      systemd.services = {
+        cardano-node.serviceConfig.Restart = lib.mkForce "no";
+        cardano-submit-api.serviceConfig.SupplementaryGroups = "cardano-node";
       };
-      systemd.services.cardano-submit-api.serviceConfig.SupplementaryGroups = "cardano-node";
     };
   };
   testScript = ''
     start_all()
     machine.wait_for_unit("cardano-node.service")
     machine.wait_for_file("/run/cardano-node/node.socket")
-    machine.wait_for_open_port(12798)
+
+    # Re-enable once PrometheusSimple with node >= 10.3 is available
+    # machine.wait_for_open_port(12798)
+
     machine.wait_for_open_port(3001)
     machine.succeed("systemctl status cardano-node")
-    # FIXME reenable and check the cli syntax when https://github.com/intersectmbo/cardano-node/pull/4664 is merged
-    #machine.succeed(
-    #    "${cardanoNodePackages.cardano-cli}/bin/cardano-cli ping -h 127.0.0.1 -c 1 -q --json | ${jq}/bin/jq -c"
-    #)
+
+    machine.succeed(
+      "${cardanoNodePackages.cardano-cli}/bin/cardano-cli ping -h 127.0.0.1 -c 1 -q --json | ${jq}/bin/jq -c"
+    )
+
     machine.wait_for_open_port(8101)
     machine.succeed("systemctl status cardano-submit-api")
   '';
-
 }

From b68f6194efd60ca58aa2b3c57e325dd916519135 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 21 Mar 2025 18:07:04 -0500
Subject: [PATCH 23/37] docker: tracer basic image, scripts, nixos svc support

---
 flake.nix                            |   8 +-
 nix/docker/tracer.nix                | 124 +++++++++++++++++++++++++++
 nix/nixos/cardano-tracer-service.nix |  54 ++++++++----
 nix/pkgs.nix                         |  40 +++++++--
 nix/scripts-submit-api.nix           |   5 +-
 nix/scripts-tracer.nix               |  45 ++++++++++
 nix/scripts.nix                      |   7 +-
 7 files changed, 251 insertions(+), 32 deletions(-)
 create mode 100644 nix/docker/tracer.nix
 create mode 100644 nix/scripts-tracer.nix

diff --git a/flake.nix b/flake.nix
index 504e7a6a1e0..c00b81b0be3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -232,8 +232,10 @@
               })
             .workbench-profile-run;
         in {
+
           "dockerImage/node" = pkgs.dockerImage;
           "dockerImage/submit-api" = pkgs.submitApiDockerImage;
+          "dockerImage/tracer" = pkgs.tracerDockerImage;
 
           # This is a very light profile, no caching and pinning needed.
           workbench-ci-test = workbenchTest {
@@ -494,7 +496,11 @@
           imports = [./nix/nixos/cardano-submit-api-service.nix];
           services.cardano-submit-api.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.system});
         };
-        cardano-tracer = { pkgs, lib, ... }: {
+        cardano-tracer = {
+          pkgs,
+          lib,
+          ...
+        }: {
           imports = [ ./nix/nixos/cardano-tracer-service.nix ];
           services.cardano-tracer.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.system});
         };
diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix
new file mode 100644
index 00000000000..663bb1598d1
--- /dev/null
+++ b/nix/docker/tracer.nix
@@ -0,0 +1,124 @@
+############################################################################
+# Docker image builder for cardano-tracer
+#
+# To build and load into the Docker engine:
+#
+#   nix build .#dockerImage/tracer
+#   docker load -i result
+#
+# Include `-L` in the nix build command args to see build logs.
+#
+# See the nix/docker/README.md file for details on modes of operation.
+############################################################################
+
+{ pkgs
+, commonLib
+, dockerTools
+
+# The main contents of the image.
+, cardano-tracer
+, scripts
+
+# Set gitrev to null, to ensure the version below is used
+, gitrev ? null
+
+# Other things to include in the image.
+, bashInteractive
+, cacert
+, coreutils
+, curl
+, glibcLocales
+, iana-etc
+, iproute
+, iputils
+, jq
+, socat
+, utillinux
+, writeScriptBin
+, runtimeShell
+, lib
+, exe
+, script
+, repoName ? "ghcr.io/intersectmbo/${exe}"
+}:
+
+let
+
+  # Layer of tools which aren't going to change much between versions.
+  baseImage = dockerTools.buildImage {
+    name = "${repoName}-env";
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = ["/"];
+      paths = [
+        cardano-tracer    # Provide cardano-tracer capability
+        bashInteractive   # Provide the BASH shell
+        cacert            # X.509 certificates of public CA's
+        coreutils         # Basic utilities expected in GNU OS's
+        curl              # CLI tool for transferring files via URLs
+        glibcLocales      # Locale information for the GNU C Library
+        iana-etc          # IANA protocol and port number assignments
+        iproute           # Utilities for controlling TCP/IP networking
+        iputils           # Useful utilities for Linux networking
+        jq                # Lightweight and flexible command-line JSON processor
+        socat             # Utility for bidirectional data transfer
+        utillinux         # System utilities for Linux
+      ];
+    };
+
+    # Set up /tmp (override with TMPDIR variable)
+    extraCommands = ''
+      mkdir -m 0777 tmp
+    '';
+  };
+
+  # For "script" mode, generate scripts for iohk-nix networks which can be
+  # utilized by setting the environment NETWORK variable to the desired
+  # network in the docker command: `-e NETWORK <network>`
+  clusterStatements = lib.concatStringsSep "\n" (lib.mapAttrsToList (env: scripts: let
+    scriptBin = scripts.${script};
+    in ''
+      elif [[ "$NETWORK" == "${env}" ]]; then
+        exec ${scriptBin}/bin/${scriptBin.name} $@
+    '') scripts);
+
+  entry-point = writeScriptBin "entry-point" ''
+    #!${runtimeShell}
+    if [[ -z "$NETWORK" ]]; then
+      exec ${pkgs.${exe}}/bin/${exe} $@
+    ${clusterStatements}
+    else
+      echo "Managed configuration for network "$NETWORK" does not exist"
+    fi
+  '';
+
+in
+  dockerTools.buildImage {
+    name = "${repoName}";
+    tag = "${gitrev}";
+    fromImage = baseImage;
+
+    # Set creation date to build time. Breaks reproducibility.
+    created = "now";
+
+    extraCommands = ''
+      # The "scripts" operation mode of this image, when the NETWORK env var is
+      # set to a valid network, will use the following default directories
+      # mounted at /:
+      mkdir -p data
+      mkdir -p ipc
+
+      # Similarly, make a root level dir for logs:
+      mkdir -p logs
+    '';
+
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = ["/"];
+      paths = [entry-point];
+    };
+
+    config = {
+      EntryPoint = [ "${entry-point}/bin/entry-point" ];
+    };
+  }
diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index ebcf384807d..3415128186f 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -13,7 +13,9 @@ with builtins; let
   configFile =
     if !isNull cfg.configFile
     then cfg.configFile
-    else "/etc/cardano-tracer/config.json";
+    else if isNull cfg.configFilePath
+    then prettyConfig
+    else cfg.configFilePath;
 
   tracerConfig =
     {
@@ -113,6 +115,8 @@ with builtins; let
     trap 'RC="$?"; echo "Service binary 'cardano-tracer' returned status: $RC" >&2; exit $RC' EXIT
     ${toString cmd}
   '';
+
+  runtimeDir = if cfg.runtimeDir == null then cfg.stateDir else "${cfg.runDirBase}${removePrefix cfg.runDirBase cfg.runtimeDir}";
 in {
   options = {
     services.cardano-tracer = {
@@ -134,7 +138,7 @@ in {
 
       acceptingSocket = mkOption {
         type = nullOr (either str path);
-        default = "${cfg.runtimeDir}/cardano-tracer.socket";
+        default = "${runtimeDir}/cardano-tracer.socket";
         description = ''
           If accepting connections from node(s) to a cardano-tracer socket, the
           path.
@@ -168,7 +172,22 @@ in {
         description = ''
           The actual cardano-tracer configuration file. If this option is set
           to null, a configuration file will be built based on the nixos
-          options and symlinked to `/etc/cardano-tracer/config.json`.
+          options.  The target location for a default configuration file will
+          depend on the configFilePath option.
+        '';
+      };
+
+      configFilePath = mkOption {
+        type = nullOr (either str path);
+        default = null;
+        example = "/etc/cardano-tracer/config.json";
+        description = ''
+          If a default config file is to be used, ie the configFile option is null,
+          the configFile will be located in:
+            * The nix store if this option is null, or
+            * Symlinked to an explicitly declared `/etc` prefixed path
+
+          The `/etc` prefix is a requirement of the nixos implementation.
         '';
       };
 
@@ -592,6 +611,13 @@ in {
             };
           };
         }));
+        default = {
+          # Provide some sane defaults
+          rpFrequencySecs = 60;
+          rpKeepFilesNum = 14;
+          rpMaxAgeHours = 24;
+          rpLogLimitBytes = 10 * 1000 * 1000;
+        };
         apply = rot:
           if isNull rot
           then rot
@@ -610,13 +636,6 @@ in {
             ''
             rot
           else rot;
-        default = {
-          # Provide some sane defaults
-          rpFrequencySecs = 60;
-          rpKeepFilesNum = 14;
-          rpMaxAgeHours = 24;
-          rpLogLimitBytes = 10 * 1000 * 1000;
-        };
         description = ''
           The rotation option describes the log rotation operation of
           cardano-tracer and is either null or a submodule with options of
@@ -684,7 +703,7 @@ in {
       };
 
       runtimeDir = mkOption {
-        type = str;
+        type = nullOr str;
         default = "${cfg.runDirBase}cardano-tracer";
         description = ''
           The directory to store any cardano-tracer runtime related data.
@@ -707,7 +726,8 @@ in {
         default = mkScript;
         internal = true;
         description = ''
-          The default nixos generated shell script used in workbench profile
+          The default nixos generated shell script used in the
+          scripts.$ENV.tracer package attribute and workbench profile
           generation.
         '';
       };
@@ -784,7 +804,7 @@ in {
   };
 
   config = mkIf cfg.enable {
-    environment.etc."cardano-tracer/config.json".source = mkIf (isNull cfg.configFile) prettyConfig;
+    environment.etc."cardano-tracer/config.json".source = mkIf (isNull cfg.configFile && isNull cfg.configFilePath) prettyConfig;
 
     systemd.services.cardano-tracer = {
       description = "cardano-tracer service";
@@ -817,7 +837,7 @@ in {
 
         WorkingDirectory = cfg.stateDir;
         StateDirectory = removePrefix cfg.stateDirBase cfg.stateDir;
-        RuntimeDirectory = removePrefix cfg.runDirBase cfg.runtimeDir;
+        RuntimeDirectory = removePrefix cfg.runDirBase runtimeDir;
 
         # Ensure quick restarts on any condition
         Restart = "always";
@@ -833,7 +853,11 @@ in {
     assertions = [
       {
         assertion = (!isNull cfg.acceptingSocket) != (!isNull cfg.connectToSocket);
-        message = "Exactly one of acceptingSocket or connectToSocket must be declared";
+        message = "In services.cardano-tracer, exactly one of acceptingSocket or connectToSocket must be declared";
+      }
+      {
+        assertion = isNull cfg.configFilePath || hasPrefix "/etc/" cfg.configFilePath;
+        message = "In services.cardano-tracer.configFilePath a non-null path must be prefixed with `/etc/`";
       }
     ];
   };
diff --git a/nix/pkgs.nix b/nix/pkgs.nix
index 9936e866a2d..7695e1b872a 100644
--- a/nix/pkgs.nix
+++ b/nix/pkgs.nix
@@ -1,7 +1,8 @@
-# our packages overlay
+# Our packages overlay
 final: prev:
 
 let
+  inherit (builtins) foldl' fromJSON listToAttrs map readFile;
   inherit (final) pkgs;
   inherit (prev.pkgs) lib;
   inherit (prev) customConfig;
@@ -35,7 +36,7 @@ in with final;
 
   cabal = haskell-nix.cabal-install.${compiler-nix-name};
 
-  # TODO Use `compiler-nix-name` here instead of `"ghc928"`
+  # TODO: Use `compiler-nix-name` here instead of `"ghc928"`
   # and fix the resulting `hlint` 3.6.1 warnings.
   hlint = haskell-nix.tool "ghc928" "hlint" ({config, ...}: {
     version = {
@@ -51,7 +52,7 @@ in with final;
     index-state = "2024-12-24T12:56:48Z";
   };
 
-  # The ghc-hls point release compatibility table is documented at
+  # The ghc-hls point release compatibility table is documented at:
   # https://haskell-language-server.readthedocs.io/en/latest/support/ghc-version-support.html
   haskell-language-server = haskell-nix.tool compiler-nix-name "haskell-language-server" rec {
     src = {
@@ -64,7 +65,7 @@ in with final;
       ghc964 = haskell-nix.sources."hls-2.6";
       ghc981 = haskell-nix.sources."hls-2.6";
     }.${compiler-nix-name} or haskell-nix.sources."hls-2.8";
-    cabalProject = builtins.readFile (src + "/cabal.project");
+    cabalProject = readFile (src + "/cabal.project");
     sha256map."https://github.com/pepeiborra/ekg-json"."7a0af7a8fd38045fd15fb13445bdcc7085325460" = "sha256-fVwKxGgM0S4Kv/4egVAAiAjV7QB5PBqMVMCfsv7otIQ=";
   };
 
@@ -87,8 +88,11 @@ in with final;
 
   cardanolib-py = callPackage ./cardanolib-py { };
 
-  scripts = lib.recursiveUpdate (import ./scripts.nix { inherit pkgs; })
-    (import ./scripts-submit-api.nix { inherit pkgs; });
+  scripts = foldl' lib.recursiveUpdate {} [
+    (import ./scripts.nix { inherit pkgs; })
+    (import ./scripts-submit-api.nix { inherit pkgs; })
+    (import ./scripts-tracer.nix { inherit pkgs; })
+  ];
 
   clusterTests = import ./workbench/tests { inherit pkgs; };
 
@@ -125,12 +129,30 @@ in with final;
       script = "submit-api";
     };
 
+  tracerDockerImage =
+    let
+      defaultConfig = {
+        # Can we easily modify to the connecting socket scenario now?
+        # Probably need environment overrides for both node and tracer images.
+        acceptingSocket = "/ipc/cardano-tracer.socket";
+        stateDir = "/logs";
+      };
+    in
+    callPackage ./docker/tracer.nix {
+      exe = "cardano-tracer";
+      scripts = import ./scripts-tracer.nix {
+        inherit pkgs;
+        customConfigs = [ defaultConfig customConfig ];
+      };
+      script = "tracer";
+    };
+
   all-profiles-json = workbench.profile-names-json;
 
   # The profile data and backend data of the cloud / "*-nomadperf" profiles.
   # Useful to mix workbench and cardano-node commits, mostly because of scripts.
-  profile-data-nomadperf = builtins.listToAttrs (
-    builtins.map
+  profile-data-nomadperf = listToAttrs (
+    map
     (cloudName:
       # Only Conway era cloud profiles are flake outputs.
       let profileName = "${cloudName}-coay";
@@ -169,7 +191,7 @@ in with final;
         }
     )
     # Fetch all "*-nomadperf" profiles.
-    (__fromJSON (__readFile
+    (fromJSON (readFile
       (pkgs.runCommand "cardano-profile-names-cloud-noera" {} ''
         ${cardanoNodePackages.cardano-profile}/bin/cardano-profile names-cloud-noera > $out
       ''
diff --git a/nix/scripts-submit-api.nix b/nix/scripts-submit-api.nix
index 7a549fa8673..3e24c7104f7 100644
--- a/nix/scripts-submit-api.nix
+++ b/nix/scripts-submit-api.nix
@@ -25,9 +25,8 @@ let
     passthru = { inherit service; };
   };
 
-  # Until complete removal of some iohk-nix deprecated environments which have
-  # dangling dependencies such as shelley_qa, explicitly declare the
-  # environments we we want included.
+  # Allow list envs we specifically want scripts for as others in iohk-nix may
+  # be transient test networks.
   environments' = pkgs.lib.getAttrs [ "mainnet" "preprod" "preview" ] environments;
 
   scripts = forEnvironmentsCustom (environment: recurseIntoAttrs {
diff --git a/nix/scripts-tracer.nix b/nix/scripts-tracer.nix
new file mode 100644
index 00000000000..513fbec8822
--- /dev/null
+++ b/nix/scripts-tracer.nix
@@ -0,0 +1,45 @@
+{ pkgs
+, customConfigs ? [ pkgs.customConfig ]
+}:
+with pkgs.commonLib;
+let
+  mkScript = envConfig: let
+    service = evalService {
+      inherit pkgs customConfigs;
+      serviceName = "cardano-tracer";
+      modules = [
+        ./nixos/cardano-tracer-service.nix
+        ({config, ...}: let cfg = config.services.cardano-tracer; in {
+          services.cardano-tracer = rec {
+            environment = mkDefault envConfig.name;
+            stateDir = mkDefault "state-tracer-${cfg.environment}";
+            runtimeDir = mkDefault null;
+            logging = mkDefault [
+              {
+                logRoot = stateDir;
+                logMode = "FileMode";
+                logFormat = "ForHuman";
+              }
+            ];
+          };
+        })
+      ];
+    };
+  scriptBin = pkgs.writeScriptBin "cardano-tracer-${service.environment}" ''
+    #!${pkgs.runtimeShell}
+    export PATH=$PATH:${makeBinPath [ pkgs.coreutils ]}
+    set -euo pipefail
+    mkdir -p "$(dirname "${service.acceptingSocket or service.connectToSocket}")"
+    ${service.script} $@
+  '';
+  in scriptBin // {
+    exePath = "${scriptBin}/bin/cardano-tracer-${service.environment}";
+  };
+
+  # Allow list envs we specifically want scripts for as others in iohk-nix may
+  # be transient test networks.
+  environments' = pkgs.lib.getAttrs [ "mainnet" "preprod" "preview" ] environments;
+
+in forEnvironmentsCustom (environment: recurseIntoAttrs {
+  tracer = mkScript environment;
+}) environments'
diff --git a/nix/scripts.nix b/nix/scripts.nix
index 9188f04c200..421d09f6530 100644
--- a/nix/scripts.nix
+++ b/nix/scripts.nix
@@ -33,11 +33,10 @@ let
     exePath = "${scriptBin}/bin/cardano-node-${service.environment}";
   };
 
-  # Until complete removal of some iohk-nix deprecated environments which have
-  # dangling dependencies such as shelley_qa, explicitly declare the
-  # environments we we want included.
+  # Allow list envs we specifically want scripts for as others in iohk-nix may
+  # be transient test networks.
   environments' = pkgs.lib.getAttrs [ "mainnet" "preprod" "preview" ] environments;
 
-in forEnvironmentsCustom (environment: recurseIntoAttrs rec {
+in forEnvironmentsCustom (environment: recurseIntoAttrs {
   node = mkScript environment;
 }) environments'

From 0c020fa04c3d5db4de500e35b244d2a9ba331241 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Sat, 29 Mar 2025 00:30:40 -0500
Subject: [PATCH 24/37] bump: iohkNix for cardano-tracer env cfgs; adjust nixos
 svc default to new tracing

---
 flake.nix                            |  3 +--
 nix/docker/submit-api.nix            |  2 +-
 nix/nixos/cardano-node-service.nix   |  2 +-
 nix/nixos/cardano-tracer-service.nix |  2 +-
 nix/pkgs.nix                         | 11 ++++++++---
 5 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/flake.nix b/flake.nix
index c00b81b0be3..c246986e72b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -232,7 +232,6 @@
               })
             .workbench-profile-run;
         in {
-
           "dockerImage/node" = pkgs.dockerImage;
           "dockerImage/submit-api" = pkgs.submitApiDockerImage;
           "dockerImage/tracer" = pkgs.tracerDockerImage;
@@ -501,7 +500,7 @@
           lib,
           ...
         }: {
-          imports = [ ./nix/nixos/cardano-tracer-service.nix ];
+          imports = [./nix/nixos/cardano-tracer-service.nix];
           services.cardano-tracer.cardanoNodePackages = lib.mkDefault (mkCardanoNodePackages flake.project.${pkgs.system});
         };
       };
diff --git a/nix/docker/submit-api.nix b/nix/docker/submit-api.nix
index 3bc39f57724..038591096b0 100644
--- a/nix/docker/submit-api.nix
+++ b/nix/docker/submit-api.nix
@@ -82,7 +82,7 @@ let
       exec ${pkgs.${exe}}/bin/${exe} $@
     ${clusterStatements}
     else
-      echo "Managed configuration for network "$NETWORK" does not exist"
+      echo "[Error] Managed configuration for network "$NETWORK" does not exist"
     fi
   '';
 
diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index b30e01cb8ec..4cde4d41931 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -584,7 +584,7 @@ in {
 
       useLegacyTracing = mkOption {
         type = bool;
-        default = true;
+        default = false;
         description = ''
           Use the legacy tracing, based on iohk-monitoring-framework.
         '';
diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index 3415128186f..257dc4757cc 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -519,7 +519,7 @@ in {
         type = bool;
         default = true;
         description = ''
-          Whether to enable a prometheus export of EKG metrics.
+          Whether to enable a prometheus export of metrics.
         '';
       };
 
diff --git a/nix/pkgs.nix b/nix/pkgs.nix
index 7695e1b872a..1a0cf8a9ee9 100644
--- a/nix/pkgs.nix
+++ b/nix/pkgs.nix
@@ -131,11 +131,16 @@ in with final;
 
   tracerDockerImage =
     let
-      defaultConfig = {
-        # Can we easily modify to the connecting socket scenario now?
-        # Probably need environment overrides for both node and tracer images.
+      defaultConfig = rec {
         acceptingSocket = "/ipc/cardano-tracer.socket";
         stateDir = "/logs";
+        logging = [
+          {
+            logRoot = stateDir;
+            logMode = "FileMode";
+            logFormat = "ForHuman";
+          }
+        ];
       };
     in
     callPackage ./docker/tracer.nix {

From cc5c0891ba28fb735e748c40ca0ddceb6b300733 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Sat, 29 Mar 2025 00:34:20 -0500
Subject: [PATCH 25/37] nodeOci: move context to container specific path and
 fix blank arg bug w/ filteredOpts

---
 nix/docker/context/{ => node}/bin/entrypoint |  0
 nix/docker/context/{ => node}/bin/run-client |  0
 nix/docker/context/{ => node}/bin/run-node   | 12 +++++-------
 nix/docker/default.nix                       |  4 ++--
 4 files changed, 7 insertions(+), 9 deletions(-)
 rename nix/docker/context/{ => node}/bin/entrypoint (100%)
 rename nix/docker/context/{ => node}/bin/run-client (100%)
 rename nix/docker/context/{ => node}/bin/run-node (99%)

diff --git a/nix/docker/context/bin/entrypoint b/nix/docker/context/node/bin/entrypoint
similarity index 100%
rename from nix/docker/context/bin/entrypoint
rename to nix/docker/context/node/bin/entrypoint
diff --git a/nix/docker/context/bin/run-client b/nix/docker/context/node/bin/run-client
similarity index 100%
rename from nix/docker/context/bin/run-client
rename to nix/docker/context/node/bin/run-client
diff --git a/nix/docker/context/bin/run-node b/nix/docker/context/node/bin/run-node
similarity index 99%
rename from nix/docker/context/bin/run-node
rename to nix/docker/context/node/bin/run-node
index a89c2b97415..6f0988c692e 100755
--- a/nix/docker/context/bin/run-node
+++ b/nix/docker/context/node/bin/run-node
@@ -92,19 +92,18 @@ cat << EOF > /usr/local/bin/env
 #!/usr/bin/env bash
 
 # Docker run ENV vars
-CARDANO_CONFIG="$CARDANO_CONFIG"
-CARDANO_TOPOLOGY="$CARDANO_TOPOLOGY"
 CARDANO_BIND_ADDR="$CARDANO_BIND_ADDR"
-CARDANO_PORT=$CARDANO_PORT
+CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER
+CARDANO_CONFIG="$CARDANO_CONFIG"
 CARDANO_DATABASE_PATH="$CARDANO_DATABASE_PATH"
-CARDANO_SOCKET_PATH="$CARDANO_SOCKET_PATH"
 CARDANO_LOG_DIR="$CARDANO_LOG_DIR"
+CARDANO_PORT=$CARDANO_PORT
+CARDANO_SOCKET_PATH="$CARDANO_SOCKET_PATH"
+CARDANO_TOPOLOGY="$CARDANO_TOPOLOGY"
 
 CARDANO_PUBLIC_IP="$CARDANO_PUBLIC_IP"
 CARDANO_CUSTOM_PEERS="$CARDANO_CUSTOM_PEERS"
 
-CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER
-
 # Mapping for topologyUpdater
 CNODE_HOSTNAME="$CARDANO_PUBLIC_IP"
 CNODE_PORT=$CARDANO_PORT
@@ -169,7 +168,6 @@ runBlockProducerNode () {
 shift
 
 # Override default values with explicit options
-
 options=("$@")
 
 for i in "${!options[@]}"
diff --git a/nix/docker/default.nix b/nix/docker/default.nix
index 1cc75c9f953..2ecc3924d21 100644
--- a/nix/docker/default.nix
+++ b/nix/docker/default.nix
@@ -93,7 +93,7 @@ let
   '';
 
   # The docker context with static content
-  context = ./context;
+  context = ./context/node;
 
   genCfgs = let
     environments' = lib.getAttrs [ "mainnet" "preprod" "preview" ] commonLib.environments;
@@ -113,7 +113,7 @@ let
         done
 
         # Adjust genesis file, config and config-bp refs
-        for i in config config-bp db-sync-config; do
+        for i in config config-bp config-legacy config-bp-legacy db-sync-config; do
           if [ -f "$out/config/$ENV/$i.json" ]; then
             sed -i "s|\"$ENV-|\"|g" "$out/config/$ENV/$i.json"
           fi

From 34d86d3134939fbe01afab762929a75f0a6e9abe Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Sat, 29 Mar 2025 00:40:24 -0500
Subject: [PATCH 26/37] tracerOci: add context specific tracer entrypoint and
 support scripts

---
 nix/docker/README.md                     |  2 +-
 nix/docker/context/tracer/bin/entrypoint | 75 +++++++++++++++++++
 nix/docker/context/tracer/bin/run-tracer | 93 ++++++++++++++++++++++++
 nix/docker/tracer.nix                    | 69 ++++++++++++++----
 4 files changed, 225 insertions(+), 14 deletions(-)
 create mode 100755 nix/docker/context/tracer/bin/entrypoint
 create mode 100755 nix/docker/context/tracer/bin/run-tracer

diff --git a/nix/docker/README.md b/nix/docker/README.md
index 50e77c7a9d8..8be3c5f2f0f 100644
--- a/nix/docker/README.md
+++ b/nix/docker/README.md
@@ -61,7 +61,7 @@ To launch cardano-node with a custom configuration, "custom" mode, provide
 entrypoint args starting with `run` and:
 * Leave the `NETWORK` env variable unset
 * Optionally include additional cardano-node args to the entrypoint after `run`
-* Optionally include environment variables interpreted by [nix/docker/context/bin/run-node](context/bin/run-node),
+* Optionally include environment variables interpreted by [nix/docker/context/node/bin/run-node](context/node/bin/run-node),
   or `/usr/local/bin/run-node` in the container
 
 For example, launch a custom cardano-node container using cardano-node args and
diff --git a/nix/docker/context/tracer/bin/entrypoint b/nix/docker/context/tracer/bin/entrypoint
new file mode 100755
index 00000000000..df4fce2aa43
--- /dev/null
+++ b/nix/docker/context/tracer/bin/entrypoint
@@ -0,0 +1,75 @@
+#!/bin/env bash
+
+[[ -n $DEBUG ]] && set -x
+
+# If the NETWORK env var is set to a valid cardano network, pre-defined
+# configuration will be used.
+if [[ -n $NETWORK ]]; then
+
+  # If CARDANO_CONFIG_JSON_MERGE env var is set, iohk-nix
+  # pre-defined NETWORK configuration will be used as a starting point and
+  # merged with custom configuration provided as json in the environment
+  # variable(s).
+  if [[ -n $CARDANO_CONFIG_JSON_MERGE ]]; then
+
+    CFG="/opt/cardano/config"
+    if ! [[ -f $CFG/$NETWORK/tracer-config.json ]]; then
+      echo "Network \"$NETWORK\" doesn't appear to have expected base configuration available at:"
+      echo "  $CFG/$NETWORK/tracer-config.json"
+      echo
+      echo "Please check that the NETWORK environment variable is set to a valid cardano network name."
+      exit 1
+    fi
+
+    # Do a recursive deep merge of iohk-nix NETWORK config and/or topology with
+    # the supplied json merge environment variable(s).
+    #
+    # In a jq deep merge, arrays are replaced, primitive values in the second
+    # object override the first, different types for the same key result in
+    # full replacement and null values persist.
+    if [[ -n $CARDANO_CONFIG_JSON_MERGE ]]; then
+      jq -S \
+        --argjson deepMerge "$CARDANO_CONFIG_JSON_MERGE" \
+        '. * $deepMerge' \
+        < "$CFG/$NETWORK/tracer-config.json" \
+        > "$CFG/$NETWORK/tracer-config-merged.json"
+      export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config-merged.json"
+    else
+      export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config.json"
+    fi
+
+    # Run cardano-tracer using iohk-nix base config merged with provided custom
+    # config for the requested NETWORK.
+    unset NETWORK
+    exec /usr/local/bin/run-tracer "$@"
+
+  else
+    # Run cardano-tracer using "scripts" mode for the requested NETWORK.
+    exec /usr/local/bin/run-network "$@"
+  fi
+
+elif [[ $1 == "run" ]]; then
+  # Run cardano-tracer using "custom" mode.
+  exec /usr/local/bin/run-tracer "$@"
+
+else
+
+  echo "Nothing to do!  Available modes of operation are:"
+  echo
+  echo "Scripts mode:"
+  echo "  * Set the NETWORK env var to a valid cardano network, such as mainnet to use default network config."
+  echo
+  echo "Custom mode:"
+  echo "  * Leave the NETWORK env var unset and provide entrypoint args starting with \"run\" and:"
+  echo "    * Optionally set environment variables interpreted by /usr/local/bin/run-tracer."
+  echo "    * Optionally include additional cardano-tracer args to the entrypoint after \"run\"."
+  echo
+  echo "Merge mode:"
+  echo "  * Set the NETWORK env var to a valid cardano network, such as mainnet, and"
+  echo "    set the CARDANO_CONFIG_JSON_MERGE env var with valid json to run cardano-tracer"
+  echo "    with deep merged base NETWORK and custom config."
+  echo "  * The extra environment variables and cardano-tracer args that can be used in custom mode"
+  echo "    are also available in merge mode."
+  exit 1
+
+fi
diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer
new file mode 100755
index 00000000000..dd3e4b92f99
--- /dev/null
+++ b/nix/docker/context/tracer/bin/run-tracer
@@ -0,0 +1,93 @@
+#!/bin/env bash
+set -eo pipefail
+
+echo "Running the cardano tracer ..."
+
+[[ -n $DEBUG ]] && set -x
+
+# Define a few defaults
+CARDANO_CONFIG_BASE="/opt/cardano/config"
+
+if [[ -z $CARDANO_CONFIG ]]; then
+  CARDANO_CONFIG="$CARDANO_CONFIG_BASE/mainnet/tracer-config.json"
+fi
+
+#####################################################################
+#
+# Print run environment
+#
+printRunEnv () {
+
+  echo "CARDANO_CONFIG=$CARDANO_CONFIG"
+  echo "CARDANO_MIN_LOG_SEVERITY=$CARDANO_MIN_LOG_SEVERITY"
+  echo "CARDANO_STATE_DIR=$CARDANO_STATE_DIR"
+}
+
+#####################################################################
+#
+# Write root env file
+#
+writeRootEnv () {
+
+cat << EOF > /usr/local/bin/env
+#!/usr/bin/env bash
+
+# Docker run ENV vars
+CARDANO_CONFIG="$CARDANO_CONFIG"
+CARDANO_MIN_LOG_SEVERITY="$CARDANO_MIN_LOG_SEVERITY"
+CARDANO_STATE_DIR="$CARDANO_STATE_DIR"
+EOF
+}
+
+#####################################################################
+#
+# Run the relay node in the background
+#
+runTracer () {
+
+  effopts=(
+    "--config" "$CARDANO_CONFIG"
+    "--state-dir" "$CARDANO_STATE_DIR"
+  )
+
+  effopts+=("${filteredOpts[@]}")
+
+  echo "cardano-tracer ${effopts[*]}"
+  exec /usr/local/bin/cardano-tracer "${effopts[@]}"
+}
+
+# Shift the first option by one index
+shift
+
+# Override default values with explicit options
+
+options=("$@")
+
+for i in "${!options[@]}"
+do
+    j=$((i + 1))
+    key=${options[i]}
+    val=${options[j]}
+    found=false
+
+    # echo "$i/$j: ${key} ${val}"
+
+    case ${key} in
+      --config) CARDANO_CONFIG=${val}; found=true;;
+      --state-dir) CARDANO_STATE_DIR=${val}; found=true;;
+      --min-log-severity) CARDANO_MIN_LOG_SEVERITY=${val}; found=true;;
+    esac
+
+    if [[ $found == true ]]; then
+      options[i]="";
+      options[j]="";
+    fi
+done
+
+# Filter blank args from match removal above
+filteredOpts=(); for arg in "${options[@]}"; do [[ $arg ]] && filteredOpts+=("$arg"); done
+
+printRunEnv
+writeRootEnv
+
+runTracer
diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix
index 663bb1598d1..e6089d18cd9 100644
--- a/nix/docker/tracer.nix
+++ b/nix/docker/tracer.nix
@@ -34,8 +34,6 @@
 , jq
 , socat
 , utillinux
-, writeScriptBin
-, runtimeShell
 , lib
 , exe
 , script
@@ -82,16 +80,43 @@ let
         exec ${scriptBin}/bin/${scriptBin.name} $@
     '') scripts);
 
-  entry-point = writeScriptBin "entry-point" ''
-    #!${runtimeShell}
+  runNetwork = pkgs.writeShellScriptBin "run-network" ''
     if [[ -z "$NETWORK" ]]; then
-      exec ${pkgs.${exe}}/bin/${exe} $@
+      echo "[Error] Cannot obtain NETWORK env variable"
+      exit 1
     ${clusterStatements}
     else
-      echo "Managed configuration for network "$NETWORK" does not exist"
+      echo "[Error] Managed configuration for network "$NETWORK" does not exist"
+      exit 1
     fi
   '';
 
+  # The docker context with static content
+  context = ./context/tracer;
+
+  genCfgs = let
+    environments' = lib.getAttrs [ "mainnet" "preprod" "preview" ] commonLib.environments;
+    cardano-deployment = commonLib.mkConfigHtml environments';
+  in
+    pkgs.runCommand "cardano-html" {} ''
+      mkdir "$out"
+
+      ENVS=(${lib.escapeShellArgs (builtins.attrNames environments')})
+      for ENV in "''${ENVS[@]}"; do
+        # Migrate each env from a flat dir to an ENV subdir
+        mkdir -p "$out/config/$ENV"
+        for i in $(find ${cardano-deployment} -type f -name "$ENV-tracer-config*" -printf "%f\n"); do
+          cp -v "${cardano-deployment}/$i" "$out/config/$ENV/''${i#"$ENV-"}"
+
+          # Adjust from iohk-nix default config for the oci environment
+          sed -i -r \
+            -e 's|"contents": ".*"|"contents": "/ipc/cardano-tracer.socket"|g' \
+            -e 's|"logRoot": ".*"|"logRoot": "/logs"|g' \
+            "$out/config/$ENV/''${i#"$ENV-"}"
+        done
+      done
+    '';
+
 in
   dockerTools.buildImage {
     name = "${repoName}";
@@ -110,15 +135,33 @@ in
 
       # Similarly, make a root level dir for logs:
       mkdir -p logs
-    '';
 
-    copyToRoot = pkgs.buildEnv {
-      name = "image-root";
-      pathsToLink = ["/"];
-      paths = [entry-point];
-    };
+      # The "custom" operation mode of this image, when the NETWORK env is
+      # unset and "run" is provided as an entrypoint arg, will use the
+      # following default directories.
+      mkdir -p opt/cardano
+
+      # Setup bins
+      mkdir -p usr/local/bin
+      cp -v ${runNetwork}/bin/* usr/local/bin
+      cp -v ${context}/bin/* usr/local/bin
+      ln -sv ${cardano-tracer}/bin/cardano-tracer usr/local/bin/cardano-tracer
+      ln -sv ${jq}/bin/jq usr/local/bin/jq
+
+      # Create iohk-nix network configs, organized by network directory.
+      SRC="${genCfgs}"
+      DST="opt/cardano"
+
+      # Make the directory structure with the iohk-nix configs mutable. This
+      # enables creation of merge mode entrypoint configs in the respective
+      # NETWORK directory. Keep config files as read-only copies from the nix
+      # store instead of direct symlinks.  This avoids volume mount failures
+      # caused by broken symlinks as seen from the host.
+      cp -R "$SRC"/* "$DST"
+      find "$DST" -mindepth 1 -type d -exec bash -c "chmod 0755 {}" \;
+    '';
 
     config = {
-      EntryPoint = [ "${entry-point}/bin/entry-point" ];
+      EntryPoint = [ "entrypoint" ];
     };
   }

From eccf6291496e523da13b8f6101db27e294757e9b Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 4 Apr 2025 14:54:48 -0500
Subject: [PATCH 27/37] tracerOci: default socket, path adjustments w/
 entrypoint fixes

---
 nix/docker/context/node/bin/run-node     |  2 +-
 nix/docker/context/tracer/bin/entrypoint | 12 +++++++++++-
 nix/docker/context/tracer/bin/run-tracer |  2 +-
 nix/docker/tracer.nix                    | 11 ++++++++---
 nix/nixos/cardano-tracer-service.nix     |  2 +-
 nix/pkgs.nix                             |  2 +-
 6 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/nix/docker/context/node/bin/run-node b/nix/docker/context/node/bin/run-node
index 6f0988c692e..90b442d0056 100755
--- a/nix/docker/context/node/bin/run-node
+++ b/nix/docker/context/node/bin/run-node
@@ -1,7 +1,7 @@
 #!/bin/env bash
 set -eo pipefail
 
-echo "Running the cardano node ..."
+echo "Running cardano node ..."
 
 [[ -n $DEBUG ]] && set -x
 
diff --git a/nix/docker/context/tracer/bin/entrypoint b/nix/docker/context/tracer/bin/entrypoint
index df4fce2aa43..bfdc5e92bea 100755
--- a/nix/docker/context/tracer/bin/entrypoint
+++ b/nix/docker/context/tracer/bin/entrypoint
@@ -38,10 +38,20 @@ if [[ -n $NETWORK ]]; then
       export CARDANO_CONFIG="$CFG/$NETWORK/tracer-config.json"
     fi
 
+    if [[ -n $DEBUG ]]; then
+      echo "Cardano config in merge mode is:"
+      cat "$CARDANO_CONFIG"
+      echo
+    fi
+
     # Run cardano-tracer using iohk-nix base config merged with provided custom
     # config for the requested NETWORK.
     unset NETWORK
-    exec /usr/local/bin/run-tracer "$@"
+    if [[ $1 == "run" ]]; then
+      exec /usr/local/bin/run-tracer "$@"
+    else
+      exec /usr/local/bin/run-tracer run "$@"
+    fi
 
   else
     # Run cardano-tracer using "scripts" mode for the requested NETWORK.
diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer
index dd3e4b92f99..23c0d3a4fca 100755
--- a/nix/docker/context/tracer/bin/run-tracer
+++ b/nix/docker/context/tracer/bin/run-tracer
@@ -1,7 +1,7 @@
 #!/bin/env bash
 set -eo pipefail
 
-echo "Running the cardano tracer ..."
+echo "Running cardano tracer ..."
 
 [[ -n $DEBUG ]] && set -x
 
diff --git a/nix/docker/tracer.nix b/nix/docker/tracer.nix
index e6089d18cd9..7dd66e94bbd 100644
--- a/nix/docker/tracer.nix
+++ b/nix/docker/tracer.nix
@@ -110,7 +110,7 @@ let
 
           # Adjust from iohk-nix default config for the oci environment
           sed -i -r \
-            -e 's|"contents": ".*"|"contents": "/ipc/cardano-tracer.socket"|g' \
+            -e 's|"contents": ".*"|"contents": "/ipc/tracer.socket"|g' \
             -e 's|"logRoot": ".*"|"logRoot": "/logs"|g' \
             "$out/config/$ENV/''${i#"$ENV-"}"
         done
@@ -130,7 +130,6 @@ in
       # The "scripts" operation mode of this image, when the NETWORK env var is
       # set to a valid network, will use the following default directories
       # mounted at /:
-      mkdir -p data
       mkdir -p ipc
 
       # Similarly, make a root level dir for logs:
@@ -138,8 +137,14 @@ in
 
       # The "custom" operation mode of this image, when the NETWORK env is
       # unset and "run" is provided as an entrypoint arg, will use the
-      # following default directories.
+      # following default directories.  To reduce confusion caused by default
+      # directory paths varying by mode, symlink these directories to the
+      # "scripts" mode default directories at the root location.  This will
+      # permit use of volume mounts at the root directory location regardless
+      # of which mode the image is operating in.
       mkdir -p opt/cardano
+      ln -sv /ipc opt/cardano/ipc
+      ln -sv /logs opt/cardano/logs
 
       # Setup bins
       mkdir -p usr/local/bin
diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index 257dc4757cc..fce2e66482e 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -138,7 +138,7 @@ in {
 
       acceptingSocket = mkOption {
         type = nullOr (either str path);
-        default = "${runtimeDir}/cardano-tracer.socket";
+        default = "${runtimeDir}/tracer.socket";
         description = ''
           If accepting connections from node(s) to a cardano-tracer socket, the
           path.
diff --git a/nix/pkgs.nix b/nix/pkgs.nix
index 1a0cf8a9ee9..ebfae10e43c 100644
--- a/nix/pkgs.nix
+++ b/nix/pkgs.nix
@@ -132,7 +132,7 @@ in with final;
   tracerDockerImage =
     let
       defaultConfig = rec {
-        acceptingSocket = "/ipc/cardano-tracer.socket";
+        acceptingSocket = "/ipc/tracer.socket";
         stateDir = "/logs";
         logging = [
           {

From b9bddd0d8785f4eceed4b6e5db573320ac441246 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 4 Apr 2025 15:10:23 -0500
Subject: [PATCH 28/37] tracerOci: add tracer image readme docs

---
 nix/docker/README.md | 152 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 140 insertions(+), 12 deletions(-)

diff --git a/nix/docker/README.md b/nix/docker/README.md
index 8be3c5f2f0f..adda397a728 100644
--- a/nix/docker/README.md
+++ b/nix/docker/README.md
@@ -1,4 +1,4 @@
-# Building Node and Submit API Images
+# Building Node, Submit API and Tracer Images
 To build and load the oci images into the Docker engine, the most
 basic commands are:
 ```
@@ -36,7 +36,8 @@ docker run \
   ghcr.io/intersectmbo/cardano-node:dev
 ```
 
-A similar command can be run to look around the cardano-submit-api container.
+Similar commands can be run to look around the cardano-submit-api and
+cardano-tracer containers.
 
 
 # Cardano Node Image Operation
@@ -61,8 +62,9 @@ To launch cardano-node with a custom configuration, "custom" mode, provide
 entrypoint args starting with `run` and:
 * Leave the `NETWORK` env variable unset
 * Optionally include additional cardano-node args to the entrypoint after `run`
-* Optionally include environment variables interpreted by [nix/docker/context/node/bin/run-node](context/node/bin/run-node),
-  or `/usr/local/bin/run-node` in the container
+* Optionally include environment variables interpreted by
+  [nix/docker/context/node/bin/run-node](context/node/bin/run-node), or
+  `/usr/local/bin/run-node` in the container
 
 For example, launch a custom cardano-node container using cardano-node args and
 a local configuration mapped into the container:
@@ -81,19 +83,19 @@ files found at `/opt/cardano/config` and organized under a subdirectory of the
 network's name.  For example, to utilize standard configs for preprod network,
 but modify the cardano-node listening port:
 ```
-  docker run \
-    -v preprod-data:/data \
-    -e CARDANO_CONFIG="/opt/cardano/config/preprod/config.json" \
-    -e CARDANO_TOPOLOGY="/opt/cardano/config/preprod/topology.json" \
-    -e CARDANO_PORT="6001" \
-    ghcr.io/intersectmbo/cardano-node:dev \
-    run
+docker run \
+  -v preprod-data:/data \
+  -e CARDANO_CONFIG="/opt/cardano/config/preprod/config.json" \
+  -e CARDANO_TOPOLOGY="/opt/cardano/config/preprod/topology.json" \
+  -e CARDANO_PORT="6001" \
+  ghcr.io/intersectmbo/cardano-node:dev \
+  run
 ```
 
 In "custom" mode, default state directories include
 `/opt/cardano/{data,ipc,logs}`, with `/opt/cardano/data/db` being the default
 database state location.  These state directories are symlinked to root in the container:
-`/opt/cardano/{data,ipc,logs} -> /{data,ipc,logs}` for more consistency between modes.
+`/opt/cardano/{data,ipc,logs} -> /{data,ipc,logs}` for consistency between modes.
 Standard network config files can be found under `/opt/cardano/config`.
 
 
@@ -250,6 +252,132 @@ both to align the default ipc socket state directory in both the cardano-node
 and cardano-submit-api images and to remain backwards compatible.
 
 
+# Cardano Tracer Image Operation
+## Scripts Mode
+To launch cardano-tracer with pre-loaded configuration, "scripts" mode,
+use the `NETWORK` env variable to declare an existing cardano network name.
+
+An example using a docker named volume to persist socket state to the host:
+```
+docker run \
+  -v node-ipc:/ipc \
+  -e NETWORK=mainnet \
+  ghcr.io/intersectmbo/cardano-tracer:dev
+```
+
+In "scripts" mode, default state directories include `/{ipc,logs}` and default
+mode of operation is to accept socket connections from node, `AcceptAt`, at a
+path of `/ipc/tracer.socket`.  Both tracer socket connection modes of
+`AcceptAt` and `ConnectTo` necessitate a shared volume mount between a node
+container and a tracer container.  See the [Cardano-node to Cardano-tracer
+Socket Connection](#cardano-node-to-cardano-tracer-socket-connection) section
+for more on making the required socket connections.
+
+
+## Custom Mode
+To launch cardano-tracer with a custom configuration, "custom" mode, provide
+entrypoint args starting with `run` and:
+* Leave the `NETWORK` env variable unset
+* Optionally include additional cardano-tracer args to the entrypoint after `run`
+* Optionally include environment variables interpreted by
+  [nix/docker/context/tracer/bin/run-tracer](context/tracer/bin/run-tracer), or
+  `/usr/local/bin/run-tracer` in the container
+
+For example, launch a custom cardano-tracer container using cardano-tracer args and
+a local configuration mapped into the container:
+```
+docker run \
+  -v node-ipc:/ipc \
+  -v "$PWD/config/tracer:/config" \
+  ghcr.io/intersectmbo/cardano-tracer:dev \
+  run \
+  --config /config/mainnet/config.json
+```
+
+Custom mode may also leverage standard mainnet or testnet network tracer config
+files found at `/opt/cardano/config` and organized under a subdirectory of the
+network's name.  For example, to utilize standard configs for preprod network,
+but modify the cardano-tracer minimum log severity:
+```
+docker run \
+  -e CARDANO_CONFIG="/opt/cardano/config/preprod/tracer-config.json" \
+  -e CARDANO_MIN_LOG_SEVERITY="Debug" \
+  ghcr.io/intersectmbo/cardano-tracer:dev \
+  run
+```
+
+In "custom" mode, default state directories include `/opt/cardano/{ipc,logs}`.
+These state directories are symlinked to root in the container:
+`/opt/cardano/{ipc,logs} -> /{ipc,logs}` for consistency between modes.
+Standard network tracer config files can be found under `/opt/cardano/config`.
+
+
+## Merge Mode
+With the `NETWORK` env variable set and `CARDANO_CONFIG_JSON_MERGE` env
+variable set and containing valid json, cardano-tracer will run with deep
+merged base `NETWORK` tracer config and json merge config.
+
+Optional env variables and cardano-tracer args which can be used in custom mode
+can also be used in this mode.  Merge mode uses the same default state
+directories as custom mode.
+
+An example which changes the prometheus binding address from a default of
+localhost (`127.0.0.1`) to `0.0.0.0`:
+```
+docker run \
+  -v node-ipc:/ipc \
+  -e NETWORK=mainnet \
+  -e CARDANO_CONFIG_JSON_MERGE='{"hasPrometheus":{"epHost": "0.0.0.0"}}' \
+  ghcr.io/intersectmbo/cardano-tracer:dev
+```
+
+Similar bind mounting, and host mounted network tracer config considerations
+exist for the cardano-tracer image as also detailed above for the cardano-node
+image.
+
+
+## Cardano-node to Cardano-tracer Socket Connection
+To establish a cardano-node container to cardano-tracer container socket
+connection, in addition to a shared volume mount where such a socket can be
+accessed by both containers, the cardano-node container will need to be started
+with an extra argument.  To include the extra argument, either custom mode or
+merge mode for the node image will be required.
+
+An example for node to connect to a tracer socket, using custom mode to
+append the extra cli arg:
+```
+docker run \
+  -v node-ipc:/ipc \
+  -e CARDANO_CONFIG="/opt/cardano/config/mainnet/config.json" \
+  -e CARDANO_TOPOLOGY="/opt/cardano/config/mainnet/topology.json" \
+  ghcr.io/intersectmbo/cardano-node:dev \
+  run \
+  --tracer-socket-path-connect /ipc/tracer.socket
+
+docker run \
+  -v node-ipc:/ipc \
+  -e NETWORK=mainnet \
+  ghcr.io/intersectmbo/cardano-tracer:dev
+```
+
+An example for node to use a socket to accept a tracer connection, using merge
+mode to append the extra cli arg:
+```
+docker run \
+  -v node-ipc:/ipc \
+  -e NETWORK=mainnet \
+  -e CARDANO_CONFIG_JSON_MERGE='{}' \
+  ghcr.io/intersectmbo/cardano-node:dev \
+  run \
+  --tracer-socket-path-accept /ipc/node-tracer.socket
+
+docker run \
+  -v node-ipc:/ipc \
+  -e NETWORK=mainnet \
+  -e CARDANO_CONFIG_JSON_MERGE='{"network":{"contents":["/ipc/node-tracer.socket"],"tag":"ConnectTo"}}' \
+  ghcr.io/intersectmbo/cardano-tracer:dev
+```
+
 # Manual Testing
 1. Run -e NETWORK=mainnet and check graceful shutdown SIGINT with -it
 2. Run -e NETWORK=mainnet and check graceful shutdown SIGTERM with --detach

From ed848f77eaca1a7a07f7e6e3bc517f5e6cf6589f Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 4 Apr 2025 15:11:58 -0500
Subject: [PATCH 29/37] ci: add tracer image to release-ghcr workflow

---
 .github/workflows/release-ghcr.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/release-ghcr.yaml b/.github/workflows/release-ghcr.yaml
index 365e706b2ac..1ddd4690344 100644
--- a/.github/workflows/release-ghcr.yaml
+++ b/.github/workflows/release-ghcr.yaml
@@ -85,6 +85,16 @@ jobs:
         skopeo copy docker-archive:./result-api docker://ghcr.io/intersectmbo/cardano-submit-api:$GITHUB_REF_NAME
         echo "::endgroup::"
 
+    - name: Uploading intersectmbo/cardano-tracer
+      run: |
+        echo "::group::Downloading from cache"
+        nix build --accept-flake-config --print-out-paths --builders "" --max-jobs 0 --out-link ./result-tracer ${{ steps.flake-metadata.outputs.LOCKED_URL }}#dockerImage/tracer
+        echo "::endgroup::"
+
+        echo "::group::Uploading to registry"
+        skopeo copy docker-archive:./result-tracer docker://ghcr.io/intersectmbo/cardano-tracer:$GITHUB_REF_NAME
+        echo "::endgroup::"
+
     - name: Obtaining latest release tag
       id: latest-tag
       run: |
@@ -110,3 +120,7 @@ jobs:
         echo "::group::Tagging latest for intersectmbo/cardano-submit-api"
         skopeo copy docker-archive:./result-api docker://ghcr.io/intersectmbo/cardano-submit-api:latest
         echo "::endgroup::"
+
+        echo "::group::Tagging latest for intersectmbo/cardano-tracer"
+        skopeo copy docker-archive:./result-tracer docker://ghcr.io/intersectmbo/cardano-tracer:latest
+        echo "::endgroup::"

From a97d5290e7dac379962d7d19d30018d643f23ca0 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 16 Apr 2025 17:10:49 -0500
Subject: [PATCH 30/37] nixosNodeSvc: add PrometheusSimple usage for node
 10.3.1 pre-release

---
 configuration/cardano/mainnet-config.json          | 1 +
 configuration/cardano/mainnet-config.yaml          | 4 +---
 configuration/cardano/testnet-template-config.json | 1 +
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/configuration/cardano/mainnet-config.json b/configuration/cardano/mainnet-config.json
index ee0adb6ad9e..3594c4d5ef9 100644
--- a/configuration/cardano/mainnet-config.json
+++ b/configuration/cardano/mainnet-config.json
@@ -47,6 +47,7 @@
       "backends": [
         "EKGBackend",
         "Forwarder",
+        "PrometheusSimple suffix 127.0.0.1 12798",
         "Stdout HumanFormatColoured"
       ],
       "detail": "DNormal",
diff --git a/configuration/cardano/mainnet-config.yaml b/configuration/cardano/mainnet-config.yaml
index a8e2f1eb137..d7a4cb39cd2 100644
--- a/configuration/cardano/mainnet-config.yaml
+++ b/configuration/cardano/mainnet-config.yaml
@@ -133,9 +133,7 @@ TraceOptions:
     # tracing system.
     - EKGBackend
     - Forwarder
-
-    # Include for the 10.3 release
-    # - PrometheusSimple suffix 127.0.0.1 12798
+    - PrometheusSimple suffix 127.0.0.1 12798
 
     # Only one of the following can be enabled, which determines for format
     # of node logging to stdout.
diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json
index ae24abea77b..dcdda9262e5 100644
--- a/configuration/cardano/testnet-template-config.json
+++ b/configuration/cardano/testnet-template-config.json
@@ -43,6 +43,7 @@
       "backends": [
         "EKGBackend",
         "Forwarder",
+        "PrometheusSimple suffix 127.0.0.1 12798",
         "Stdout HumanFormatColoured"
       ],
       "detail": "DNormal",

From f5a31c94ea57d43572c79ed8bd793e171fdc9c43 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 16 Apr 2025 17:11:38 -0500
Subject: [PATCH 31/37] nixosNodeSvc: add genesis mode config to legacy and
 testnet template cfg

---
 configuration/cardano/mainnet-config-legacy.json   | 10 +++++++++-
 configuration/cardano/testnet-template-config.json |  6 ++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/configuration/cardano/mainnet-config-legacy.json b/configuration/cardano/mainnet-config-legacy.json
index 27b2e6218b8..817fcb22083 100644
--- a/configuration/cardano/mainnet-config-legacy.json
+++ b/configuration/cardano/mainnet-config-legacy.json
@@ -3,6 +3,9 @@
   "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874",
   "ByronGenesisFile": "mainnet-byron-genesis.json",
   "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb",
+  "CheckpointsFile": "mainnet-checkpoints.json",
+  "CheckpointsFileHash": "3e6dee5bae7acc6d870187e72674b37c929be8c66e62a552cf6a876b1af31ade",
+  "ConsensusMode": "PraosMode",
   "ConwayGenesisFile": "mainnet-conway-genesis.json",
   "ConwayGenesisHash": "15a199f895e461ec0ffc6dd4e4028af28a492ab4e806d39cb674c88f7643ef62",
   "EnableP2P": true,
@@ -10,12 +13,17 @@
   "LastKnownBlockVersion-Major": 3,
   "LastKnownBlockVersion-Minor": 0,
   "MaxKnownMajorProtocolVersion": 2,
-  "MinNodeVersion": "10.1.4",
+  "MinBigLedgerPeersForTrustedState": 5,
+  "MinNodeVersion": "10.2.1",
   "PeerSharing": true,
   "Protocol": "Cardano",
   "RequiresNetworkMagic": "RequiresNoMagic",
   "ShelleyGenesisFile": "mainnet-shelley-genesis.json",
   "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81",
+  "SyncTargetNumberOfActiveBigLedgerPeers": 30,
+  "SyncTargetNumberOfActivePeers": 0,
+  "SyncTargetNumberOfEstablishedBigLedgerPeers": 50,
+  "SyncTargetNumberOfKnownBigLedgerPeers": 100,
   "TargetNumberOfActivePeers": 20,
   "TargetNumberOfEstablishedPeers": 40,
   "TargetNumberOfKnownPeers": 150,
diff --git a/configuration/cardano/testnet-template-config.json b/configuration/cardano/testnet-template-config.json
index dcdda9262e5..16e92bd011d 100644
--- a/configuration/cardano/testnet-template-config.json
+++ b/configuration/cardano/testnet-template-config.json
@@ -3,6 +3,7 @@
   "ApplicationName": "cardano-sl",
   "ApplicationVersion": 0,
   "ByronGenesisFile": "byron-genesis.json",
+  "ConsensusMode": "PraosMode",
   "ConwayGenesisFile": "conway-genesis.json",
   "EnableP2P": true,
   "ExperimentalHardForksEnabled": true,
@@ -18,10 +19,15 @@
   },
   "MaxConcurrencyDeadline": 4,
   "MaxKnownMajorProtocolVersion": 2,
+  "MinBigLedgerPeersForTrustedState": 5,
   "PBftSignatureThreshold": 1.1,
   "Protocol": "Cardano",
   "RequiresNetworkMagic": "RequiresMagic",
   "ShelleyGenesisFile": "shelley-genesis.json",
+  "SyncTargetNumberOfActiveBigLedgerPeers": 30,
+  "SyncTargetNumberOfActivePeers": 0,
+  "SyncTargetNumberOfEstablishedBigLedgerPeers": 50,
+  "SyncTargetNumberOfKnownBigLedgerPeers": 100,
   "TargetNumberOfActivePeers": 20,
   "TargetNumberOfEstablishedPeers": 40,
   "TargetNumberOfKnownPeers": 100,

From fa4ce367ff27779b5401487cfaaf873681c4d06a Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 16 Apr 2025 18:28:47 -0500
Subject: [PATCH 32/37] ci: update mn cfg-bp file and updater script for new
 tracing

---
 configuration/cardano/mainnet-config-bp.json | 182 ++++++++++---------
 1 file changed, 98 insertions(+), 84 deletions(-)

diff --git a/configuration/cardano/mainnet-config-bp.json b/configuration/cardano/mainnet-config-bp.json
index 69e1a88ddcc..a12abacf085 100644
--- a/configuration/cardano/mainnet-config-bp.json
+++ b/configuration/cardano/mainnet-config-bp.json
@@ -34,91 +34,105 @@
   "TargetNumberOfEstablishedPeers": 40,
   "TargetNumberOfKnownPeers": 100,
   "TargetNumberOfRootPeers": 100,
-  "TraceAcceptPolicy": true,
-  "TraceBlockFetchClient": false,
-  "TraceBlockFetchDecisions": false,
-  "TraceBlockFetchProtocol": false,
-  "TraceBlockFetchProtocolSerialised": false,
-  "TraceBlockFetchServer": false,
-  "TraceChainDb": true,
-  "TraceChainSyncBlockServer": false,
-  "TraceChainSyncClient": false,
-  "TraceChainSyncHeaderServer": false,
-  "TraceChainSyncProtocol": false,
-  "TraceConnectionManager": true,
-  "TraceDNSResolver": true,
-  "TraceDNSSubscription": true,
-  "TraceDiffusionInitialization": true,
-  "TraceErrorPolicy": true,
-  "TraceForge": true,
-  "TraceHandshake": true,
-  "TraceInboundGovernor": true,
-  "TraceIpSubscription": true,
-  "TraceLedgerPeers": true,
-  "TraceLocalChainSyncProtocol": false,
-  "TraceLocalConnectionManager": true,
-  "TraceLocalErrorPolicy": true,
-  "TraceLocalHandshake": true,
-  "TraceLocalRootPeers": true,
-  "TraceLocalTxSubmissionProtocol": false,
-  "TraceLocalTxSubmissionServer": false,
-  "TraceMempool": false,
-  "TraceMux": false,
-  "TracePeerSelection": true,
-  "TracePeerSelectionActions": true,
-  "TracePublicRootPeers": true,
-  "TraceServer": true,
-  "TraceTxInbound": false,
-  "TraceTxOutbound": false,
-  "TraceTxSubmissionProtocol": false,
-  "TracingVerbosity": "NormalVerbosity",
-  "TurnOnLogMetrics": true,
-  "TurnOnLogging": true,
-  "UseTraceDispatcher": false,
-  "defaultBackends": [
-    "KatipBK"
-  ],
-  "defaultScribes": [
-    [
-      "StdoutSK",
-      "stdout"
-    ]
-  ],
-  "hasEKG": 12788,
-  "hasPrometheus": [
-    "127.0.0.1",
-    12798
-  ],
-  "minSeverity": "Info",
-  "options": {
-    "mapBackends": {
-      "cardano.node.metrics": [
-        "EKGViewBK"
+  "TraceOptionForwarder": {
+    "connQueueSize": 64,
+    "disconnQueueSize": 128,
+    "maxReconnectDeplay": 30
+  },
+  "TraceOptionMetricsPrefix": "cardano.node.metrics.",
+  "TraceOptionPeerFrequency": 2000,
+  "TraceOptionResourceFrequency": 1000,
+  "TraceOptions": {
+    "": {
+      "backends": [
+        "EKGBackend",
+        "Forwarder",
+        "PrometheusSimple suffix 127.0.0.1 12798",
+        "Stdout HumanFormatColoured"
       ],
-      "cardano.node.resources": [
-        "EKGViewBK"
-      ]
-    },
-    "mapSubtrace": {
-      "cardano.node.metrics": {
-        "subtrace": "Neutral"
-      }
+      "detail": "DNormal",
+      "severity": "Notice"
+    },
+    "BlockFetch.Client.CompletedBlockFetch": {
+      "maxFrequency": 2
+    },
+    "BlockFetch.Decision": {
+      "severity": "Silence"
+    },
+    "ChainDB": {
+      "severity": "Info"
+    },
+    "ChainDB.AddBlockEvent.AddBlockValidation": {
+      "severity": "Silence"
+    },
+    "ChainDB.AddBlockEvent.AddBlockValidation.ValidCandidate": {
+      "maxFrequency": 2
+    },
+    "ChainDB.AddBlockEvent.AddedBlockToQueue": {
+      "maxFrequency": 2
+    },
+    "ChainDB.AddBlockEvent.AddedBlockToVolatileDB": {
+      "maxFrequency": 2
+    },
+    "ChainDB.CopyToImmutableDBEvent.CopiedBlockToImmutableDB": {
+      "maxFrequency": 2
+    },
+    "ChainSync.Client": {
+      "severity": "Warning"
+    },
+    "Forge.Loop": {
+      "severity": "Info"
+    },
+    "Forge.StateInfo": {
+      "severity": "Info"
+    },
+    "Mempool": {
+      "severity": "Silence"
+    },
+    "Net.ConnectionManager.Remote": {
+      "severity": "Info"
+    },
+    "Net.ConnectionManager.Remote.ConnectionManagerCounters": {
+      "severity": "Silence"
+    },
+    "Net.ErrorPolicy": {
+      "severity": "Info"
+    },
+    "Net.ErrorPolicy.Local": {
+      "severity": "Info"
+    },
+    "Net.InboundGovernor": {
+      "severity": "Warning"
+    },
+    "Net.InboundGovernor.Remote": {
+      "severity": "Info"
+    },
+    "Net.Mux.Remote": {
+      "severity": "Info"
+    },
+    "Net.PeerSelection": {
+      "severity": "Silence"
+    },
+    "Net.Subscription.DNS": {
+      "severity": "Info"
+    },
+    "Net.Subscription.IP": {
+      "severity": "Info"
+    },
+    "Resources": {
+      "severity": "Silence"
+    },
+    "Startup.DiffusionInit": {
+      "severity": "Info"
     }
   },
-  "rotation": {
-    "rpKeepFilesNum": 10,
-    "rpLogLimitBytes": 5000000,
-    "rpMaxAgeHours": 24
-  },
-  "setupBackends": [
-    "KatipBK"
-  ],
-  "setupScribes": [
-    {
-      "scFormat": "ScText",
-      "scKind": "StdoutSK",
-      "scName": "stdout",
-      "scRotation": null
-    }
-  ]
+  "TurnOnLogMetrics": true,
+  "TurnOnLogging": true,
+  "UseTraceDispatcher": true,
+  "defaultBackends": [],
+  "defaultScribes": [],
+  "minSeverity": "Critical",
+  "options": {},
+  "setupBackends": [],
+  "setupScribes": []
 }

From e1fa5a32e7eddfc84be28f3ca4236870754f5c85 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Tue, 22 Apr 2025 14:53:49 -0500
Subject: [PATCH 33/37] nixosTests: re-enable 12798 edge node port check w/
 promSimple

---
 nix/nixos/tests/cardano-node-artifact.nix |  2 ++
 nix/nixos/tests/cardano-node-edge.nix     | 24 ++++++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/nix/nixos/tests/cardano-node-artifact.nix b/nix/nixos/tests/cardano-node-artifact.nix
index bc2131c422e..c2f3484fbd6 100644
--- a/nix/nixos/tests/cardano-node-artifact.nix
+++ b/nix/nixos/tests/cardano-node-artifact.nix
@@ -39,6 +39,8 @@
     };
   };
 
+  # Only newer nixpkgs have have timeout args for all wait_for_.* fns.
+  # Use the generic wait_until_succeeds w/ timeout arg until nixpkgs is bumped.
   mkScriptTest = env: ''
     machine.systemctl("start cardano-node-${env}")
     machine.wait_until_succeeds("[ -S /var/lib/cardano-node-${env}/node.socket ]", timeout=${timeout})
diff --git a/nix/nixos/tests/cardano-node-edge.nix b/nix/nixos/tests/cardano-node-edge.nix
index f07ba9952f9..06a974687f7 100644
--- a/nix/nixos/tests/cardano-node-edge.nix
+++ b/nix/nixos/tests/cardano-node-edge.nix
@@ -1,6 +1,9 @@
 {pkgs, ...}:
 with pkgs; let
   environment = "mainnet";
+
+  # NixosTest script fns supporting a timeout have a default of 900 seconds.
+  timeout = toString 30;
 in {
   name = "cardano-node-edge-test";
   nodes = {
@@ -45,22 +48,21 @@ in {
       };
     };
   };
+
+  # Only newer nixpkgs have have timeout args for all wait_for_.* fns.
+  # Use the generic wait_until_succeeds w/ timeout arg until nixpkgs is bumped.
   testScript = ''
     start_all()
-    machine.wait_for_unit("cardano-node.service")
-    machine.wait_for_file("/run/cardano-node/node.socket")
-
-    # Re-enable once PrometheusSimple with node >= 10.3 is available
-    # machine.wait_for_open_port(12798)
-
-    machine.wait_for_open_port(3001)
+    machine.wait_for_unit("cardano-node.service", timeout=${timeout})
+    machine.wait_until_succeeds("[ -S /run/cardano-node/node.socket ]", timeout=${timeout})
+    machine.wait_until_succeeds("nc -z localhost 12798", timeout=${timeout})
+    machine.wait_until_succeeds("nc -z localhost 3001", timeout=${timeout})
     machine.succeed("systemctl status cardano-node")
-
-    machine.succeed(
+    out = machine.succeed(
       "${cardanoNodePackages.cardano-cli}/bin/cardano-cli ping -h 127.0.0.1 -c 1 -q --json | ${jq}/bin/jq -c"
     )
-
-    machine.wait_for_open_port(8101)
+    print("ping:", out)
+    machine.wait_until_succeeds("nc -z localhost 8101", timeout=${timeout})
     machine.succeed("systemctl status cardano-submit-api")
   '';
 }

From 7efa41ec28b30bfac7eaf49ab7e1bd7a312b571a Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 23 Apr 2025 20:43:22 -0500
Subject: [PATCH 34/37] nixosTracerSvc: improve opt desc during iohk-nix desc
 review

---
 nix/nixos/cardano-tracer-service.nix | 89 ++++++++++++++++------------
 1 file changed, 51 insertions(+), 38 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index fce2e66482e..c4d06c5dd9a 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -124,9 +124,8 @@ in {
         type = bool;
         default = false;
         description = ''
-          Enable cardano-tracer, a service for logging and monitoring of
-          Cardano nodes. After it is connected to the node(s), it periodically
-          asks for different information, receives it, and handles it.
+          Enable cardano-tracer, a service for logging and monitoring of log
+          and metrics providers such as cardano-node.
         '';
       };
 
@@ -140,8 +139,15 @@ in {
         type = nullOr (either str path);
         default = "${runtimeDir}/tracer.socket";
         description = ''
-          If accepting connections from node(s) to a cardano-tracer socket, the
-          path.
+          Declaring this option means that cardano-tracer will operate in an
+          `AcceptAt` tag mode where cardano-tracer works as a server: it
+          receives network connections from providers such as node via a single
+          local socket provided by cardano-tracer.
+
+          Except for special use cases, declaring this `acceptingSocket` option
+          instead of the `connectToSocket` option is recommended, as the
+          `AcceptAt` tag mode supports dynamic provider addition or removal
+          without requiring cardano-tracer reconfiguration and restart.
 
           Either this option, or the connectToSocket option must be declared.
         '';
@@ -192,10 +198,18 @@ in {
       };
 
       connectToSocket = mkOption {
-        type = nullOr (either str path);
+        type = nullOr (listOf (either str path));
         default = null;
         description = ''
-          If connecting to a cardano-node socket, the path.
+          Declaring this option means that cardano-tracer will operate in a
+          `ConnectTo` tag mode where cardano-tracer works as a client: it
+          establishes network connections to local socket(s) provided by the
+          provider(s). In this case a socket is used for each provider.
+
+          Except for special use cases, declaring `acceptingSocket` instead of
+          this option is recommended, as the `AcceptAt` tag mode supports
+          dynamic provider addition or removal without requiring cardano-tracer
+          reconfiguration and restart.
 
           Either this option, or the acceptingSocket option must be declared.
         '';
@@ -229,7 +243,7 @@ in {
         type = nullOr bool;
         default = null;
         description = ''
-          When recieving forwarded metrics, cardano-tracer can request a full
+          When receiving forwarded metrics, cardano-tracer can request a full
           set of EKG metrics, or a delta from the previous request.
 
           If true, a full set of metrics will be sent on each request.
@@ -248,7 +262,7 @@ in {
           will be requested, in seconds. For example, if ekgRequestFreq is 10,
           cardano-tracer will ask for new EKG metrics every ten seconds. There
           is no limit as loRequestNum, so every request returns all the metrics
-          the node has in this moment of time.
+          the provider has at that moment of time.
 
           If null cardano-tracer will set a default: 1.
         '';
@@ -281,7 +295,6 @@ in {
       executable = mkOption {
         type = str;
         default = "${cfg.package}/bin/cardano-tracer";
-        defaultText = "cardano-node";
         description = ''
           The cardano-tracer executable invocation to use.
         '';
@@ -330,7 +343,7 @@ in {
                 The logFormat option specifies the format of logs. There are two
                 possible modes: `ForMachine` and `ForHuman`. ForMachine is for JSON
                 format, ForHuman is for human-friendly text format. Since the logging
-                option accepts a list more than one logging section can be declared.
+                option accepts a list, more than one logging section can be declared.
               '';
             };
 
@@ -352,7 +365,7 @@ in {
                 The logRoot option specifies the path to the root directory. This
                 directory will contain all the subdirectories with the log files
                 inside. Remember that each subdirectory corresponds to the particular
-                node. If the root directory does not exist, it will be created.
+                provider. If the root directory does not exist, it will be created.
               '';
             };
           };
@@ -371,15 +384,15 @@ in {
         type = nullOr ints.positive;
         default = null;
         description = ''
-          This optional attribute specifies the number of log items
-          that will be requested from the node. For example, if loRequestNum is
-          10, cardano-tracer will periodically ask 10 log items in one request.
-          This value is useful for fine-tuning network traffic: it is possible
-          to ask 50 log items in one request, or ask them in 50 requests one at
-          a time. loRequestNum is the maximum number of log items. For example,
-          if cardano-tracer requests 50 log items but the node has only 40 at
-          that moment, these 40 items will be returned, the request won't block
-          to wait for an additional 10 items.
+          This optional attribute specifies the number of log items that will
+          be requested from the providing source. For example, if loRequestNum
+          is 10, cardano-tracer will periodically ask for 10 log items in one
+          request. This value is useful for fine-tuning network traffic: it is
+          possible to ask 50 log items in one request, or ask for them in 50
+          requests one at a time. loRequestNum is the maximum number of log
+          items. For example, if cardano-tracer requests 50 log items but the
+          provider has only 40 at that moment, these 40 items will be returned,
+          and the request won't block to wait for an additional 10 items.
 
           If null cardano-tracer will set a default: 100.
         '';
@@ -400,18 +413,18 @@ in {
         type = nullOr (either str (attrsOf str));
         default = null;
         description = ''
-          Passing metric help annotations to cardano-tracer can be done as a an
-          attribute set of strings from metric name to help text where
-          cardano-tracer's internal metric names have to be used as attribute
-          names.
+          Passing metrics help annotations to cardano-tracer can be done as an
+          attribute set of strings from metric's name to help text where
+          cardano-tracer's internal metric names have to be used as the
+          attribute names.
 
           If such a set is already available as JSON in a file, this option can
-          be declared as a string of the path to such file.
+          be declared as a string of the path to such a file.
 
-          Any metrics prefix name declared with `TraceOptionMetricsPrefix` in
-          cardano-node config should not be included in the attribute name.
-          Similarly metric type suffixes, such as `.int` or `.real` should also
-          not be included.
+          Any metrics prefix declared in provider config, such as
+          `TraceOptionMetricsPrefix` in cardano-node, should not be included in
+          the attribute name. Similarly, metric type suffixes such as `.int` or
+          `.real`, should also not be included.
 
           The effect of this option applies to prometheus metrics only, ie: not
           EKG.
@@ -427,7 +440,7 @@ in {
         default = null;
         description = ''
           If set true, metrics name suffixes, like "_int", will be dropped,
-          increases similiarity with old system names.
+          thus increasing the similarity with legacy tracing system names.
 
           The effect of this option applies to prometheus metrics only, ie: not
           EKG.
@@ -538,7 +551,6 @@ in {
           The port to listen on if prometheus is enabled.
 
           Defaults to the legacy prometheus listening port, 12798, plus 10.
-
           This avoids a conflict with the cardano-node default metrics port binding
           when PrometheusSimple backend is in use.
         '';
@@ -548,9 +560,10 @@ in {
         type = nullOr ints.positive;
         default = null;
         description = ''
-          The period for tracing cardano-tracer resource usage in milliseconds.
-          The frequency will be 1/resourceFreq times per millisecond.  If null
-          cardano-tracer will not display resource usage.
+          The period for tracing cardano-tracer's own resource usage in
+          milliseconds. For example, if set to 60000, resources will traced
+          every 60 seconds. If null cardano-tracer will not display resource
+          usage.
         '';
       };
 
@@ -642,8 +655,8 @@ in {
           `rpFrequencySecs`, `rpKeepFilesNum`, `rpLogLimitBytes`,
           `rpMaxAgeMinutes`.
 
-          Please note that if you skip this field, all log items will be stored
-          in a single file, and usually that's not what is desired.
+          Please note that if the rotation declaration is skipped, all log items will
+          be stored in a single file, and usually that's not what is desired.
 
           This option will be ignored if all logging has `logMode` configured
           as `JournalMode`.
@@ -777,7 +790,7 @@ in {
         type = nullOr (enum ["Minimum" "ErrorsOnly" "Maximum"]);
         default = null;
         description = ''
-          The optional attribute specifies the verbosity level for the
+          The verbosity optional attribute specifies the level for
           cardano-tracer itself.  There are 3 levels:
 
             Minimum - cardano-tracer will work as silently as possible.

From 6018d28591e8f60d61894fe4d3c9e7aa5e4b97ec Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Wed, 23 Apr 2025 20:44:21 -0500
Subject: [PATCH 35/37] nixosTracerSvc: rm warnRTViewMissing option config

---
 nix/nixos/cardano-tracer-service.nix | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index c4d06c5dd9a..7f6282ce9f9 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -60,8 +60,6 @@ with builtins; let
           // optionalAttrs (!isNull cfg.rotation.rpMaxAgeMinutes) {
             inherit (cfg.rotation) rpMaxAgeMinutes;
           };
-
-      WarnRTViewMissing = cfg.warnRtviewMissing;
     }
     // optionalAttrs cfg.ekgEnable {
       hasEKG = {
@@ -800,19 +798,6 @@ in {
           If null cardano-tracer will set a default: ErrorsOnly.
         '';
       };
-
-      warnRtviewMissing = mkOption {
-        type = nullOr bool;
-        default = null;
-        description = ''
-          Whether to provide a warning if RTView is requested in config but
-          cardano-tracer was built without it.
-
-          If null cardano-tracer will set a default: true if RTView
-          config is provided but cardano-tracer was built without it, false
-          otherwise.
-        '';
-      };
     };
   };
 

From 9cb63664e86bc25891b19238e212fb7c2fdd98f7 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Thu, 24 Apr 2025 13:14:06 -0500
Subject: [PATCH 36/37] nixosTracerSvc: drop hasForwarding opt

---
 nix/nixos/cardano-tracer-service.nix | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/nix/nixos/cardano-tracer-service.nix b/nix/nixos/cardano-tracer-service.nix
index 7f6282ce9f9..d61e5a404a8 100644
--- a/nix/nixos/cardano-tracer-service.nix
+++ b/nix/nixos/cardano-tracer-service.nix
@@ -326,11 +326,6 @@ in {
         '';
       };
 
-      # Consider adding hasForwarding config once more documentation is
-      # available on the feature. Ex:
-      #
-      # hasForwarding = {...};
-
       logging = mkOption {
         type = listOf (submodule {
           options = {

From a220eca536fc4b0e4dc7b7a32ad99d55778475d6 Mon Sep 17 00:00:00 2001
From: John Lotoski <john.lotoski@iohk.io>
Date: Fri, 2 May 2025 18:47:51 -0500
Subject: [PATCH 37/37] nodeOci: extend node tracer socket options to vars;
 update readme

---
 nix/docker/README.md                     |  4 ++--
 nix/docker/context/node/bin/run-node     | 28 +++++++++++++++++++++++-
 nix/docker/context/tracer/bin/run-tracer |  5 ++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/nix/docker/README.md b/nix/docker/README.md
index adda397a728..4afee3f9761 100644
--- a/nix/docker/README.md
+++ b/nix/docker/README.md
@@ -366,10 +366,10 @@ mode to append the extra cli arg:
 docker run \
   -v node-ipc:/ipc \
   -e NETWORK=mainnet \
+  -e CARDANO_TRACER_SOCKET_PATH_ACCEPT="/ipc/node-tracer.socket" \
   -e CARDANO_CONFIG_JSON_MERGE='{}' \
   ghcr.io/intersectmbo/cardano-node:dev \
-  run \
-  --tracer-socket-path-accept /ipc/node-tracer.socket
+  run
 
 docker run \
   -v node-ipc:/ipc \
diff --git a/nix/docker/context/node/bin/run-node b/nix/docker/context/node/bin/run-node
index 90b442d0056..c666b8b1b04 100755
--- a/nix/docker/context/node/bin/run-node
+++ b/nix/docker/context/node/bin/run-node
@@ -61,6 +61,8 @@ printRunEnv () {
   echo "CARDANO_PORT=$CARDANO_PORT"
   echo "CARDANO_SOCKET_PATH=$CARDANO_SOCKET_PATH"
   echo "CARDANO_TOPOLOGY=$CARDANO_TOPOLOGY"
+  [[ -n $CARDANO_TRACER_SOCKET_PATH_ACCEPT ]] && echo "CARDANO_TRACER_SOCKET_PATH_ACCEPT=$CARDANO_TRACER_SOCKET_PATH_ACCEPT"
+  [[ -n $CARDANO_TRACER_SOCKET_PATH_CONNECT ]] && echo "CARDANO_TRACER_SOCKET_PATH_CONNECT=$CARDANO_TRACER_SOCKET_PATH_CONNECT"
 
   if [[ ${CARDANO_BLOCK_PRODUCER} == true ]]; then
 
@@ -92,6 +94,19 @@ cat << EOF > /usr/local/bin/env
 #!/usr/bin/env bash
 
 # Docker run ENV vars
+EOF
+
+if [[ -n $CARDANO_TRACER_SOCKET_PATH_ACCEPT ]]; then
+  echo "CARDANO_TRACER_SOCKET_PATH_ACCEPT=\"$CARDANO_TRACER_SOCKET_PATH_ACCEPT\"" \
+    >> /usr/local/bin/env
+fi
+
+if [[ -n $CARDANO_TRACER_SOCKET_PATH_CONNECT ]]; then
+  echo "CARDANO_TRACER_SOCKET_PATH_CONNECT=\"$CARDANO_TRACER_SOCKET_PATH_CONNECT\"" \
+    >> /usr/local/bin/env
+fi
+
+cat << EOF >> /usr/local/bin/env
 CARDANO_BIND_ADDR="$CARDANO_BIND_ADDR"
 CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER
 CARDANO_CONFIG="$CARDANO_CONFIG"
@@ -134,6 +149,9 @@ runRelayNode () {
     "--port" "$CARDANO_PORT"
   )
 
+  [[ -n $CARDANO_TRACER_SOCKET_PATH_ACCEPT ]] && effopts+=("--tracer-socket-path-accept" "$CARDANO_TRACER_SOCKET_PATH_ACCEPT")
+  [[ -n $CARDANO_TRACER_SOCKET_PATH_CONNECT ]] && effopts+=("--tracer-socket-path-connect" "$CARDANO_TRACER_SOCKET_PATH_CONNECT")
+
   effopts+=("${filteredOpts[@]}")
 
   echo "cardano-node run ${effopts[*]}"
@@ -158,6 +176,9 @@ runBlockProducerNode () {
     "--shelley-operational-certificate" "$CARDANO_SHELLEY_OPERATIONAL_CERTIFICATE"
   )
 
+  [[ -n $CARDANO_TRACER_SOCKET_PATH_ACCEPT ]] && effopts+=("--tracer-socket-path-accept" "$CARDANO_TRACER_SOCKET_PATH_ACCEPT")
+  [[ -n $CARDANO_TRACER_SOCKET_PATH_CONNECT ]] && effopts+=("--tracer-socket-path-connect" "$CARDANO_TRACER_SOCKET_PATH_CONNECT")
+
   effopts+=("${filteredOpts[@]}")
 
   echo "cardano-node run ${effopts[*]}"
@@ -189,6 +210,8 @@ do
       --shelley-kes-key) CARDANO_SHELLEY_KES_KEY=${val}; found=true;;
       --shelley-vrf-key) CARDANO_SHELLEY_VRF_KEY=${val}; found=true;;
       --shelley-operational-certificate) CARDANO_SHELLEY_OPERATIONAL_CERTIFICATE=${val}; found=true;;
+      --tracer-socket-path-accept) CARDANO_TRACER_SOCKET_PATH_ACCEPT=${val}; found=true;;
+      --tracer-socket-path-connect) CARDANO_TRACER_SOCKET_PATH_CONNECT=${val}; found=true;;
     esac
 
     if [[ $found == true ]]; then
@@ -198,7 +221,10 @@ do
 done
 
 # Filter blank args from match removal above
-filteredOpts=(); for arg in "${options[@]}"; do [[ $arg ]] && filteredOpts+=("$arg"); done
+filteredOpts=()
+for arg in "${options[@]}"; do
+  [[ -n $arg ]] && filteredOpts+=("$arg")
+done
 
 printRunEnv
 writeRootEnv
diff --git a/nix/docker/context/tracer/bin/run-tracer b/nix/docker/context/tracer/bin/run-tracer
index 23c0d3a4fca..f14a27f6e3a 100755
--- a/nix/docker/context/tracer/bin/run-tracer
+++ b/nix/docker/context/tracer/bin/run-tracer
@@ -85,7 +85,10 @@ do
 done
 
 # Filter blank args from match removal above
-filteredOpts=(); for arg in "${options[@]}"; do [[ $arg ]] && filteredOpts+=("$arg"); done
+filteredOpts=()
+for arg in "${options[@]}"; do
+  [[ -n $arg ]] && filteredOpts+=("$arg")
+done
 
 printRunEnv
 writeRootEnv