Make max delay for connection retry configurable

cardano-tracer/test/Cardano/Tracer/Test/Forwarder.hs

@@ -20,13 +20,12 @@ import           Cardano.Tracer.Configuration (Verbosity (..))
 import           Cardano.Tracer.Test.TestSetup
 import           Cardano.Tracer.Test.Utils
 import           Cardano.Tracer.Utils
-import qualified Network.Mux as Mux
 import           Ouroboros.Network.Driver.Limits (ProtocolTimeLimits)
 import           Ouroboros.Network.ErrorPolicy (nullErrorPolicies)
 import           Ouroboros.Network.IOManager (IOManager, withIOManager)
 import           Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (..),
-                   MiniProtocolNum (..), OuroborosApplication (..),
-                   RunMiniProtocol (..), miniProtocolLimits, miniProtocolNum, miniProtocolRun)
+                   MiniProtocolNum (..), OuroborosApplication (..), RunMiniProtocol (..),
+                   miniProtocolLimits, miniProtocolNum, miniProtocolRun)
 import           Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec,
                    codecHandshake, noTimeLimitsHandshake)
 import           Ouroboros.Network.Protocol.Handshake.Type (Handshake)
@@ -52,6 +51,7 @@ import           Data.Time.Clock (getCurrentTime)
 import           Data.Void (Void, absurd)
 import           Data.Word (Word16)
 import           GHC.Generics
+import qualified Network.Mux as Mux
 import           System.Directory
 import qualified System.Metrics as EKG
 import qualified System.Metrics.Configuration as EKGF

trace-dispatcher/doc/trace-dispatcher.md

@@ -332,6 +332,7 @@ TraceOptionForwarder: # Configure the forwarder
     address:
       filePath: /tmp/forwarder.sock
     mode: Initiator
+    maxReconnectDelay: 20
 
 # Frequency of Peer messages set to two seconds
 TraceOptionPeerFrequency: 2000
@@ -367,7 +368,8 @@ The same in JSON looks like this:
     "address": {
       "filePath": "/tmp/forwarder.sock"
     },
-    "mode": "Initiator"
+    "mode": "Initiator",
+    "maxReconnectDelay": 30
   },
   "TraceOptionPeerFrequency": 2000,
   "TraceOptionMetricsPrefix": "cardano.node"

trace-dispatcher/src/Cardano/Logging/Forwarding.hs

@@ -12,14 +12,13 @@ module Cardano.Logging.Forwarding
 import           Cardano.Logging.Types
 import           Cardano.Logging.Utils (runInLoop)
 import           Cardano.Logging.Version
-import qualified Network.Mux as Mux
 import           Ouroboros.Network.Driver.Limits (ProtocolTimeLimits)
 import           Ouroboros.Network.ErrorPolicy (nullErrorPolicies)
 import           Ouroboros.Network.IOManager (IOManager)
 import           Ouroboros.Network.Magic (NetworkMagic)
 import           Ouroboros.Network.Mux (MiniProtocol (..), MiniProtocolLimits (..),
-                   MiniProtocolNum (..), OuroborosApplication (..),
-                   RunMiniProtocol (..), miniProtocolLimits, miniProtocolNum, miniProtocolRun)
+                   MiniProtocolNum (..), OuroborosApplication (..), RunMiniProtocol (..),
+                   miniProtocolLimits, miniProtocolNum, miniProtocolRun)
 import           Ouroboros.Network.Protocol.Handshake.Codec (cborTermVersionDataCodec,
                    codecHandshake, noTimeLimitsHandshake)
 import           Ouroboros.Network.Protocol.Handshake.Type (Handshake)
@@ -34,13 +33,14 @@ import           Ouroboros.Network.Socket (AcceptedConnectionsLimit (..), Connec
 
 import           Codec.CBOR.Term (Term)
 import           Control.Concurrent.Async (async, race_, wait)
-import           Control.Monad (void)
 import           Control.Exception (throwIO)
+import           Control.Monad (void)
 import           Control.Monad.IO.Class
 import           "contra-tracer" Control.Tracer (Tracer, contramap, nullTracer, stdoutTracer)
 import qualified Data.ByteString.Lazy as LBS
 import           Data.Void (Void, absurd)
 import           Data.Word (Word16)
+import qualified Network.Mux as Mux
 import           System.IO (hPutStrLn, stderr)
 import qualified System.Metrics as EKG
 import qualified System.Metrics.Configuration as EKGF
@@ -88,12 +88,14 @@ initForwardingDelayed iomgr config magic ekgStore tracerSocketMode = liftIO $ do
       forwardSink
       dpStore
       tracerSocketMode
+      maxReconnectDelay
   pure (forwardSink, dpStore, kickoffForwarder)
  where
   p = maybe "" fst tracerSocketMode
   connSize = tofConnQueueSize config
   disconnSize = tofDisconnQueueSize config
   verbosity = tofVerbosity config
+  maxReconnectDelay = tofMaxReconnectDelay config
 
   ekgConfig :: EKGF.ForwarderConfiguration
   ekgConfig =
@@ -148,10 +150,11 @@ launchForwarders
   -> ForwardSink TraceObject
   -> DataPointStore
   -> Maybe (FilePath, ForwarderMode)
+  -> Word
   -> IO ()
 launchForwarders iomgr magic
                  ekgConfig tfConfig dpfConfig
-                 ekgStore sink dpStore tracerSocketMode =
+                 ekgStore sink dpStore tracerSocketMode maxReconnectDelay =
   -- If 'tracerSocketMode' is not specified, it's impossible to establish
   -- network connection with acceptor application (for example, 'cardano-tracer').
   -- In this case, we should not lauch forwarders.
@@ -171,7 +174,9 @@ launchForwarders iomgr magic
              dpStore
              socketPath
              mode)
-          socketPath 1
+          socketPath
+          1
+          maxReconnectDelay
 
 launchForwardersViaLocalSocket
   :: IOManager

trace-dispatcher/src/Cardano/Logging/Types.hs

@@ -451,9 +451,10 @@ instance AE.FromJSON Verbosity where
                                     <> "Unknown Verbosity: " <> show other
 
 data TraceOptionForwarder = TraceOptionForwarder {
-    tofConnQueueSize    :: Word
-  , tofDisconnQueueSize :: Word
-  , tofVerbosity        :: Verbosity
+    tofConnQueueSize       :: Word
+  , tofDisconnQueueSize    :: Word
+  , tofVerbosity           :: Verbosity
+  , tofMaxReconnectDelay   :: Word
 } deriving (Eq, Generic, Ord, Show, AE.ToJSON)
 
 -- A word regarding queue sizes:
@@ -474,17 +475,19 @@ data TraceOptionForwarder = TraceOptionForwarder {
 instance AE.FromJSON TraceOptionForwarder where
     parseJSON (AE.Object obj) =
       TraceOptionForwarder
-        <$> obj AE..:? "connQueueSize"    AE..!= 1024
-        <*> obj AE..:? "disconnQueueSize" AE..!= 2048
-        <*> obj AE..:? "verbosity"        AE..!= Minimum
+        <$> obj AE..:? "connQueueSize"      AE..!= 1024
+        <*> obj AE..:? "disconnQueueSize"   AE..!= 2048
+        <*> obj AE..:? "verbosity"          AE..!= Minimum
+        <*> obj AE..:? "maxReconnectDelay"  AE..!= 60
     parseJSON _ = mempty
 
 
 defaultForwarder :: TraceOptionForwarder
 defaultForwarder = TraceOptionForwarder {
-    tofConnQueueSize    = 1024
-  , tofDisconnQueueSize = 2048
-  , tofVerbosity        = Minimum
+    tofConnQueueSize       = 1024
+  , tofDisconnQueueSize    = 2048
+  , tofVerbosity           = Minimum
+  , tofMaxReconnectDelay   = 60
 }
 
 instance AE.FromJSON ForwarderMode where

trace-dispatcher/src/Cardano/Logging/Utils.hs

@@ -15,13 +15,13 @@ import           GHC.Conc (labelThread, myThreadId)
 
 -- | Run monadic action in a loop. If there's an exception, it will re-run
 --   the action again, after pause that grows.
-runInLoop :: IO () -> FilePath -> Word -> IO ()
-runInLoop action localSocket prevDelayInSecs =
+runInLoop :: IO () -> FilePath -> Word -> Word -> IO ()
+runInLoop action localSocket prevDelayInSecs maxReconnectDelay =
   tryJust excludeAsyncExceptions action >>= \case
     Left e -> do
       logTrace $ "connection with " <> show localSocket <> " failed: " <> show e
       threadDelay . fromIntegral $ currentDelayInSecs * 1000000
-      runInLoop action localSocket currentDelayInSecs
+      runInLoop action localSocket currentDelayInSecs maxReconnectDelay
     Right _ -> return ()
  where
   excludeAsyncExceptions e =
@@ -32,9 +32,7 @@ runInLoop action localSocket prevDelayInSecs =
   logTrace = traceWith stdoutTracer
 
   currentDelayInSecs =
-    if prevDelayInSecs < 60
-      then prevDelayInSecs * 2
-      else 60 -- After we reached 60+ secs delay, repeat an attempt every minute.
+    min (prevDelayInSecs * 2) maxReconnectDelay
 
 -- | Convenience function for a Show instance to be converted to text immediately
 {-# INLINE showT #-}
@@ -45,6 +43,7 @@ showT = T.pack . show
 showTHex :: Integral a => a -> T.Text
 showTHex = TL.toStrict . T.toLazyText . T.hexadecimal
 
+{-# INLINE showTReal #-}
 showTReal :: RealFloat a => a -> T.Text
 showTReal = TL.toStrict . T.toLazyText . T.realFloat
 

trace-dispatcher/test/Cardano/Logging/Test/Unit/Configuration.hs

@@ -55,6 +55,7 @@ config1 = TraceConfig {
         tofConnQueueSize = 100
       , tofDisconnQueueSize = 1000
       , tofVerbosity = Minimum
+      , tofMaxReconnectDelay = 60
       }
     , tcNodeName = Nothing
     , tcPeerFrequency = Nothing
@@ -73,6 +74,7 @@ config2 = TraceConfig {
         tofConnQueueSize = 100
       , tofDisconnQueueSize = 1000
       , tofVerbosity = Minimum
+      , tofMaxReconnectDelay = 60
       }
     , tcNodeName = Just "node-1"
     , tcPeerFrequency = Nothing

trace-dispatcher/test/Cardano/Logging/Test/Unit/Documentation.hs

@@ -38,6 +38,7 @@ config1 = TraceConfig {
         tofConnQueueSize = 100
       , tofDisconnQueueSize = 1000
       , tofVerbosity = Minimum
+      , tofMaxReconnectDelay = 60
       }
     , tcNodeName = Nothing
     , tcPeerFrequency = Nothing