feat: add prefix health checks

cmd/bundle.go

@@ -6,8 +6,8 @@ import (
 	"os"
 
 	"github.com/encodeous/nylon/state"
+	"github.com/goccy/go-yaml"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
 )
 
 var sealCmd = &cobra.Command{

core/entrypoint.go

@@ -19,8 +19,8 @@ import (
 	"github.com/encodeous/nylon/perf"
 	"github.com/encodeous/nylon/state"
 	"github.com/encodeous/tint"
+	"github.com/goccy/go-yaml"
 	slogmulti "github.com/samber/slog-multi"
-	"gopkg.in/yaml.v3"
 )
 
 func setupDebugging() {

core/ipc.go

@@ -84,9 +84,9 @@ func HandleNylonIPCGet(s *state.State, rw *bufio.ReadWriter) error {
 		for prefix, adv := range s.Advertised {
 			timeRem := adv.Expiry.Sub(time.Now())
 			if timeRem > time.Hour*24 {
-				rt = append(rt, fmt.Sprintf(" - %s expires never nh %s", prefix, adv.NodeId))
+				rt = append(rt, fmt.Sprintf(" - %s expires never nh %s metric %d", prefix, adv.NodeId, adv.MetricFn()))
 			} else {
-				rt = append(rt, fmt.Sprintf(" - %s expires %.2fs nh %s", prefix, timeRem.Seconds(), adv.NodeId))
+				rt = append(rt, fmt.Sprintf(" - %s expires %.2fs nh %s metric %d", prefix, timeRem.Seconds(), adv.NodeId, adv.MetricFn()))
 			}
 		}
 		slices.Sort(rt)

core/nylon.go

@@ -3,6 +3,7 @@ package core
 import (
 	"context"
 	"net"
+	"net/netip"
 	"time"
 
 	"github.com/encodeous/nylon/polyamide/device"
@@ -13,12 +14,13 @@ import (
 
 // Nylon struct must be thread safe, since it can receive packets through PolyReceiver
 type Nylon struct {
-	PingBuf *ttlcache.Cache[uint64, EpPing]
-	Device  *device.Device
-	Tun     tun.Device
-	wgUapi  net.Listener
-	env     *state.Env
-	itfName string
+	PingBuf             *ttlcache.Cache[uint64, EpPing]
+	Device              *device.Device
+	Tun                 tun.Device
+	wgUapi              net.Listener
+	env                 *state.Env
+	itfName             string
+	prevInstalledRoutes []netip.Prefix
 }
 
 func (n *Nylon) Init(s *state.State) error {
@@ -90,6 +92,12 @@ func (n *Nylon) Init(s *state.State) error {
 		return n.probeNew(s)
 	}, state.ProbeDiscoveryDelay)
 
+	// prefix healthcheck
+	for _, ph := range s.GetNode(s.Id).Prefixes {
+		s.Log.Info("starting prefix healthcheck", "prefix", ph.GetPrefix())
+		ph.Start(s.Log)
+	}
+
 	err = n.initPassiveClient(s)
 	if err != nil {
 		return err
@@ -107,6 +115,9 @@ func (n *Nylon) Init(s *state.State) error {
 
 func (n *Nylon) Cleanup(s *state.State) error {
 	n.PingBuf.Stop()
+	for _, ph := range s.GetNode(s.Id).Prefixes {
+		ph.Stop()
+	}
 
 	return n.cleanupWireGuard(s)
 }

core/nylon_distribution.go

@@ -9,7 +9,7 @@ import (
 	"os"
 
 	"github.com/encodeous/nylon/state"
-	"gopkg.in/yaml.v3"
+	"github.com/goccy/go-yaml"
 )
 
 // fetches and unbundles central config from url

core/nylon_passive.go

@@ -28,7 +28,7 @@ func scanPassivePeers(s *state.State) error {
 			for _, prefix := range ncfg.Prefixes {
 				for _, neigh := range s.Neighbours {
 					for _, route := range neigh.Routes {
-						if route.Prefix == prefix && route.NodeId != s.Id && route.FD.Metric != state.INF {
+						if route.Prefix == prefix.GetPrefix() && route.NodeId != s.Id && route.FD.Metric != state.INF {
 							hasOtherAdvertisers = true
 							goto foundAdvertiser
 						}
@@ -43,7 +43,7 @@ func scanPassivePeers(s *state.State) error {
 			if s.IsClient(*nid) {
 				// we have a passive client
 				for _, newPrefix := range ncfg.Prefixes {
-					recentlyAdvertised := r.hasRecentlyAdvertised(newPrefix)
+					recentlyAdvertised := r.hasRecentlyAdvertised(newPrefix.GetPrefix())
 					if recentlyUpdated || !hasOtherAdvertisers && recentlyAdvertised {
 						r.updatePassiveClient(s, newPrefix, *nid, !recentlyUpdated)
 					}

core/nylon_wireguard.go

@@ -82,15 +82,15 @@ listen_port=%d
 
 	// configure system networking
 
-	if !s.NoNetConfigure {
-		// run pre-up commands
-		for _, cmd := range s.PreUp {
-			err = ExecSplit(s.Log, cmd)
-			if err != nil {
-				s.Log.Error("failed to run pre-up command", "err", err)
-			}
+	// run pre-up commands
+	for _, cmd := range s.PreUp {
+		err = ExecSplit(s.Log, cmd)
+		if err != nil {
+			s.Log.Error("failed to run pre-up command", "err", err)
 		}
+	}
 
+	if !s.NoNetConfigure {
 		for _, addr := range s.GetRouter(s.Id).Addresses {
 			err := ConfigureAlias(s.Log, itfName, addr)
 			if err != nil {
@@ -99,44 +99,33 @@ listen_port=%d
 		}
 
 		err = InitInterface(s.Log, itfName)
-
 		if err != nil {
 			return err
 		}
+	}
 
-		// configure prefixes
-		exclude := append(state.SubtractPrefix(s.CentralCfg.ExcludeIPs, s.IncludeIPs), s.LocalCfg.ExcludeIPs...)
-		for _, excl := range exclude {
-			s.Log.Debug("Computed Exclude Prefix", "prefix", excl.String())
-		}
-		computed := state.SubtractPrefix(s.GetPrefixes(), exclude)
-		for _, pre := range computed {
-			s.Log.Debug("Computed Prefix", "prefix", pre.String())
-		}
-		for _, prefix := range computed {
-			err := ConfigureRoute(s.Log, n.Tun, itfName, prefix)
-			if err != nil {
-				s.Log.Error("failed to configure route", "err", err)
-			}
-		}
-
-		// run post-up commands
-		for _, cmd := range s.PostUp {
-			err = ExecSplit(s.Log, cmd)
-			if err != nil {
-				s.Log.Error("failed to run post-up command", "err", err)
-			}
+	// run post-up commands
+	for _, cmd := range s.PostUp {
+		err = ExecSplit(s.Log, cmd)
+		if err != nil {
+			s.Log.Error("failed to run post-up command", "err", err)
 		}
 	}
 
 	// init wireguard related tasks
-
 	s.RepeatTask(UpdateWireGuard, state.ProbeDelay)
 
 	return nil
 }
 
 func (n *Nylon) cleanupWireGuard(s *state.State) error {
+	// remove routes
+	for _, route := range n.prevInstalledRoutes {
+		err := RemoveRoute(s.Log, n.Tun, n.itfName, route)
+		if err != nil {
+			s.Log.Error("failed to remove route", "err", err)
+		}
+	}
 	// run pre-down commands
 	for _, cmd := range s.PreUp {
 		err := ExecSplit(s.Log, cmd)
@@ -197,5 +186,33 @@ func UpdateWireGuard(s *state.State) error {
 		wgPeer := dev.LookupPeer(device.NoisePublicKey(pcfg.PubKey))
 		wgPeer.SetEndpoints(eps)
 	}
+
+	// configure changed route table entries
+	if !s.NoNetConfigure {
+		router := Get[*NylonRouter](s)
+		newEntries := router.ComputeSysRouteTable()
+		oldEntries := n.prevInstalledRoutes
+		for _, oldEntry := range oldEntries {
+			if !slices.Contains(newEntries, oldEntry) {
+				// uninstall route
+				s.Log.Debug("removing old route", "prefix", oldEntry.String())
+				err := RemoveRoute(s.Log, n.Tun, n.itfName, oldEntry)
+				if err != nil {
+					s.Log.Error("failed to remove route", "err", err)
+				}
+			}
+		}
+		for _, newEntry := range newEntries {
+			if !slices.Contains(oldEntries, newEntry) {
+				// install route
+				s.Log.Debug("installing new route", "prefix", newEntry.String())
+				err := ConfigureRoute(s.Log, n.Tun, n.itfName, newEntry)
+				if err != nil {
+					s.Log.Error("failed to configure route", "err", err)
+				}
+			}
+		}
+		n.prevInstalledRoutes = newEntries
+	}
 	return nil
 }

core/router.go

@@ -170,7 +170,12 @@ func (r *NylonRouter) Init(s *state.State) error {
 	}
 	maxTime := time.Unix(1<<63-62135596801, 999999999)
 	for _, prefix := range s.Env.GetRouter(s.Id).Prefixes {
-		s.RouterState.Advertised[prefix] = state.Advertisement{NodeId: s.Id, Expiry: maxTime, IsPassiveHold: false}
+		s.RouterState.Advertised[prefix.GetPrefix()] = state.Advertisement{
+			NodeId:        s.Id,
+			Expiry:        maxTime,
+			IsPassiveHold: false,
+			MetricFn:      prefix.GetMetric,
+		}
 	}
 
 	s.Log.Debug("schedule router tasks")
@@ -188,25 +193,49 @@ func (r *NylonRouter) Init(s *state.State) error {
 	return nil
 }
 
-func (r *NylonRouter) updatePassiveClient(s *state.State, prefix netip.Prefix, node state.NodeId, passiveHold bool) {
+// ComputeSysRouteTable computes: computed = prefixes - (((r.CentralCfg.ExcludeIPs U selected self prefixes) - r.LocalCfg.UnexcludeIPs) U r.LocalCfg.ExcludeIPs)
+func (r *NylonRouter) ComputeSysRouteTable() []netip.Prefix {
+	prefixes := make([]netip.Prefix, 0)
+	selectedSelf := make(map[netip.Prefix]struct{})
+	for entry, v := range r.ForwardTable.All() {
+		prefixes = append(prefixes, entry)
+		if v.Nh == r.Id {
+			selectedSelf[entry] = struct{}{}
+		}
+	}
+
+	defaultExcludes := r.CentralCfg.ExcludeIPs
+	for p := range selectedSelf {
+		defaultExcludes = append(defaultExcludes, p)
+	}
+	exclude := append(state.SubtractPrefix(defaultExcludes, r.LocalCfg.UnexcludeIPs), r.LocalCfg.ExcludeIPs...)
+	return state.SubtractPrefix(prefixes, exclude)
+}
+
+func (r *NylonRouter) updatePassiveClient(s *state.State, prefix state.PrefixHealthWrapper, node state.NodeId, passiveHold bool) {
 	// inserts an artificial route into the table
 
 	hasPassiveHold := false
-	old, ok := s.RouterState.Advertised[prefix]
+	old, ok := s.RouterState.Advertised[prefix.GetPrefix()]
 	if ok && old.NodeId == node {
 		hasPassiveHold = old.IsPassiveHold
 	}
 
 	if passiveHold && !hasPassiveHold {
 		// the first time we enter passive hold, we should increment the seqno to prevent other nodes from switching away from the route
 		// this reduces a lot of route flapping when the client wakes up, sends some traffic and then goes back to sleep
-		r.SetSeqno(prefix, s.RouterState.GetSeqno(prefix)+1)
+		r.SetSeqno(prefix.GetPrefix(), s.RouterState.GetSeqno(prefix.GetPrefix())+1)
 	}
 
-	s.Advertised[prefix] = state.Advertisement{
+	prefix.Start(s.Log)
+	s.Advertised[prefix.GetPrefix()] = state.Advertisement{
 		NodeId:        node,
 		Expiry:        time.Now().Add(state.ClientKeepaliveInterval),
 		IsPassiveHold: passiveHold,
+		MetricFn:      prefix.GetMetric,
+		ExpiryFn: func() {
+			prefix.Stop()
+		},
 	}
 }
 

core/router_algo.go

@@ -143,6 +143,9 @@ func RunGC(s *state.RouterState, r Router) {
 		if now.After(exp.Expiry) {
 			// advertised route expired, remove it
 			delete(s.Advertised, svc)
+			if exp.ExpiryFn != nil {
+				exp.ExpiryFn()
+			}
 		}
 	}
 
@@ -441,6 +444,8 @@ func ComputeRoutes(s *state.RouterState, r Router) {
 		if adv.IsPassiveHold {
 			// The metric should be high enough so that if the passive client connects to any other node, our route will be immediately unselected
 			advMetric = state.INFM / 2
+		} else if adv.MetricFn != nil {
+			advMetric = adv.MetricFn()
 		}
 		newTable[prefix] = state.SelRoute{
 			PubRoute: state.PubRoute{
@@ -498,9 +503,6 @@ func ComputeRoutes(s *state.RouterState, r Router) {
 
 		// enumerate through neighbour advertisements
 		for S, adv := range neigh.Routes {
-			if _, ok := s.Advertised[S.Prefix]; ok {
-				continue // skip self routes
-			}
 			prefix := S.Prefix
 
 			// Cost(A, B) + Cost(S, B)