Report unreachable daemon discovery errors

daemon/AGENTS.md

@@ -11,6 +11,9 @@ databases, command parsing, and shutdown policy belong to the caller.
 
 - Never infer live daemon state from a runtime record alone. Probe the endpoint
   before claiming a process is reachable.
+- With PID-aware discovery, retain a live record's probe failure until another
+  reachable record wins. Absence is a definite result, not a fallback for an
+  endpoint that could not be probed.
 - Treat process-creation identity as opaque and exact-match only. An unknown
   identity never authorizes destructive action against a process or record.
 - Never send a bearer credential to a runtime-record endpoint before it proves

daemon/manager_test.go

@@ -84,6 +84,40 @@ func TestManagerFindSkipsIncompatibleDaemon(t *testing.T) {
 	assert.False(t, ok)
 }
 
+func TestManagerEnsureDoesNotStartWhenDiscoveryIsUnreachable(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not ready", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+	store := daemon.RuntimeStore{Dir: t.TempDir()}
+	_, err := store.Write(daemon.NewRuntimeRecord("tool", "v1", daemon.Endpoint{
+		Network: daemon.NetworkTCP,
+		Address: listenerAddr(t, server),
+	}))
+	require.NoError(err)
+
+	started := false
+	manager := daemon.Manager{
+		Store: store,
+		Discover: daemon.DiscoverOptions{
+			Probe:           daemon.ProbeOptions{ExpectedService: "tool"},
+			RequirePIDAlive: true,
+		},
+		Start: func(context.Context) error {
+			started = true
+			return nil
+		},
+	}
+
+	_, _, err = manager.Ensure(context.Background(), time.Second)
+	require.Error(err)
+	require.ErrorIs(err, daemon.ErrDaemonUnreachable)
+	assert.False(started)
+}
+
 func TestManagerFindScansPastIncompatibleDaemon(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)

daemon/probe.go

@@ -133,11 +133,43 @@ func probeHTTP(
 type DiscoverOptions struct {
 	Probe ProbeOptions
 	// Proof enables proof-of-possession probing for each runtime record.
-	Proof           *Proof
+	Proof *Proof
+	// RequirePIDAlive rejects dead processes and definite process-identity
+	// mismatches. It also makes failed probes indeterminate: Discover returns an
+	// UnreachableError when no later record succeeds. The false zero value keeps
+	// the earlier compatibility behavior, where failed probes are skipped and
+	// discovery can report definite absence.
 	RequirePIDAlive bool
 	Accept          func(RuntimeRecord, PingInfo) bool
 }
 
+// ErrDaemonUnreachable reports a live runtime record whose endpoint could not
+// be proved usable. It classifies every Probe or proof-of-possession failure,
+// including transport, HTTP, decoding, service-identity, and proof errors.
+// Callers should inspect the wrapped error before choosing operator guidance.
+var ErrDaemonUnreachable = errors.New("daemon is unreachable")
+
+// UnreachableError retains the runtime and probe failure for a live daemon
+// candidate that discovery could not safely select.
+type UnreachableError struct {
+	Record   RuntimeRecord
+	Endpoint Endpoint
+	Err      error
+}
+
+func (e *UnreachableError) Error() string {
+	return fmt.Sprintf("%v: pid %d at %s: %v",
+		ErrDaemonUnreachable, e.Record.PID, e.Endpoint.ConfigAddress(), e.Err)
+}
+
+// Unwrap preserves both the classification and the endpoint probe failure.
+func (e *UnreachableError) Unwrap() []error {
+	if e.Err == nil {
+		return []error{ErrDaemonUnreachable}
+	}
+	return []error{ErrDaemonUnreachable, e.Err}
+}
+
 // Discover scans runtime records and returns the first live daemon.
 func Discover(ctx context.Context, store RuntimeStore, opts DiscoverOptions) (RuntimeRecord, PingInfo, bool, error) {
 	if err := ctx.Err(); err != nil {
@@ -147,23 +179,35 @@ func Discover(ctx context.Context, store RuntimeStore, opts DiscoverOptions) (Ru
 	if err != nil {
 		return RuntimeRecord{}, PingInfo{}, false, err
 	}
+	var unreachable error
 	for _, rec := range records {
 		if err := ctx.Err(); err != nil {
 			return RuntimeRecord{}, PingInfo{}, false, err
 		}
-		if opts.RequirePIDAlive && !ProcessAlive(rec.PID) {
-			continue
+		if opts.RequirePIDAlive {
+			if !ProcessAlive(rec.PID) ||
+				CompareRuntimeProcessIdentity(rec) == ProcessIdentityMismatch {
+				continue
+			}
 		}
+		ep := rec.Endpoint()
 		var info PingInfo
 		if opts.Proof == nil {
-			info, err = Probe(ctx, rec.Endpoint(), opts.Probe)
+			info, err = Probe(ctx, ep, opts.Probe)
 		} else {
 			info, err = opts.Proof.Probe(ctx, rec, opts.Probe)
 		}
 		if err != nil {
 			if ctxErr := ctx.Err(); ctxErr != nil {
 				return RuntimeRecord{}, PingInfo{}, false, ctxErr
 			}
+			if opts.RequirePIDAlive && unreachable == nil {
+				unreachable = &UnreachableError{
+					Record:   rec,
+					Endpoint: ep,
+					Err:      err,
+				}
+			}
 			continue
 		}
 		if opts.RequirePIDAlive && info.PID != rec.PID {
@@ -174,5 +218,5 @@ func Discover(ctx context.Context, store RuntimeStore, opts DiscoverOptions) (Ru
 		}
 		return rec, info, true, nil
 	}
-	return RuntimeRecord{}, PingInfo{}, false, nil
+	return RuntimeRecord{}, PingInfo{}, false, unreachable
 }

daemon/probe_test.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -169,6 +170,141 @@ func TestDiscoverRejectsPIDMismatchWhenRequiringLivePID(t *testing.T) {
 	assert.False(t, ok)
 }
 
+func TestDiscoverSkipsMismatchedProcessIdentityWithoutProbing(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	livePID := startLivePIDHelper(t)
+	recordedIdentity, ok := daemon.ReadProcessIdentity(os.Getpid())
+	require.True(ok)
+	require.Equal(daemon.ProcessIdentityMismatch,
+		daemon.CompareProcessIdentity(livePID, recordedIdentity))
+
+	var probes atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		probes.Add(1)
+		http.Error(w, "must not be probed", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	store := daemon.RuntimeStore{Dir: t.TempDir()}
+	_, err := store.Write(daemon.RuntimeRecord{
+		PID:             livePID,
+		ProcessIdentity: recordedIdentity,
+		Network:         daemon.NetworkTCP,
+		Address:         listenerAddr(t, server),
+		Service:         "kata",
+		StartedAt:       time.Now(),
+	})
+	require.NoError(err)
+
+	_, _, found, err := daemon.Discover(context.Background(), store, daemon.DiscoverOptions{
+		Probe:           daemon.ProbeOptions{ExpectedService: "kata"},
+		RequirePIDAlive: true,
+	})
+	require.NoError(err)
+	assert.False(found)
+	assert.Zero(probes.Load())
+}
+
+func TestDiscoverWithoutPIDCheckKeepsFailedProbeAsAbsence(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not ready", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	store := daemon.RuntimeStore{Dir: t.TempDir()}
+	_, err := store.Write(daemon.NewRuntimeRecord("kata", "v1", daemon.Endpoint{
+		Network: daemon.NetworkTCP,
+		Address: listenerAddr(t, server),
+	}))
+	require.NoError(t, err)
+
+	_, _, found, err := daemon.Discover(context.Background(), store, daemon.DiscoverOptions{
+		Probe: daemon.ProbeOptions{ExpectedService: "kata"},
+	})
+	require.NoError(t, err)
+	assert.False(t, found)
+}
+
+func TestDiscoverReturnsUnreachableErrorForLiveRuntime(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not ready", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	rec := daemon.NewRuntimeRecord("kata", "v1", daemon.Endpoint{
+		Network: daemon.NetworkTCP,
+		Address: listenerAddr(t, server),
+	})
+	store := daemon.RuntimeStore{Dir: t.TempDir()}
+	_, err := store.Write(rec)
+	require.NoError(err)
+
+	_, _, ok, err := daemon.Discover(context.Background(), store, daemon.DiscoverOptions{
+		Probe:           daemon.ProbeOptions{ExpectedService: "kata"},
+		RequirePIDAlive: true,
+	})
+	require.Error(err)
+	assert.False(ok)
+	require.ErrorIs(err, daemon.ErrDaemonUnreachable)
+	unreachable, ok := errors.AsType[*daemon.UnreachableError](err)
+	if !ok || unreachable == nil {
+		require.FailNow("expected UnreachableError")
+		return
+	}
+	assert.Equal(rec.PID, unreachable.Record.PID)
+	assert.Equal(rec.Endpoint(), unreachable.Endpoint)
+	require.Error(unreachable.Err)
+	require.ErrorIs(err, unreachable.Err)
+}
+
+func TestDiscoverScansPastUnreachableLiveRuntime(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	unreachableServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not ready", http.StatusServiceUnavailable)
+	}))
+	defer unreachableServer.Close()
+	reachableServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = fmt.Fprintf(w, `{"ok":true,"service":"kata","pid":%d}`, os.Getpid())
+	}))
+	defer reachableServer.Close()
+
+	livePID := startLivePIDHelper(t)
+	identity, ok := daemon.ReadProcessIdentity(livePID)
+	require.True(ok)
+	store := daemon.RuntimeStore{Dir: t.TempDir()}
+	_, err := store.Write(daemon.RuntimeRecord{
+		PID:             livePID,
+		ProcessIdentity: identity,
+		Network:         daemon.NetworkTCP,
+		Address:         listenerAddr(t, unreachableServer),
+		Service:         "kata",
+		StartedAt:       time.Now().Add(-time.Minute),
+	})
+	require.NoError(err)
+	reachable := daemon.NewRuntimeRecord("kata", "v1", daemon.Endpoint{
+		Network: daemon.NetworkTCP,
+		Address: listenerAddr(t, reachableServer),
+	})
+	_, err = store.Write(reachable)
+	require.NoError(err)
+
+	found, info, ok, err := daemon.Discover(context.Background(), store, daemon.DiscoverOptions{
+		Probe:           daemon.ProbeOptions{ExpectedService: "kata"},
+		RequirePIDAlive: true,
+	})
+	require.NoError(err)
+	require.True(ok)
+	assert.Equal(reachable.PID, found.PID)
+	assert.Equal("kata", info.Service)
+}
+
 func TestManagerFindDoesNotDiscloseCredentialBeforeProof(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
@@ -212,8 +348,9 @@ func TestManagerFindDoesNotDiscloseCredentialBeforeProof(t *testing.T) {
 	}
 
 	_, _, ok, err := manager.Find(context.Background())
-	require.NoError(err)
+	require.Error(err)
 	assert.False(ok)
+	require.ErrorIs(err, daemon.ErrDaemonUnreachable)
 	assert.False(credentialDisclosed.Load(), "credential material reached an unproved endpoint")
 }