build multiple candidate payloads per slot for reorg and payload-miss testing

CLAUDE.md

@@ -198,6 +198,43 @@ npm run clean
      still-open slots (`> currentSlot`, not yet frozen) — past slots keep their
      recorded history instead of being described by today's rules.
 
+0b. **Chain view / head tracker** (`pkg/chain/headtracker.go`) — buildoor's own
+   canonical-chain oracle: current head, bounded block-by-root ancestry cache
+   (shared with the inclusion tracker), per-block Gloas payload reveal status
+   (payload-available events + child-block evidence + PAYLOAD_DUE fallback),
+   reorg detection (head-change events with depth/common-ancestor; the beacon
+   `chain_reorg` SSE topic is subscribed as a cross-check), and build-parent
+   candidate resolution. Also: epoch-stats refetch when a head event's duty
+   dependent root proves a reorg crossed the epoch boundary.
+
+0c. **Build candidates** — a slot may build SEVERAL payloads, one per parent
+   tuple (candidate keys: `parent_full`, `parent_empty` = Gloas payload miss,
+   `grandparent_full` = deliberate reorg of the head block,
+   `grandparent_empty`). Policy per key: `auto` (chain signals: parent payload
+   reveal status, head-vote weakness below `build.auto_weak_head_pct`) /
+   `always` / `never` via `build.candidate_*` settings, per-slot overridable
+   through the action plan's `build.candidates` map (resolved into
+   `FrozenPlan.Build.CandidateModes`). Attributes are cached per
+   (slot, parent_root, parent_hash) variant — last-writer-wins per variant —
+   sanitized against the chain view (Grandine's wrong miss-case parent is
+   corrected, missing parent numbers backfilled) and synthesized for uncovered
+   candidates (empty-parent withdrawals from the parent slot's attributes;
+   grandparent candidates re-target the parent slot's attributes, refused
+   across epoch boundaries). Sequential builds run speculative candidates
+   first, canonical last (`build.parallel` opts into concurrent engine
+   builds). Consumers select at request time: p2p bids per
+   `epbs.bid_candidate` (auto = match chain view, sticky per slot unless
+   `epbs.bid_candidate_switch`; `all` = deliberate multi-parent gossip),
+   builder-api serves whichever candidate matches the requested parent
+   (`builder_api.serve_candidates` policy, optional
+   `builder_api.on_demand_build`). Reveals re-bind (rebuilt, re-signed
+   envelope) when a won payload is re-included under a different block root
+   (`reveal.rebind_on_reorg`); orphaned wins dispute pending payments and
+   clear won markers until the block returns. Flag-gated
+   `build.enforce_bid_gas_limit` adjusts built payloads to the exact
+   bid-gossip-legal gas limit (EL parent gas limit stepped toward the
+   proposer target).
+
 1. **Builder Service** (`pkg/payload_builder/`)
    - Main orchestrator for payload building
    - Subscribes to beacon node's `payload_attributes` events

cmd/root.go

@@ -61,6 +61,8 @@ func init() {
 	rootCmd.PersistentFlags().Bool("builder-api-enabled", defaults.BuilderAPIEnabled, "Enable traditional Builder API at startup (served on --api-port)")
 	rootCmd.PersistentFlags().Uint64("builder-api-subsidy", defaults.BuilderAPI.BlockValueSubsidyGwei, "Gwei added to the bid value in both Fulu (getHeader) and Gloas (ExecutionPayment) Builder API bids")
 	rootCmd.PersistentFlags().Uint64("builder-api-value-override", defaults.BuilderAPI.ValueOverrideGwei, "Absolute total value in gwei served in Builder API bids, replacing block value + subsidy (0 = disabled)")
+	rootCmd.PersistentFlags().String("builder-api-serve-candidates", defaults.BuilderAPI.ServeCandidates, "Which built candidate payloads bid requests are answered from: all, canonical_only, or a comma-separated candidate key list")
+	rootCmd.PersistentFlags().Bool("builder-api-on-demand-build", defaults.BuilderAPI.OnDemandBuild, "Build a payload on the fly when a bid request asks for a legal parent no candidate covers yet")
 	rootCmd.PersistentFlags().String("builder-api-url", defaults.BuilderAPI.BuilderURL, "Publicly reachable URL of this builder (e.g. https://builder.example.com); used to validate builder_url in SignedRequestAuthV1")
 	rootCmd.PersistentFlags().Bool("builder-api-require-auth", defaults.BuilderAPI.RequireRequestAuth, "Require SignedRequestAuthV1 on getExecutionPayloadBid requests; reject unauthenticated requests with 401")
 	rootCmd.PersistentFlags().Uint64("deposit-amount", defaults.DepositAmount, "Builder deposit amount in Gwei")
@@ -89,6 +91,18 @@ func init() {
 	rootCmd.PersistentFlags().Uint64("epbs-bid-subsidy", defaults.EPBS.BidSubsidy, "Gwei added to every bid so it clears the proposer's local-EL threshold")
 	rootCmd.PersistentFlags().Uint64("epbs-bid-value-override", defaults.EPBS.BidValueOverride, "Absolute p2p bid base value in gwei, replacing max(blockValue, bid-min) + subsidy (0 = disabled); allows underbidding the block value for testing")
 	rootCmd.PersistentFlags().Uint64("epbs-vote-threshold", defaults.EPBS.HeadVoteThresholdPct, "Head-vote participation threshold in percent; crossing it fires an immediate threshold_met update (0 = disabled)")
+	rootCmd.PersistentFlags().String("epbs-bid-candidate", defaults.EPBS.BidCandidate, "Which built candidate payload p2p bids commit to: auto, parent_full, parent_empty, grandparent_full, grandparent_empty or all")
+	rootCmd.PersistentFlags().Bool("epbs-bid-candidate-switch", defaults.EPBS.BidCandidateSwitch, "Allow the auto bid candidate selection to switch mid-slot when the chain view changes")
+
+	// Payload build candidates (reorg / payload-miss preparedness)
+	rootCmd.PersistentFlags().String("build-candidate-parent-full", defaults.Build.CandidateParentFull, "Build the normal candidate on the head block and its payload: auto, always or never")
+	rootCmd.PersistentFlags().String("build-candidate-parent-empty", defaults.Build.CandidateParentEmpty, "Build the payload-miss candidate on the head block but its execution parent: auto, always or never")
+	rootCmd.PersistentFlags().String("build-candidate-grandparent-full", defaults.Build.CandidateGrandparentFull, "Build the reorg candidate on the head block's parent: auto, always or never")
+	rootCmd.PersistentFlags().String("build-candidate-grandparent-empty", defaults.Build.CandidateGrandparentEmpty, "Build the reorg + payload-miss candidate: auto, always or never")
+	rootCmd.PersistentFlags().Bool("build-parallel", defaults.Build.Parallel, "Build selected candidates concurrently against the EL instead of sequentially (canonical first, speculative after)")
+	rootCmd.PersistentFlags().Uint64("build-speculative-build-time", defaults.Build.SpeculativeBuildTimeMs, "EL build time in ms for speculative (non parent_full) candidates (0 = use payload-build-time)")
+	rootCmd.PersistentFlags().Uint64("build-auto-weak-head-pct", defaults.Build.AutoWeakHeadPct, "Head-vote participation in percent below which the head counts as contested and auto-mode reorg candidates build (0 = disabled)")
+	rootCmd.PersistentFlags().Bool("build-enforce-bid-gas-limit", defaults.Build.EnforceBidGasLimit, "Adjust the built payload's gas limit to the exact bid-gossip-legal value when the EL ignored the proposer's target")
 
 	// Payload reveal (shared by the p2p bidder and Builder API flows)
 	rootCmd.PersistentFlags().Bool("reveal-enabled", defaults.Reveal.Enabled, "Globally enable payload reveals (per-slot action plans can still force/suppress)")
@@ -98,6 +112,7 @@ func init() {
 	rootCmd.PersistentFlags().String("reveal-broadcast-validation", defaults.Reveal.BroadcastValidation, "Envelope broadcast validation: gossip, consensus or consensus_and_equivocation")
 	rootCmd.PersistentFlags().Uint64("reveal-max-attempts", defaults.Reveal.MaxAttempts, "Total publish attempts per reveal")
 	rootCmd.PersistentFlags().Int64("reveal-retry-interval", defaults.Reveal.RetryIntervalMs, "Wait between failed reveal attempts in ms")
+	rootCmd.PersistentFlags().Bool("reveal-rebind-on-reorg", defaults.Reveal.RebindOnReorg, "Re-bind a slot's reveal (rebuilt, re-signed envelope) when our payload is re-included under a different block root after a reorg")
 
 	// Payload Build Time (0 = auto from slot time, scaled from the 12s value)
 	rootCmd.PersistentFlags().Uint64("payload-build-time", 0, "Time to allow the EL to build the payload in ms (0 = auto: 2100ms @12s, scaled to slot time)")
@@ -182,6 +197,8 @@ func initConfig() error {
 			RequireRequestAuth:    v.GetBool("builder-api-require-auth"),
 			BlockValueSubsidyGwei: v.GetUint64("builder-api-subsidy"),
 			ValueOverrideGwei:     v.GetUint64("builder-api-value-override"),
+			ServeCandidates:       v.GetString("builder-api-serve-candidates"),
+			OnDemandBuild:         v.GetBool("builder-api-on-demand-build"),
 		},
 		DepositMaxFeeGwei: v.GetUint64("deposit-max-fee"),
 		DepositAmount:     v.GetUint64("deposit-amount"),
@@ -204,6 +221,8 @@ func initConfig() error {
 			BidSubsidy:           v.GetUint64("epbs-bid-subsidy"),
 			BidValueOverride:     v.GetUint64("epbs-bid-value-override"),
 			HeadVoteThresholdPct: v.GetUint64("epbs-vote-threshold"),
+			BidCandidate:         v.GetString("epbs-bid-candidate"),
+			BidCandidateSwitch:   v.GetBool("epbs-bid-candidate-switch"),
 		},
 		Reveal: config.RevealConfig{
 			Enabled:             v.GetBool("reveal-enabled"),
@@ -213,6 +232,17 @@ func initConfig() error {
 			BroadcastValidation: v.GetString("reveal-broadcast-validation"),
 			MaxAttempts:         v.GetUint64("reveal-max-attempts"),
 			RetryIntervalMs:     v.GetInt64("reveal-retry-interval"),
+			RebindOnReorg:       v.GetBool("reveal-rebind-on-reorg"),
+		},
+		Build: config.BuildConfig{
+			CandidateParentFull:       v.GetString("build-candidate-parent-full"),
+			CandidateParentEmpty:      v.GetString("build-candidate-parent-empty"),
+			CandidateGrandparentFull:  v.GetString("build-candidate-grandparent-full"),
+			CandidateGrandparentEmpty: v.GetString("build-candidate-grandparent-empty"),
+			Parallel:                  v.GetBool("build-parallel"),
+			SpeculativeBuildTimeMs:    v.GetUint64("build-speculative-build-time"),
+			AutoWeakHeadPct:           v.GetUint64("build-auto-weak-head-pct"),
+			EnforceBidGasLimit:        v.GetBool("build-enforce-bid-gas-limit"),
 		},
 		PayloadBuildTime:            v.GetUint64("payload-build-time"),
 		SlotResultRetentionEpochs:   v.GetUint64("slot-result-retention-epochs"),
@@ -229,6 +259,17 @@ func initConfig() error {
 		return fmt.Errorf("provide only one of --builder-privkey or --builder-mnemonic, not both")
 	}
 
+	for flag, mode := range map[string]string{
+		"--build-candidate-parent-full":       cfg.Build.CandidateParentFull,
+		"--build-candidate-parent-empty":      cfg.Build.CandidateParentEmpty,
+		"--build-candidate-grandparent-full":  cfg.Build.CandidateGrandparentFull,
+		"--build-candidate-grandparent-empty": cfg.Build.CandidateGrandparentEmpty,
+	} {
+		if mode != config.NormalizedCandidateMode(mode, "") {
+			return fmt.Errorf("invalid %s %q: must be auto, always or never", flag, mode)
+		}
+	}
+
 	if cfg.Reveal.GateMode != cfg.Reveal.NormalizedGateMode() {
 		return fmt.Errorf("invalid --reveal-gate-mode %q: must be time, vote, vote_or_time or vote_and_time",
 			cfg.Reveal.GateMode)

cmd/run.go

@@ -379,6 +379,7 @@ and begins building blocks according to configuration.`,
 
 			if revealSvc != nil {
 				builderAPISrv.SetRevealService(revealSvc)
+				builderAPISrv.SetOnDemandBuilder(builderSvc)
 			}
 
 			if propPrefSvc != nil {

pkg/action_plan/frozen.go

@@ -6,6 +6,7 @@ import (
 	"github.com/ethpandaops/go-eth2-client/spec/phase0"
 	"github.com/ethpandaops/go-eth2-client/spec/version"
 
+	"github.com/ethpandaops/buildoor/pkg/chain"
 	"github.com/ethpandaops/buildoor/pkg/config"
 )
 
@@ -87,6 +88,11 @@ type ResolvedBuildSettings struct {
 	// execution payload instead of the immediate parent — a deliberate
 	// parent-payload reorg attempt (see BuildPlan.ReorgParentPayload).
 	ReorgParentPayload bool `json:"reorg_parent_payload,omitempty"`
+
+	// CandidateModes is the effective build-candidate policy for the slot:
+	// candidate key -> auto/always/never, merged from the global config and
+	// the plan's build.candidates overrides.
+	CandidateModes map[string]string `json:"candidate_modes,omitempty"`
 }
 
 // ResolvedBidSettings are the effective p2p bidding parameters for the slot.
@@ -105,6 +111,10 @@ type ResolvedBidSettings struct {
 	// IgnoreMissingPrefs bids without gossip proposer preferences.
 	IgnoreMissingPrefs bool `json:"ignore_missing_prefs,omitempty"`
 
+	// BidCandidate is the effective bid candidate selection for the slot:
+	// auto, all, or a specific candidate key.
+	BidCandidate string `json:"bid_candidate,omitempty"`
+
 	// Forced marks that the plan activated bidding although the module is
 	// globally disabled.
 	Forced bool `json:"forced,omitempty"`
@@ -121,6 +131,10 @@ type ResolvedBuilderAPISettings struct {
 
 	DelayMs int64 `json:"delay_ms,omitempty"`
 
+	// ServeCandidates is the effective serve-candidates policy for the slot
+	// (all, canonical_only, or a comma-separated candidate key list).
+	ServeCandidates string `json:"serve_candidates,omitempty"`
+
 	// Forced marks that the plan activated serving although the module is
 	// globally disabled.
 	Forced bool `json:"forced,omitempty"`
@@ -208,6 +222,8 @@ func resolveBuild(frozen *FrozenPlan, cfg *config.Config, slotsBuilt uint64) *Re
 		build.ReorgParentPayload = frozen.Plan.Build.ReorgParentPayload
 	}
 
+	build.CandidateModes = resolveCandidateModes(frozen.Plan, cfg)
+
 	// A plan that explicitly activates (mode custom) an available consumer
 	// forces the build past the schedule. A merely-inherited active consumer
 	// never forces, and a custom category whose consumer is unavailable
@@ -275,6 +291,27 @@ func resolveBuild(frozen *FrozenPlan, cfg *config.Config, slotsBuilt uint64) *Re
 	return build
 }
 
+// resolveCandidateModes merges the global build-candidate policy with the
+// plan's per-slot overrides into the complete effective mode map.
+func resolveCandidateModes(plan *SlotPlan, cfg *config.Config) map[string]string {
+	modes := make(map[string]string, len(chain.AllCandidateKeys))
+
+	for _, key := range chain.AllCandidateKeys {
+		modes[string(key)] = cfg.Build.CandidateMode(string(key))
+	}
+
+	if plan != nil && plan.Build != nil {
+		for key, mode := range plan.Build.Candidates {
+			if normalized := config.NormalizedCandidateMode(mode, ""); normalized != "" &&
+				chain.IsValidCandidateKey(key) {
+				modes[key] = normalized
+			}
+		}
+	}
+
+	return modes
+}
+
 // planForcesConsumer reports whether the plan explicitly activates (mode
 // custom) a consumer that is actually available for the slot.
 func planForcesConsumer(frozen *FrozenPlan) bool {
@@ -342,6 +379,7 @@ func resolveBid(plan *SlotPlan, cfg *config.Config, fork version.DataVersion) *R
 		MinGwei:      cfg.EPBS.BidMinAmount,
 		IncreaseGwei: cfg.EPBS.BidIncrease,
 		SubsidyGwei:  cfg.EPBS.BidSubsidy,
+		BidCandidate: cfg.EPBS.BidCandidate,
 		Forced:       forced,
 	}
 
@@ -364,6 +402,10 @@ func resolveBid(plan *SlotPlan, cfg *config.Config, fork version.DataVersion) *R
 		}
 
 		resolved.IgnoreMissingPrefs = bid.IgnoreMissingPrefs
+
+		if bid.BidCandidate != nil {
+			resolved.BidCandidate = *bid.BidCandidate
+		}
 	}
 
 	return resolved
@@ -389,8 +431,9 @@ func resolveBuilderAPI(plan *SlotPlan, cfg *config.Config) *ResolvedBuilderAPISe
 	}
 
 	resolved := &ResolvedBuilderAPISettings{
-		SubsidyGwei: cfg.BuilderAPI.BlockValueSubsidyGwei,
-		Forced:      forced,
+		SubsidyGwei:     cfg.BuilderAPI.BlockValueSubsidyGwei,
+		ServeCandidates: cfg.BuilderAPI.ServeCandidates,
+		Forced:          forced,
 	}
 
 	if cfg.BuilderAPI.ValueOverrideGwei > 0 {
@@ -406,6 +449,10 @@ func resolveBuilderAPI(plan *SlotPlan, cfg *config.Config) *ResolvedBuilderAPISe
 		if api.TotalValueOverrideGwei != nil {
 			resolved.TotalValueGwei = cloneScalar(api.TotalValueOverrideGwei)
 		}
+
+		if api.ServeCandidates != nil {
+			resolved.ServeCandidates = *api.ServeCandidates
+		}
 	}
 
 	return resolved

pkg/action_plan/service_test.go

@@ -691,3 +691,58 @@ func TestResolveRevealSettings(t *testing.T) {
 		}
 	})
 }
+
+func TestFrozenCandidateSettings(t *testing.T) {
+	cfg := config.DefaultConfig()
+	cfg.EPBSEnabled = true
+	cfg.APIPort = 8080
+	cfg.BuilderAPIEnabled = true
+
+	log := logrus.New()
+	log.SetLevel(logrus.PanicLevel)
+
+	svc := NewPlanService(cfg, newStubChain(), log)
+
+	// Without a plan: the frozen snapshot carries the complete global
+	// candidate policy and the global bid/serve selections.
+	frozen := svc.Freeze(2000)
+	require.NotNil(t, frozen.Build.CandidateModes)
+	assert.Equal(t, config.CandidateModeAlways, frozen.Build.CandidateModes["parent_full"])
+	assert.Equal(t, config.CandidateModeAuto, frozen.Build.CandidateModes["parent_empty"])
+	assert.Equal(t, config.CandidateModeNever, frozen.Build.CandidateModes["grandparent_empty"])
+	require.NotNil(t, frozen.Bid)
+	assert.Equal(t, "auto", frozen.Bid.BidCandidate)
+	require.NotNil(t, frozen.BuilderAPI)
+	assert.Equal(t, "all", frozen.BuilderAPI.ServeCandidates)
+
+	// Plan overrides merge into the frozen snapshot.
+	_, err := svc.ApplyUpdates([]*PlanUpdate{{
+		Slots: []uint64{2100},
+		Build: json.RawMessage(`{"candidates":{"grandparent_full":"always","parent_empty":"never"}}`),
+		Bid:   json.RawMessage(`{"mode":"custom","bid_candidate":"parent_empty"}`),
+		BuilderAPI: json.RawMessage(
+			`{"mode":"custom","serve_candidates":"canonical_only"}`),
+	}}, "test")
+	require.NoError(t, err)
+
+	frozen = svc.Freeze(2100)
+	assert.Equal(t, config.CandidateModeAlways, frozen.Build.CandidateModes["grandparent_full"])
+	assert.Equal(t, config.CandidateModeNever, frozen.Build.CandidateModes["parent_empty"])
+	assert.Equal(t, config.CandidateModeAlways, frozen.Build.CandidateModes["parent_full"],
+		"unoverridden keys inherit the global policy")
+	assert.Equal(t, "parent_empty", frozen.Bid.BidCandidate)
+	assert.Equal(t, "canonical_only", frozen.BuilderAPI.ServeCandidates)
+
+	// Invalid candidate values are rejected.
+	_, err = svc.ApplyUpdates([]*PlanUpdate{{
+		Slots: []uint64{2200},
+		Build: json.RawMessage(`{"candidates":{"parent_full":"sometimes"}}`),
+	}}, "test")
+	require.Error(t, err)
+
+	_, err = svc.ApplyUpdates([]*PlanUpdate{{
+		Slots: []uint64{2200},
+		Bid:   json.RawMessage(`{"mode":"custom","bid_candidate":"bogus"}`),
+	}}, "test")
+	require.Error(t, err)
+}