feat(speculation): rank bestfirst on evidence Scorer (#684)

## Summary

### Why?

bestfirst already ranks on one probability per dependency. Wiring a
sibling Predictor would reintroduce the factory the RFC dropped. YAML
also had to show evidence as the outer scorer, not factors hanging off
heuristic. A factors map or nested base under heuristic/composite used
to load and then be ignored, so a misplaced pathPassed looked configured
while ranking stayed at 1.

### What?

Pass the speculate path-set snapshot into `Generate` and into `Score`.
Default `scorer.type` is `evidence` wrapping a nested `base`. Named
`factors` overlay; a present `base` replaces wholesale. Top-level `type:
heuristic` is rejected. Content scorers reject `factors` and a nested
`base` the same way. Drop the `predictor:` profile key. `bestfirst`
depends only on `scorer.Scorer`. The land-stage YAML key is `landing`.

## Test Plan

- ✅ `./tool/bazel test
//service/submitqueue/orchestrator/server:go_default_test
//submitqueue/extension/speculation/generator/bestfirst:go_default_test
//submitqueue/extension/speculation/speculator/standard:go_default_test`
- ✅ `go test ./service/submitqueue/orchestrator/server/ -run
TestLoadProfilesConfig_RejectsBadScorers`

## Stack

- #684 ⬅️
- #686

Author: behinddwalls

service/submitqueue/orchestrator/server/BUILD.bazel

@@ -55,6 +55,7 @@ go_library(
         "//submitqueue/extension/speculation/generator/bestfirst:go_default_library",
         "//submitqueue/extension/speculation/scorer:go_default_library",
         "//submitqueue/extension/speculation/scorer/composite:go_default_library",
+        "//submitqueue/extension/speculation/scorer/evidence:go_default_library",
         "//submitqueue/extension/speculation/scorer/fake:go_default_library",
         "//submitqueue/extension/speculation/scorer/heuristic:go_default_library",
         "//submitqueue/extension/speculation/speculator:go_default_library",

service/submitqueue/orchestrator/server/config.go

@@ -16,6 +16,8 @@ package main
 
 import (
 	"fmt"
+	"maps"
+	"math"
 	"os"
 	"time"
 
@@ -53,6 +55,7 @@ const (
 
 // Scorer types selectable from configuration.
 const (
+	scorerTypeEvidence  = "evidence"
 	scorerTypeHeuristic = "heuristic"
 	scorerTypeComposite = "composite"
 )
@@ -67,6 +70,19 @@ const (
 // Ways a composite scorer combines its components.
 const combineAvg = "avg"
 
+// Evidence an evidence scorer prices, as named in configuration. The set is
+// closed: a factor under any other name would be applied to nothing and never
+// noticed.
+const (
+	factorPathPassed = "pathPassed"
+	factorPathFailed = "pathFailed"
+	factorLanding    = "landing"
+	factorCancelling = "cancelling"
+)
+
+// neutralFactor leaves the scorer's price untouched.
+const neutralFactor = 1.0
+
 // defaultBuildBudget is how many builds a queue may have occupying CI at once
 // when it states no budget of its own. Four is enough for speculation to be
 // visible — a queue that can only build one path never speculates — while
@@ -208,11 +224,19 @@ type analyzerConfig struct {
 	FailAlways bool `yaml:"failAlways"`
 }
 
-// scorerConfig selects how a queue ranks candidate speculation paths. There is
-// no scoring stage: the scorer feeds the queue's speculator, which is composed
-// from it rather than configured separately.
+// scorerConfig selects how a queue ranks candidate speculation paths. The
+// ranking scorer is evidence wrapping a nested content base. Heuristic and
+// composite belong on base (and on composite components), not at the top level.
 type scorerConfig struct {
 	Type string `yaml:"type"`
+	// Factors revise the base price, one per piece of evidence (evidence only).
+	// An omitted key keeps the inherited value, or 1 if neither defaults nor
+	// the queue named it.
+	Factors map[string]float64 `yaml:"factors"`
+	// Base is the content scorer evidence revises (evidence only). An omitted
+	// base on defaults is the default heuristic; a present base on a queue
+	// replaces the default base wholesale.
+	Base *scorerConfig `yaml:"base"`
 	// Buckets map a batch's total lines changed onto a score (heuristic only).
 	Buckets []bucketConfig `yaml:"buckets"`
 	// Components are the scorers a composite combines, keyed by name.
@@ -291,7 +315,7 @@ func (c *profilesConfig) normalizeAndValidate() error {
 			}
 		}
 		if q.Scorer != nil {
-			if err := q.Scorer.normalizeAndValidate(where); err != nil {
+			if err := q.Scorer.normalizeOverlay(where); err != nil {
 				return err
 			}
 		}
@@ -353,14 +377,36 @@ func (c profilesConfig) resolve(q namedQueueProfileConfig) queueProfileConfig {
 		profile.Analyzer = *q.Analyzer
 	}
 	if q.Scorer != nil {
-		profile.Scorer = *q.Scorer
+		profile.Scorer = overlayScorer(profile.Scorer, *q.Scorer)
 	}
 	if q.Speculator != nil {
 		profile.Speculator = *q.Speculator
 	}
 	return profile
 }
 
+// overlayScorer keeps default factors the queue did not name. A present base
+// replaces the default base wholesale. Type stays evidence unless the override
+// names one, which must still be evidence.
+func overlayScorer(base, override scorerConfig) scorerConfig {
+	if override.Type != "" {
+		base.Type = override.Type
+	}
+	if len(override.Factors) > 0 {
+		merged := maps.Clone(base.Factors)
+		if merged == nil {
+			merged = make(map[string]float64, len(override.Factors))
+		}
+		maps.Copy(merged, override.Factors)
+		base.Factors = merged
+	}
+	if override.Base != nil {
+		copied := *override.Base
+		base.Base = &copied
+	}
+	return base
+}
+
 func (p *queueProfileConfig) normalizeAndValidate(where string) error {
 	if err := p.ChangeProvider.normalizeAndValidate(where); err != nil {
 		return err
@@ -374,7 +420,10 @@ func (p *queueProfileConfig) normalizeAndValidate(where string) error {
 	if err := p.Scorer.normalizeAndValidate(where); err != nil {
 		return err
 	}
-	return p.Speculator.normalizeAndValidate(where)
+	if err := p.Speculator.normalizeAndValidate(where); err != nil {
+		return err
+	}
+	return nil
 }
 
 func (c *changeProviderConfig) normalizeAndValidate(where string) error {
@@ -514,10 +563,62 @@ func (a *analyzerConfig) normalizeAndValidate(where string) error {
 	}
 }
 
-// normalizeAndValidate applies defaults and rejects a scorer that could not be
-// built. An empty block is a flat heuristic: every batch scores the same, which
-// is the neutral choice for a queue with no opinion about ordering.
+// normalizeAndValidate applies defaults and rejects a ranking scorer that
+// could not be built. An empty block is evidence wrapping the default
+// heuristic, with every factor neutral.
 func (s *scorerConfig) normalizeAndValidate(where string) error {
+	return s.normalizeRanking(where, true)
+}
+
+// normalizeOverlay validates a queue's scorer override without inventing a
+// base: omitted base means inherit the default base.
+func (s *scorerConfig) normalizeOverlay(where string) error {
+	return s.normalizeRanking(where, false)
+}
+
+func (s *scorerConfig) normalizeRanking(where string, fillBase bool) error {
+	if s.Type == "" {
+		s.Type = scorerTypeEvidence
+	}
+	if s.Type != scorerTypeEvidence {
+		return fmt.Errorf("%s: scorer type %q belongs under base, not at the ranking layer", where, s.Type)
+	}
+	if len(s.Buckets) > 0 || len(s.Components) > 0 || s.Combine != "" {
+		return fmt.Errorf("%s: buckets, components, and combine belong under base", where)
+	}
+	if err := validateFactors(where, s.Factors); err != nil {
+		return err
+	}
+	if s.Base != nil {
+		return s.Base.normalizeContent(where + " base")
+	}
+	if fillBase {
+		s.Base = &scorerConfig{}
+		return s.Base.normalizeContent(where + " base")
+	}
+	return nil
+}
+
+func validateFactors(where string, factors map[string]float64) error {
+	for name, factor := range factors {
+		switch name {
+		case factorPathPassed, factorPathFailed, factorLanding, factorCancelling:
+		default:
+			return fmt.Errorf("%s: unknown scorer factor %q", where, name)
+		}
+		// Zero would permanently pin matching batches to 0; negatives cannot
+		// represent either direction in the factor contract.
+		if !(factor > 0) || math.IsInf(factor, 0) {
+			return fmt.Errorf("%s: scorer factor %q is %v, must be finite and positive", where, name, factor)
+		}
+	}
+	return nil
+}
+
+func (s *scorerConfig) normalizeContent(where string) error {
+	if len(s.Factors) > 0 || s.Base != nil {
+		return fmt.Errorf("%s: factors and base belong on the ranking scorer, not under base", where)
+	}
 	if s.Type == "" {
 		s.Type = scorerTypeHeuristic
 	}
@@ -539,7 +640,7 @@ func (s *scorerConfig) normalizeAndValidate(where string) error {
 			return fmt.Errorf("%s: composite scorer needs at least one component", where)
 		}
 		for name, component := range s.Components {
-			if err := component.normalizeAndValidate(fmt.Sprintf("%s component %q", where, name)); err != nil {
+			if err := component.normalizeContent(fmt.Sprintf("%s component %q", where, name)); err != nil {
 				return err
 			}
 			s.Components[name] = component

service/submitqueue/orchestrator/server/config_test.go

@@ -350,20 +350,24 @@ func TestDefaultProfilesConfig_KeepsPerQueueScorers(t *testing.T) {
 		byName[q.Name] = q
 	}
 
-	assert.Equal(t, scorerTypeHeuristic, cfg.Defaults.Scorer.Type)
-	assert.Len(t, cfg.Defaults.Scorer.Buckets, 1, "the baseline scores every batch alike")
+	assert.Equal(t, scorerTypeEvidence, cfg.Defaults.Scorer.Type)
+	require.NotNil(t, cfg.Defaults.Scorer.Base)
+	assert.Equal(t, scorerTypeHeuristic, cfg.Defaults.Scorer.Base.Type)
+	assert.Len(t, cfg.Defaults.Scorer.Base.Buckets, 1, "the baseline scores every batch alike")
 
 	bucketed, ok := byName["test-queue"]
 	require.True(t, ok)
 	require.NotNil(t, bucketed.Scorer)
-	assert.Equal(t, scorerTypeHeuristic, bucketed.Scorer.Type)
-	assert.Len(t, bucketed.Scorer.Buckets, 4, "smaller batches must rank ahead of larger ones")
+	require.NotNil(t, bucketed.Scorer.Base)
+	assert.Equal(t, scorerTypeHeuristic, bucketed.Scorer.Base.Type)
+	assert.Len(t, bucketed.Scorer.Base.Buckets, 4, "smaller batches must rank ahead of larger ones")
 
 	comp, ok := byName["e2e-test-queue"]
 	require.True(t, ok)
 	require.NotNil(t, comp.Scorer)
-	assert.Equal(t, scorerTypeComposite, comp.Scorer.Type)
-	assert.ElementsMatch(t, []string{"size", "flat"}, keysOf(comp.Scorer.Components))
+	require.NotNil(t, comp.Scorer.Base)
+	assert.Equal(t, scorerTypeComposite, comp.Scorer.Base.Type)
+	assert.ElementsMatch(t, []string{"size", "flat"}, keysOf(comp.Scorer.Base.Components))
 }
 
 func keysOf(m map[string]scorerConfig) []string {
@@ -660,10 +664,19 @@ func TestLoadProfilesConfig_RejectsBadScorers(t *testing.T) {
 		contents string
 	}{
 		{name: "unknown scorer type", contents: "defaults:\n  scorer: {type: vibes}\n"},
-		{name: "composite with no components", contents: "defaults:\n  scorer: {type: composite}\n"},
-		{name: "unknown combine", contents: "defaults:\n  scorer:\n    type: composite\n    combine: median\n    components: {a: {type: heuristic}}\n"},
-		{name: "score out of range", contents: "defaults:\n  scorer:\n    type: heuristic\n    buckets: [{min: 0, max: 10, score: 2.0}]\n"},
-		{name: "inverted bucket", contents: "defaults:\n  scorer:\n    type: heuristic\n    buckets: [{min: 10, max: 1, score: 0.5}]\n"},
+		{name: "top-level heuristic", contents: "defaults:\n  scorer: {type: heuristic}\n"},
+		{name: "top-level buckets without type", contents: "defaults:\n  scorer:\n    buckets: [{min: 0, max: 10, score: 0.9}]\n"},
+		{name: "buckets next to evidence", contents: "defaults:\n  scorer:\n    type: evidence\n    buckets: [{min: 0, max: 10, score: 0.9}]\n"},
+		{name: "top-level combine", contents: "defaults:\n  scorer:\n    combine: avg\n"},
+		{name: "queue overlay buckets without type", contents: "defaults: {}\nqueues:\n  - name: q\n    scorer:\n      buckets: [{min: 0, max: 10, score: 0.9}]\n"},
+		{name: "composite with no components", contents: "defaults:\n  scorer:\n    base: {type: composite}\n"},
+		{name: "unknown combine", contents: "defaults:\n  scorer:\n    base:\n      type: composite\n      combine: median\n      components: {a: {type: heuristic}}\n"},
+		{name: "score out of range", contents: "defaults:\n  scorer:\n    base:\n      type: heuristic\n      buckets: [{min: 0, max: 10, score: 2.0}]\n"},
+		{name: "inverted bucket", contents: "defaults:\n  scorer:\n    base:\n      type: heuristic\n      buckets: [{min: 10, max: 1, score: 0.5}]\n"},
+		{name: "factors under heuristic base", contents: "defaults:\n  scorer:\n    base:\n      type: heuristic\n      factors: {pathPassed: 10}\n"},
+		{name: "nested base under heuristic", contents: "defaults:\n  scorer:\n    base:\n      type: heuristic\n      base: {type: heuristic}\n"},
+		{name: "factors on a composite component", contents: "defaults:\n  scorer:\n    base:\n      type: composite\n      components:\n        a:\n          type: heuristic\n          factors: {pathPassed: 10}\n"},
+		{name: "factors under queue overlay base", contents: "defaults: {}\nqueues:\n  - name: q\n    scorer:\n      base:\n        type: heuristic\n        factors: {pathPassed: 10}\n"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -672,3 +685,91 @@ func TestLoadProfilesConfig_RejectsBadScorers(t *testing.T) {
 		})
 	}
 }
+
+func TestLoadProfilesConfig_RejectsBadScorerFactors(t *testing.T) {
+	tests := []struct {
+		name     string
+		contents string
+	}{
+		{name: "unknown factor", contents: "defaults:\n  scorer:\n    factors: {pathPased: 2}\n"},
+		{name: "zero factor", contents: "defaults:\n  scorer:\n    factors: {landing: 0}\n"},
+		{name: "negative factor", contents: "defaults:\n  scorer:\n    factors: {pathFailed: -1}\n"},
+		{name: "infinite factor", contents: "defaults:\n  scorer:\n    factors: {pathPassed: .inf}\n"},
+		{name: "bad factor on a queue override", contents: "defaults: {}\nqueues:\n  - name: q\n    scorer:\n      factors: {landing: 0}\n"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := loadProfilesConfig(writeProfiles(t, tt.contents))
+			require.Error(t, err)
+		})
+	}
+}
+
+// An omitted factors map leaves the queue ranking on its base price
+// alone, which is what every queue does until someone states a factor.
+func TestLoadProfilesConfig_DefaultsTheScorerToEvidence(t *testing.T) {
+	cfg, err := loadProfilesConfig(writeProfiles(t, "defaults: {}\nqueues:\n  - name: q\n"))
+	require.NoError(t, err)
+
+	assert.Equal(t, scorerTypeEvidence, cfg.Defaults.Scorer.Type)
+	require.NotNil(t, cfg.Defaults.Scorer.Base)
+	assert.Equal(t, scorerTypeHeuristic, cfg.Defaults.Scorer.Base.Type)
+
+	factors := factorsFrom(cfg.resolve(cfg.Queues[0]).Scorer)
+	assert.Equal(t, neutralFactor, factors.PathPassed)
+	assert.Equal(t, neutralFactor, factors.PathFailed)
+	assert.Equal(t, neutralFactor, factors.Landing)
+	assert.Equal(t, neutralFactor, factors.Cancelling)
+}
+
+func TestLoadProfilesConfig_ReadsScorerFactors(t *testing.T) {
+	cfg, err := loadProfilesConfig(writeProfiles(t,
+		"defaults:\n  scorer:\n    factors: {pathPassed: 10, pathFailed: 0.3, landing: 12, cancelling: 0.1}\n"))
+	require.NoError(t, err)
+
+	factors := factorsFrom(cfg.Defaults.Scorer)
+	assert.Equal(t, 10.0, factors.PathPassed)
+	assert.Equal(t, 0.3, factors.PathFailed)
+	assert.Equal(t, 12.0, factors.Landing)
+	assert.Equal(t, 0.1, factors.Cancelling)
+}
+
+func TestLoadProfilesConfig_QueueScorerFactorsOverlayDefaults(t *testing.T) {
+	cfg, err := loadProfilesConfig(writeProfiles(t,
+		"defaults:\n  scorer:\n    factors: {pathPassed: 10, pathFailed: 0.3, landing: 12, cancelling: 0.1}\nqueues:\n  - name: q\n    scorer:\n      factors: {pathPassed: 4}\n"))
+	require.NoError(t, err)
+
+	factors := factorsFrom(cfg.resolve(cfg.Queues[0]).Scorer)
+	assert.Equal(t, 4.0, factors.PathPassed)
+	assert.Equal(t, 0.3, factors.PathFailed)
+	assert.Equal(t, 12.0, factors.Landing)
+	assert.Equal(t, 0.1, factors.Cancelling)
+
+	defaults := factorsFrom(cfg.Defaults.Scorer)
+	assert.Equal(t, 10.0, defaults.PathPassed)
+}
+
+func TestLoadProfilesConfig_QueueScorerBaseReplacesDefaultBase(t *testing.T) {
+	cfg, err := loadProfilesConfig(writeProfiles(t, ""+
+		"defaults:\n"+
+		"  scorer:\n"+
+		"    factors: {pathPassed: 10}\n"+
+		"    base:\n"+
+		"      type: heuristic\n"+
+		"      buckets: [{min: 0, max: 1000, score: 0.4}]\n"+
+		"queues:\n"+
+		"  - name: q\n"+
+		"    scorer:\n"+
+		"      base:\n"+
+		"        type: heuristic\n"+
+		"        buckets: [{min: 0, max: 1000, score: 0.9}]\n"))
+	require.NoError(t, err)
+
+	resolved := cfg.resolve(cfg.Queues[0]).Scorer
+	require.NotNil(t, resolved.Base)
+	assert.Equal(t, 0.9, resolved.Base.Buckets[0].Score)
+	assert.Equal(t, 10.0, factorsFrom(resolved).PathPassed)
+
+	require.NotNil(t, cfg.Defaults.Scorer.Base)
+	assert.Equal(t, 0.4, cfg.Defaults.Scorer.Base.Buckets[0].Score)
+}

service/submitqueue/orchestrator/server/main.go

@@ -389,24 +389,28 @@ func defaultProfilesConfig() profilesConfig {
 			// Bucketed scoring: smaller batches are likelier to land, so they
 			// rank ahead of larger ones. Conflicts stay conservative.
 			{Name: "test-queue", Scorer: &scorerConfig{
-				Type: scorerTypeHeuristic,
-				Buckets: []bucketConfig{
-					{Min: 0, Max: 1, Score: 0.95},
-					{Min: 2, Max: 5, Score: 0.80},
-					{Min: 6, Max: 20, Score: 0.60},
-					{Min: 21, Max: maxBucket, Score: 0.40},
+				Base: &scorerConfig{
+					Type: scorerTypeHeuristic,
+					Buckets: []bucketConfig{
+						{Min: 0, Max: 1, Score: 0.95},
+						{Min: 2, Max: 5, Score: 0.80},
+						{Min: 6, Max: 20, Score: 0.60},
+						{Min: 21, Max: maxBucket, Score: 0.40},
+					},
 				},
 			}},
 			// Maximum parallelism: nothing ever conflicts. Scored by a
 			// composite, which exercises the combining path.
 			{Name: "e2e-test-queue",
 				Analyzer: &analyzerConfig{Type: analyzerTypeNone},
 				Scorer: &scorerConfig{
-					Type:    scorerTypeComposite,
-					Combine: combineAvg,
-					Components: map[string]scorerConfig{
-						"size": {Type: scorerTypeHeuristic, Buckets: []bucketConfig{{Min: 0, Max: maxBucket, Score: 0.8}}},
-						"flat": {Type: scorerTypeHeuristic, Buckets: []bucketConfig{{Min: 0, Max: maxBucket, Score: 0.6}}},
+					Base: &scorerConfig{
+						Type:    scorerTypeComposite,
+						Combine: combineAvg,
+						Components: map[string]scorerConfig{
+							"size": {Type: scorerTypeHeuristic, Buckets: []bucketConfig{{Min: 0, Max: maxBucket, Score: 0.8}}},
+							"flat": {Type: scorerTypeHeuristic, Buckets: []bucketConfig{{Min: 0, Max: maxBucket, Score: 0.6}}},
+						},
 					},
 				},
 			},