feat(entity,storage): rework speculation tree model and store

Reshape the speculation tree data model around the Base/Head path that the build stage actually consumes, and align its store with the speculation design.

entity: SpeculationPath{Base, Head} becomes the unit; SpeculationPathInfo carries the path plus its Score, controller-owned Status (candidate/selected/building/passed/failed/cancelled), and BuildID. Score is scorer-computed and controller-persisted dynamic state — recomputed on every respeculate, not set at enumeration. Adds SpeculationPathAction and SpeculationPathDecision for the selector seam. SpeculationTree.Speculations becomes Paths. The Build entity uses the shared SpeculationPath (Head = the batch under verification) and drops its own Score, which was never populated or read; the prioritizer seam re-adds a build priority when it lands.

storage: SpeculationTreeStore.UpdateSpeculations(batchID, []SpeculationInfo) becomes Update(ctx, SpeculationTree) — symmetric with Create, keyed by tree.BatchID. The MySQL column speculations is renamed paths to match the entity field. MySQL impl, mock, and storage integration coverage (create/get round-trip, duplicate, whole-tree overwrite, not-found) added/updated.

Author: behinddwalls

submitqueue/entity/build.go

@@ -53,12 +53,6 @@ func (s BuildStatus) IsTerminal() bool {
 	return s == BuildStatusSucceeded || s == BuildStatusFailed || s == BuildStatusCancelled
 }
 
-// SpeculationPathInfo represents the base and head commits of a speculation path used in a build.
-type SpeculationPathInfo struct {
-	// Base is a list of batchIDs(in order) that form the base of this speculation path.
-	Base []string
-}
-
 // Build represents a build scheduled for a batch along a specific speculation path.
 // All fields except the Status are immutable after creation.
 type Build struct {
@@ -69,10 +63,9 @@ type Build struct {
 	BatchID string
 	// SpeculationPath is the speculation path that represents this build. For
 	// a given batch this path is crafted from the graph that is generated from the
-	// dependencies of this batch.
-	SpeculationPath SpeculationPathInfo
-	// Score represents the build prediction score for this speculation path.
-	Score float32
+	// dependencies of this batch. Its Head is the batch being verified (equal to
+	// BatchID) and its Base is the assumed-good prefix of predecessor batches.
+	SpeculationPath SpeculationPath
 	// Status represents the state of the build lifecycle this build is in.
 	Status BuildStatus
 }

submitqueue/entity/build_test.go

@@ -70,10 +70,10 @@ func TestBuild_ToBytes(t *testing.T) {
 	build := Build{
 		ID:      "build-1",
 		BatchID: "batch-1",
-		SpeculationPath: SpeculationPathInfo{
+		SpeculationPath: SpeculationPath{
 			Base: []string{"batch-0", "batch-prev"},
+			Head: "batch-1",
 		},
-		Score:  0.85,
 		Status: BuildStatusAccepted,
 	}
 
@@ -92,10 +92,10 @@ func TestBuildFromBytes(t *testing.T) {
 	original := Build{
 		ID:      "build-42",
 		BatchID: "batch-7",
-		SpeculationPath: SpeculationPathInfo{
+		SpeculationPath: SpeculationPath{
 			Base: []string{"batch-5", "batch-6"},
+			Head: "batch-7",
 		},
-		Score:  0.92,
 		Status: BuildStatusAccepted,
 	}
 
@@ -111,7 +111,6 @@ func TestBuildFromBytes(t *testing.T) {
 	assert.Equal(t, original.ID, deserialized.ID)
 	assert.Equal(t, original.BatchID, deserialized.BatchID)
 	assert.Equal(t, original.SpeculationPath.Base, deserialized.SpeculationPath.Base)
-	assert.Equal(t, original.Score, deserialized.Score)
 	assert.Equal(t, original.Status, deserialized.Status)
 }
 
@@ -132,7 +131,6 @@ func TestBuildFromBytes_EmptyData(t *testing.T) {
 	assert.Empty(t, build.ID)
 	assert.Empty(t, build.BatchID)
 	assert.Equal(t, BuildStatusUnknown, build.Status)
-	assert.Equal(t, float32(0), build.Score)
 }
 
 func TestBuild_SerializationRoundTrip(t *testing.T) {
@@ -145,10 +143,10 @@ func TestBuild_SerializationRoundTrip(t *testing.T) {
 			build: Build{
 				ID:      "build-100",
 				BatchID: "batch-50",
-				SpeculationPath: SpeculationPathInfo{
+				SpeculationPath: SpeculationPath{
 					Base: []string{"batch-48", "batch-49"},
+					Head: "batch-50",
 				},
-				Score:  0.75,
 				Status: BuildStatusAccepted,
 			},
 		},
@@ -157,19 +155,18 @@ func TestBuild_SerializationRoundTrip(t *testing.T) {
 			build: Build{
 				ID:      "build-200",
 				BatchID: "batch-60",
-				Score:   1.0,
 				Status:  BuildStatusSucceeded,
 			},
 		},
 		{
-			name: "failed build with zero score",
+			name: "failed build",
 			build: Build{
 				ID:      "build-300",
 				BatchID: "batch-70",
-				SpeculationPath: SpeculationPathInfo{
+				SpeculationPath: SpeculationPath{
 					Base: []string{"batch-65"},
+					Head: "batch-70",
 				},
-				Score:  0,
 				Status: BuildStatusFailed,
 			},
 		},

submitqueue/entity/speculation_tree.go

@@ -14,38 +14,120 @@
 
 package entity
 
-// SpeculationPathAction defines the possible actions for a speculation path.
+// SpeculationPath is a single speculation path: an assumed-good prefix of
+// predecessor batches (Base) on top of which the batch under verification
+// (Head) is built and validated.
+//
+// This is the unit the build stage consumes: Base maps to the build runner's
+// base changes (an assumed-good prefix to apply) and Head maps to the changes
+// being validated.
+type SpeculationPath struct {
+	// Base is the ordered list of predecessor batch IDs assumed to have passed.
+	// Empty means the path builds the head batch directly on the target branch.
+	Base []string
+	// Head is the batch ID being verified by this path.
+	Head string
+}
+
+// SpeculationPathStatus is the observed lifecycle state of a speculation path.
+// It is written only by the orchestrator's speculate controller (into the
+// speculation tree store) and read by the path selector as input; enumerators
+// and selectors never write it.
+type SpeculationPathStatus string
+
+const (
+	// SpeculationPathStatusUnknown is the unreachable zero value, set by default
+	// on init. A persisted path always carries a real status (candidate onward),
+	// so this should never be seen in the store.
+	SpeculationPathStatusUnknown SpeculationPathStatus = ""
+	// SpeculationPathStatusCandidate is a freshly enumerated path the controller
+	// has persisted but not yet sent to build.
+	SpeculationPathStatusCandidate SpeculationPathStatus = "candidate"
+	// SpeculationPathStatusSelected is a path the controller has sent to the build
+	// controller (in response to a selector Build action) but for which no build
+	// signal has arrived yet — the build system may not have started it
+	// (resource-gated), so whether it is actually building is not yet known.
+	SpeculationPathStatusSelected SpeculationPathStatus = "selected"
+	// SpeculationPathStatusBuilding is a path a build signal has confirmed is in
+	// flight; its BuildID is known.
+	SpeculationPathStatusBuilding SpeculationPathStatus = "building"
+	// SpeculationPathStatusPassed is a path whose build succeeded.
+	SpeculationPathStatusPassed SpeculationPathStatus = "passed"
+	// SpeculationPathStatusFailed is a path whose build failed.
+	SpeculationPathStatusFailed SpeculationPathStatus = "failed"
+	// SpeculationPathStatusCancelled is a path that is no longer pursued — its
+	// base was invalidated, its build was cancelled, or the selector dropped it.
+	SpeculationPathStatusCancelled SpeculationPathStatus = "cancelled"
+)
+
+// SpeculationPathAction is the action a path selector asks the controller to
+// take for a path. It is the selector's only output: ephemeral (recomputed
+// every time the selector runs) and never persisted. The controller enacts it
+// and records the resulting SpeculationPathStatus.
 type SpeculationPathAction string
 
 const (
-	// SpeculationPathActionUnknown is the default zero value for SpeculationPathAction.
+	// SpeculationPathActionUnknown is the unreachable zero value. A real decision
+	// always carries Build or Cancel; the selector expresses "leave this path
+	// as-is" by omitting it from its decisions, not by returning this.
 	SpeculationPathActionUnknown SpeculationPathAction = ""
-	// TODO: Add comprehensive list of actions
+	// SpeculationPathActionBuild asks the controller to send this path to the
+	// build controller (which triggers a build subject to resources). The path moves
+	// to Selected on send, then Building once a build signal confirms it.
+	SpeculationPathActionBuild SpeculationPathAction = "build"
+	// SpeculationPathActionCancel asks the controller to drop this path and
+	// cancel any build in flight for it.
+	SpeculationPathActionCancel SpeculationPathAction = "cancel"
 )
 
-// SpeculationInfo represents metadata about a single speculation path, including the path through the dependency graph, its current state, and the predicted build score.
-type SpeculationInfo struct {
-	// Path represents the speculation path; which is an ordered list of batches.
-	Path []string
-	// Action is a state that this path is in.
-	Action SpeculationPathAction
-	// Score is score for this speculation path.
+// SpeculationPathInfo is the per-path entry in a speculation tree: a path, its
+// latest predicted-success score, its controller-owned status, and a link to
+// the build dispatched for it (if any).
+type SpeculationPathInfo struct {
+	// Path is the Base/Head split this entry covers.
+	Path SpeculationPath
+	// Score is the path's predicted-success score. It is computed by the scorer
+	// and persisted by the controller, not set at enumeration — the enumerator
+	// produces structure only. It is dynamic: the controller re-runs the scorer
+	// on every respeculate (as dependencies land, dependency builds pass, or
+	// sibling paths fail), so the value tracks the latest state rather than a
+	// figure frozen when the path was first enumerated (~0 until the first pass).
 	Score float32
+	// Status is the observed lifecycle state of the path. Written only by the
+	// controller; read by the selector.
+	Status SpeculationPathStatus
+	// BuildID links this path to its build. Empty until a build signal confirms
+	// the build and the controller records it (Selected -> Building); the
+	// controller never knows the ID at send time.
+	BuildID string
+}
+
+// SpeculationPathDecision is a path selector's decision for a single path: the
+// action the controller should take for it. It is the selector's output and is
+// not persisted.
+type SpeculationPathDecision struct {
+	// Path identifies the speculation path the action applies to.
+	Path SpeculationPath
+	// Action is what the controller should do for the path.
+	Action SpeculationPathAction
 }
 
-// SpeculationTree represents the set of speculation paths constructed for a batch based on its dependency graph.
+// SpeculationTree is the set of candidate speculation paths for a batch, built
+// from its dependency graph.
 type SpeculationTree struct {
 	// BatchID is the batch for which this speculation tree is constructed.
 	BatchID string
-	// Speculations is a list of speculation paths for this batch based on a graph of its
-	// dependents.
+	// Paths is the candidate speculation paths for this batch, derived from a
+	// graph of its dependencies. Each entry's per-path dynamic state (Score,
+	// Status, BuildID) is documented on SpeculationPathInfo.
 	//
 	// For e.g - Consider batches - queueA/batch/1, queueA/batch/2, queueA/batch/3
-	// such that - queueA/batch/2 and queueA/batch/3 depend on queueA/batch/1
+	// such that - queueA/batch/2 and queueA/batch/3 depend on queueA/batch/1.
+	// Each dependent batch gets two paths: build alone, or build on the
+	// assumed-good predecessor. Just after enumeration every path is a candidate:
 	//
-	// Speculations for queueA/batch/1 - [{Path: []string{"queueA/batch/1"}, State: "scheduled", Score: 0.1}]
-	// Speculations for queueA/batch/2 - [{Path: []string{"queueA/batch/2"}, State: "scheduled", Score: 0.9}, {Path: []string{"queueA/batch/1", "queueA/batch/2"}, State: "scheduled", Score: 0.3}]
-	// Speculations for queueA/batch/3 - [{Path: []string{"queueA/batch/3"}, State: "scheduled", Score: 0.9}, {Path: []string{"queueA/batch/1", "queueA/batch/3"}, State: "scheduled", Score: 0.3}]
+	// Paths for queueA/batch/2 - [{Path: {Base: [], Head: "queueA/batch/2"}, Status: "candidate"}, {Path: {Base: ["queueA/batch/1"], Head: "queueA/batch/2"}, Status: "candidate"}]
+	// Paths for queueA/batch/3 - [{Path: {Base: [], Head: "queueA/batch/3"}, Status: "candidate"}, {Path: {Base: ["queueA/batch/1"], Head: "queueA/batch/3"}, Status: "candidate"}]
 	//
-	Speculations []SpeculationInfo
+	Paths []SpeculationPathInfo
 }

submitqueue/extension/storage/mock/speculation_tree_store_mock.go

@@ -48,9 +48,9 @@ func (s *buildStore) Get(ctx context.Context, id string) (ret entity.Build, retE
 	var speculationPathJSON []byte
 
 	err := s.db.QueryRowContext(ctx,
-		"SELECT id, batch_id, speculation_path, score, status FROM build WHERE id = ?",
+		"SELECT id, batch_id, speculation_path, status FROM build WHERE id = ?",
 		id,
-	).Scan(&build.ID, &build.BatchID, &speculationPathJSON, &build.Score, &build.Status)
+	).Scan(&build.ID, &build.BatchID, &speculationPathJSON, &build.Status)
 
 	if errors.Is(err, sql.ErrNoRows) {
 		return entity.Build{}, storage.WrapNotFound(err)
@@ -77,8 +77,8 @@ func (s *buildStore) Create(ctx context.Context, build entity.Build) (retErr err
 	}
 
 	_, err = s.db.ExecContext(ctx,
-		"INSERT INTO build (id, batch_id, speculation_path, score, status) VALUES (?, ?, ?, ?, ?)",
-		build.ID, build.BatchID, speculationPathJSON, build.Score, build.Status,
+		"INSERT INTO build (id, batch_id, speculation_path, status) VALUES (?, ?, ?, ?)",
+		build.ID, build.BatchID, speculationPathJSON, build.Status,
 	)
 	if err != nil {
 		var mysqlErr *mysql.MySQLError

submitqueue/extension/storage/mysql/build_store.go

@@ -2,7 +2,6 @@ CREATE TABLE IF NOT EXISTS build (
     id VARCHAR(255) NOT NULL,
     batch_id VARCHAR(255) NOT NULL,
     speculation_path JSON NOT NULL,
-    score FLOAT NOT NULL,
     status VARCHAR(64) NOT NULL,
     PRIMARY KEY (id)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

submitqueue/extension/storage/mysql/schema/build.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS speculation_tree (
     batch_id VARCHAR(255) NOT NULL,
-    speculations JSON NOT NULL,
+    paths JSON NOT NULL,
     PRIMARY KEY (batch_id)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

submitqueue/extension/storage/mysql/schema/speculation_tree.sql

@@ -45,12 +45,12 @@ func (s *speculationTreeStore) Get(ctx context.Context, batchID string) (ret ent
 	defer func() { op.Complete(retErr) }()
 
 	var st entity.SpeculationTree
-	var speculationsJSON []byte
+	var pathsJSON []byte
 
 	err := s.db.QueryRowContext(ctx,
-		"SELECT batch_id, speculations FROM speculation_tree WHERE batch_id = ?",
+		"SELECT batch_id, paths FROM speculation_tree WHERE batch_id = ?",
 		batchID,
-	).Scan(&st.BatchID, &speculationsJSON)
+	).Scan(&st.BatchID, &pathsJSON)
 
 	if errors.Is(err, sql.ErrNoRows) {
 		return entity.SpeculationTree{}, storage.WrapNotFound(err)
@@ -59,8 +59,8 @@ func (s *speculationTreeStore) Get(ctx context.Context, batchID string) (ret ent
 		return entity.SpeculationTree{}, fmt.Errorf("failed to get speculation tree entity batchID=%s from the database: %w", batchID, err)
 	}
 
-	if err := json.Unmarshal(speculationsJSON, &st.Speculations); err != nil {
-		return entity.SpeculationTree{}, fmt.Errorf("failed to unmarshal speculations for speculation tree entity batchID=%s from the database: %w", batchID, err)
+	if err := json.Unmarshal(pathsJSON, &st.Paths); err != nil {
+		return entity.SpeculationTree{}, fmt.Errorf("failed to unmarshal paths for speculation tree entity batchID=%s from the database: %w", batchID, err)
 	}
 
 	return st, nil
@@ -71,14 +71,14 @@ func (s *speculationTreeStore) Create(ctx context.Context, speculationTree entit
 	op := metrics.Begin(s.scope, "create")
 	defer func() { op.Complete(retErr) }()
 
-	speculationsJSON, err := json.Marshal(speculationTree.Speculations)
+	pathsJSON, err := json.Marshal(speculationTree.Paths)
 	if err != nil {
-		return fmt.Errorf("failed to marshal speculations batchID=%s for Create speculation tree entity: %w", speculationTree.BatchID, err)
+		return fmt.Errorf("failed to marshal paths batchID=%s for Create speculation tree entity: %w", speculationTree.BatchID, err)
 	}
 
 	_, err = s.db.ExecContext(ctx,
-		"INSERT INTO speculation_tree (batch_id, speculations) VALUES (?, ?)",
-		speculationTree.BatchID, speculationsJSON,
+		"INSERT INTO speculation_tree (batch_id, paths) VALUES (?, ?)",
+		speculationTree.BatchID, pathsJSON,
 	)
 	if err != nil {
 		var mysqlErr *mysql.MySQLError
@@ -91,31 +91,32 @@ func (s *speculationTreeStore) Create(ctx context.Context, speculationTree entit
 	return nil
 }
 
-// UpdateSpeculations updates the speculations of a speculation tree. Returns ErrNotFound if the speculation tree is not found.
-func (s *speculationTreeStore) UpdateSpeculations(ctx context.Context, batchID string, speculations []entity.SpeculationInfo) (retErr error) {
-	op := metrics.Begin(s.scope, "update_speculations")
+// Update overwrites the paths of an existing speculation tree, identified by
+// speculationTree.BatchID. Returns ErrNotFound if the speculation tree is not found.
+func (s *speculationTreeStore) Update(ctx context.Context, speculationTree entity.SpeculationTree) (retErr error) {
+	op := metrics.Begin(s.scope, "update")
 	defer func() { op.Complete(retErr) }()
 
-	speculationsJSON, err := json.Marshal(speculations)
+	pathsJSON, err := json.Marshal(speculationTree.Paths)
 	if err != nil {
-		return fmt.Errorf("failed to marshal speculations batchID=%s for UpdateSpeculations: %w", batchID, err)
+		return fmt.Errorf("failed to marshal paths batchID=%s for Update: %w", speculationTree.BatchID, err)
 	}
 
 	result, err := s.db.ExecContext(ctx,
-		"UPDATE speculation_tree SET speculations = ? WHERE batch_id = ?",
-		speculationsJSON, batchID,
+		"UPDATE speculation_tree SET paths = ? WHERE batch_id = ?",
+		pathsJSON, speculationTree.BatchID,
 	)
 	if err != nil {
-		return fmt.Errorf("failed to update speculations for batchID=%q: %w", batchID, err)
+		return fmt.Errorf("failed to update speculation tree for batchID=%q: %w", speculationTree.BatchID, err)
 	}
 
 	rowsAffected, err := result.RowsAffected()
 	if err != nil {
-		return fmt.Errorf("failed to get rows affected from update for batchID=%q: %w", batchID, err)
+		return fmt.Errorf("failed to get rows affected from update for batchID=%q: %w", speculationTree.BatchID, err)
 	}
 
 	if rowsAffected != 1 {
-		return storage.WrapNotFound(fmt.Errorf("speculation tree entity batchID=%s", batchID))
+		return storage.WrapNotFound(fmt.Errorf("speculation tree entity batchID=%s", speculationTree.BatchID))
 	}
 
 	return nil