Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions rest-api/docs/index.html

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions rest-api/flow/internal/converter/dao/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ func OperationRunFrom(dao *model.OperationRun) *operationrun.OperationRun {
StatusReason: dao.StatusReason,
StatusMessage: dao.StatusMessage,
CurrentPhaseIndex: dao.CurrentPhaseIndex,
TotalPhases: dao.TotalPhases,
Selector: dao.Selector,
Options: dao.Options,
OperationTemplate: dao.OperationTemplate,
Expand All @@ -368,6 +369,7 @@ func OperationRunTo(run *operationrun.OperationRun) *model.OperationRun {
StatusReason: run.StatusReason,
StatusMessage: run.StatusMessage,
CurrentPhaseIndex: run.CurrentPhaseIndex,
TotalPhases: run.TotalPhases,
Selector: run.Selector,
Options: run.Options,
OperationTemplate: run.OperationTemplate,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1125,7 +1125,7 @@ func OperationRunTo(run *operationrun.OperationRun) (*pb.OperationRun, error) {
}

result := &pb.OperationRun{
Summary: operationRunSummaryTo(run, &options),
Summary: operationRunSummaryTo(run),
Configuration: &pb.OperationRunConfiguration{
Selector: pbSelector,
Options: pbOptions,
Expand All @@ -1140,26 +1140,19 @@ func OperationRunTo(run *operationrun.OperationRun) (*pb.OperationRun, error) {
}

// OperationRunSummaryTo converts a domain operation run to the lightweight
// list shape. It intentionally avoids unmarshalling selector and operation
// template JSON.
// list shape without unmarshalling configuration JSON.
func OperationRunSummaryTo(
run *operationrun.OperationRun,
) (*pb.OperationRunSummary, error) {
if run == nil {
return nil, nil
}

var options operationrun.Options
if err := operationrun.UnmarshalConfig(run.Options, &options); err != nil {
return nil, fmt.Errorf("unmarshal options: %w", err)
}

return operationRunSummaryTo(run, &options), nil
return operationRunSummaryTo(run), nil
}

func operationRunSummaryTo(
run *operationrun.OperationRun,
options *operationrun.Options,
) *pb.OperationRunSummary {
summary := &pb.OperationRunSummary{
Id: UUIDTo(run.ID),
Expand All @@ -1168,7 +1161,7 @@ func operationRunSummaryTo(
OperationKind: OperationKindTo(run.OperationType, run.OperationCode),
State: OperationRunStateTo(run.Status, run.StatusReason),
StatusMessage: run.StatusMessage,
TotalPhases: operationRunPhaseCount(&options.PhasePolicy),
TotalPhases: run.TotalPhases,
CreatedAt: timestamppb.New(run.CreatedAt),
UpdatedAt: timestamppb.New(run.UpdatedAt),
}
Expand Down Expand Up @@ -1428,38 +1421,6 @@ func OperationRunTargetListOptionsFrom(
return opts, nil
}

func operationRunPhaseCount(policy *operationrun.PhasePolicy) int32 {
if policy == nil {
return 0
}
if policy.Plan == nil {
return 0
}

switch policy.Plan.PhasePlanKind() {
case operationrun.PhasePlanKindEqual:
equal, ok := policy.Plan.(*operationrun.EqualPhases)
if !ok || equal == nil {
return 0
}
return equal.PhaseCount
case operationrun.PhasePlanKindPercentage:
percentage, ok := policy.Plan.(*operationrun.PercentagePhases)
if !ok || percentage == nil {
return 0
}
return int32(len(percentage.Phases))
case operationrun.PhasePlanKindCount:
count, ok := policy.Plan.(*operationrun.CountPhases)
if !ok || count == nil {
return 0
}
return int32(len(count.Phases) + 1)
default:
return 0
}
}

// OperationRunStatusTo converts a domain operation-run status to proto.
func OperationRunStatusTo(
status operationrun.OperationRunStatus,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,13 @@ func TestOperationRunFromDefaults(t *testing.T) {
func TestOperationRunToRebuildsConfigurationFromInternalJSON(t *testing.T) {
run, err := OperationRunFrom(validCreateRequest())
require.NoError(t, err)
run.TotalPhases = 2

got, err := OperationRunTo(run)
require.NoError(t, err)

require.NotNil(t, got.GetConfiguration())
require.Equal(t, int32(2), got.GetSummary().GetTotalPhases())
require.EqualValues(
t,
10,
Expand Down Expand Up @@ -138,6 +140,18 @@ func TestOperationRunToRebuildsConfigurationFromInternalJSON(t *testing.T) {
)
}

func TestOperationRunSummaryToUsesMaterializedPhaseCount(t *testing.T) {
run := &operationrun.OperationRun{
TotalPhases: 2,
Options: []byte("{"),
}

got, err := OperationRunSummaryTo(run)

require.NoError(t, err)
require.Equal(t, int32(2), got.GetTotalPhases())
}

func TestOperationRunStatusConversionIncludesCompletedWithFailures(t *testing.T) {
require.Equal(
t,
Expand Down
1 change: 1 addition & 0 deletions rest-api/flow/internal/db/model/operation_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type OperationRun struct {
StatusReason operationrun.OperationRunStatusReason `bun:"status_reason,type:varchar(64),notnull"` //nolint:lll
StatusMessage string `bun:"status_message,nullzero"`
CurrentPhaseIndex int32 `bun:"current_phase_index,notnull"`
TotalPhases int32 `bun:"total_phases,scanonly"`
Selector json.RawMessage `bun:"selector,type:jsonb,notnull"`
Options json.RawMessage `bun:"options,type:jsonb,notnull"`
OperationTemplate json.RawMessage `bun:"operation_template,type:jsonb,notnull"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,14 @@ func (d *Dispatcher) decide(
)
switch phaseDecision.action {
case phaseDecisionActionAdvance:
if phaseDecision.nextPhaseIndex == nil {
return dispatchDecision{}, fmt.Errorf(
"advance phase decision is missing next phase index",
)
}
// The next phase's targets were not locked during this transaction.
// Persist the phase pointer first; the next pass will lock and submit them.
prep.run.CurrentPhaseIndex++
prep.run.CurrentPhaseIndex = *phaseDecision.nextPhaseIndex
return newStopDecision(startRunTransition(phaseDecision.message)), nil
case phaseDecisionActionPause:
return newStopDecision(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,62 @@ func TestDispatchRunLocksOnlyCurrentPhaseTargets(t *testing.T) {
require.Equal(t, operationrun.OperationRunStatusReasonPhaseGate, store.run.StatusReason)
}

func TestDispatchRunSkipsLegacyConsecutiveEmptyPhases(t *testing.T) {
tests := []struct {
name string
autoAdvance bool
}{
{name: "automatic phase policy", autoAdvance: true},
{name: "manual phase policy"},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runID := uuid.New()
nextRackID := uuid.New()
nextTaskID := uuid.New()
now := time.Date(2026, 8, 14, 10, 0, 0, 0, time.UTC)
completed := testTarget(runID, uuid.New(), uuid.New(), 0, 0)
completed.Status = operationrun.OperationRunTargetStatusCompleted
pending := testTarget(runID, nextRackID, uuid.New(), 1, 3)

store := newFakeStore(
testRun(t, runID, 1, operationrun.PhasePolicy{
AdvancePolicy: operationrun.PhaseAdvancePolicy{
AutoAdvance: tt.autoAdvance,
},
}),
[]*operationrun.OperationRunTarget{completed, pending},
)
store.run.Status = operationrun.OperationRunStatusRunning
store.run.CurrentPhaseIndex = 1
taskManager := &fakeTaskManager{
results: map[uuid.UUID]submitResult{
nextRackID: {ids: []uuid.UUID{nextTaskID}},
},
}
dispatcher := newTestDispatcherAt(t, Dependencies{
Store: store,
TaskManager: taskManager,
TaskStore: &fakeTaskStore{},
}, Config{FetchBatch: 10}, now)

err := dispatcher.dispatchRun(context.Background(), runID)
require.NoError(t, err)
require.Equal(t, int32(3), store.run.CurrentPhaseIndex)
require.Equal(t, operationrun.OperationRunStatusRunning, store.run.Status)
require.Equal(t, "skipped empty phase", store.run.StatusMessage)
require.Empty(t, taskManager.requests)

err = dispatcher.dispatchRun(context.Background(), runID)
require.NoError(t, err)
require.Len(t, taskManager.requests, 1)
require.Equal(t, nextRackID, taskManager.requests[0].RequiredRackID)
require.Equal(t, operationrun.OperationRunTargetStatusSubmitted, pending.Status)
})
}
}

func TestDispatchRunPausesWhenSafetyGateTrips(t *testing.T) {
runID := uuid.New()
taskID := uuid.New()
Expand Down Expand Up @@ -628,7 +684,6 @@ func TestDecideSetsTerminalStatusFromTargetOutcomes(t *testing.T) {
Status: operationrun.OperationRunStatusRunning,
}
summary := operationrun.TargetPhaseSummary{}
summary.TotalPhases = 1
summary.CurrentPhaseStats.SelectedTargets = len(tt.statuses)
for _, status := range tt.statuses {
summary.CurrentPhaseStats.StatusCounts.Add(status)
Expand Down Expand Up @@ -713,19 +768,21 @@ func (s *fakeStore) GetTargetPhaseAggregate(
runID uuid.UUID,
currentPhaseIndex int32,
) (operationrun.TargetPhaseAggregate, error) {
var totalPhases int32
var nextPhaseIndex *int32
completedStats := operationrun.PhaseStats{PhaseIndex: max(currentPhaseIndex-1, 0)}
for _, target := range s.targets {
if target.PhaseIndex+1 > totalPhases {
totalPhases = target.PhaseIndex + 1
}
if target.PhaseIndex < currentPhaseIndex {
completedStats.AddTarget(target)
}
if target.PhaseIndex > currentPhaseIndex &&
(nextPhaseIndex == nil || target.PhaseIndex < *nextPhaseIndex) {
candidate := target.PhaseIndex
nextPhaseIndex = &candidate
}
}

return operationrun.TargetPhaseAggregate{
TotalPhases: totalPhases,
NextPhaseIndex: nextPhaseIndex,
CompletedPhaseStats: completedStats,
}, nil
}
Expand Down
30 changes: 22 additions & 8 deletions rest-api/flow/internal/operationrun/manager/dispatcher/phase.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,18 @@ const (
)

type phaseDecision struct {
action phaseDecisionAction
reason operationrun.OperationRunStatusReason
message string
action phaseDecisionAction
nextPhaseIndex *int32
reason operationrun.OperationRunStatusReason
message string
}

func newAdvancePhaseDecision(nextPhaseIndex int32, message string) phaseDecision {
return phaseDecision{
action: phaseDecisionActionAdvance,
nextPhaseIndex: &nextPhaseIndex,
message: message,
}
}

func newPhasePolicy(options *operationrun.Options) (*phasePolicyRuntime, error) {
Expand All @@ -40,17 +49,22 @@ func newPhasePolicy(options *operationrun.Options) (*phasePolicyRuntime, error)
func (p phasePolicyRuntime) evaluate(
summary operationrun.TargetPhaseSummary,
) phaseDecision {
if !summary.CurrentPhaseTerminal() || !summary.HasNextPhase() {
nextPhaseIndex, hasNextPhase := summary.NextPhase()

// Empty phases are planner artifacts rather than rollout boundaries. Skip
// legacy gaps without applying the configured automatic or manual gate.
if summary.CurrentPhaseEmpty() && hasNextPhase {
return newAdvancePhaseDecision(nextPhaseIndex, "skipped empty phase")
}

if !summary.CurrentPhaseTerminal() || !hasNextPhase {
return phaseDecision{
action: phaseDecisionActionClaim,
}
}

if p.autoAdvance {
return phaseDecision{
action: phaseDecisionActionAdvance,
message: "advanced to next phase",
}
return newAdvancePhaseDecision(nextPhaseIndex, "advanced to next phase")
}

return phaseDecision{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package dispatcher

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestNewAdvancePhaseDecisionCarriesExplicitPhaseIndex(t *testing.T) {
tests := []struct {
name string
nextPhaseIndex int32
}{
{name: "phase zero is explicit", nextPhaseIndex: 0},
{name: "later phase", nextPhaseIndex: 3},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
decision := newAdvancePhaseDecision(tt.nextPhaseIndex, "advance")

require.Equal(t, phaseDecisionActionAdvance, decision.action)
require.NotNil(t, decision.nextPhaseIndex)
require.Equal(t, tt.nextPhaseIndex, *decision.nextPhaseIndex)
})
}
}
36 changes: 31 additions & 5 deletions rest-api/flow/internal/operationrun/manager/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,30 @@ func TestAdvancePhaseChecksExpectedPhase(t *testing.T) {
require.Zero(t, store.updateRunCalls)
}

func TestAdvancePhaseSkipsLegacyEmptyPhases(t *testing.T) {
runID := uuid.MustParse("11111111-1111-1111-1111-111111111111")
store := &mockStore{
lockRun: testLockedRun(
t,
runID,
operationrun.OperationRunStatusPaused,
operationrun.OperationRunStatusReasonPhaseGate,
),
lockedTargets: []*operationrun.OperationRunTarget{
testOperationRunTarget(runID, 0, operationrun.OperationRunTargetStatusCompleted),
testOperationRunTarget(runID, 3, operationrun.OperationRunTargetStatusPending),
},
}
manager := newTestManager(t, store, planner.New(&mockTargetLookup{}, planner.Config{}))
expectedPhase := int32(3)

got, err := manager.AdvancePhase(context.Background(), runID, &expectedPhase)

require.NoError(t, err)
require.Equal(t, int32(3), got.CurrentPhaseIndex)
require.Equal(t, "advanced to phase 3", got.StatusMessage)
}

func TestAdvancePhaseCompletesAllTerminalRun(t *testing.T) {
runID := uuid.MustParse("11111111-1111-1111-1111-111111111111")
store := &mockStore{
Expand Down Expand Up @@ -724,19 +748,21 @@ func (m *mockStore) GetTargetPhaseAggregate(
runID uuid.UUID,
currentPhaseIndex int32,
) (operationrun.TargetPhaseAggregate, error) {
var totalPhases int32
var nextPhaseIndex *int32
completedStats := operationrun.PhaseStats{PhaseIndex: max(currentPhaseIndex-1, 0)}
for _, target := range m.lockedTargets {
if target.PhaseIndex+1 > totalPhases {
totalPhases = target.PhaseIndex + 1
}
if target.PhaseIndex < currentPhaseIndex {
completedStats.AddTarget(target)
}
if target.PhaseIndex > currentPhaseIndex &&
(nextPhaseIndex == nil || target.PhaseIndex < *nextPhaseIndex) {
candidate := target.PhaseIndex
nextPhaseIndex = &candidate
}
}

return operationrun.TargetPhaseAggregate{
TotalPhases: totalPhases,
NextPhaseIndex: nextPhaseIndex,
CompletedPhaseStats: completedStats,
}, nil
}
Expand Down
Loading