diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index 2e1612ed77a..4ab4b552eae 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1617,8 +1617,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1792,6 +1793,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"violations-analyzed","question":"Did the workflow analyze changed Go or JavaScript files from the last 24 hours for architecture violations, or correctly conclude there were none?"},{"id":"issue-created-or-noop","question":"Was an architecture report issue created when violations were found, or was noop or skip behavior used appropriately when there was nothing actionable?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index a8bc5891a08..b913aa98d75 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1558,8 +1558,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 535260cc1d3..6c2b6150884 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -1686,8 +1686,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1857,6 +1858,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"audit-completed","question":"Did the workflow complete the blog availability audit, including page access and content validation checks?"},{"id":"discussion-created","question":"Was an audit discussion created summarizing the pass or fail results and suggested remediation when needed?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 932924895d6..5730ea767cc 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1614,8 +1614,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index 6bf5ab0efcf..5f2b1421fb1 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -1670,8 +1670,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1843,6 +1844,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"repair-or-optimization-path","question":"Did the workflow check validation-status first and then follow the correct repair or optimization path for this run?"},{"id":"pr-created-or-noop","question":"Was a focused CI improvement pull request created when actionable work was found, or was noop called when CI was already healthy?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/contribution-check.lock.yml b/.github/workflows/contribution-check.lock.yml index 86556705543..415de8b780e 100644 --- a/.github/workflows/contribution-check.lock.yml +++ b/.github/workflows/contribution-check.lock.yml @@ -1672,8 +1672,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index b108fc219dd..f90f920c5a6 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1799,8 +1799,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1970,6 +1971,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"gaps-confirmed","question":"Did the workflow identify at least one confirmed documentation gap to fix, or correctly conclude that no actionable gap remained?"},{"id":"pr-issue-or-noop","question":"Was a documentation pull request or issue created for confirmed gaps, or was noop used appropriately when nothing required action?"}]' GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index a01527c95c0..f63966bc683 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -1863,8 +1863,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -2036,6 +2037,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"report-generated","question":"Did the workflow analyze the prefetched issues data and generate a clustered daily issues report with metrics and trends?"},{"id":"discussion-created","question":"Was a daily issues discussion created successfully with the report findings and recommendations?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index 64da77c50b2..a65278c6e0b 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -1815,8 +1815,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1965,6 +1966,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"digest-generated","question":"Did the workflow analyze the prefetched repository activity and generate a coherent daily news digest?"},{"id":"discussion-created","question":"Was a daily-news discussion created or updated successfully with the report output?"}]' GH_AW_EVALS_MODEL: "copilot/gpt-5.4" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/draft-pr-cleanup.lock.yml b/.github/workflows/draft-pr-cleanup.lock.yml index bc855a84b50..ba1669649d8 100644 --- a/.github/workflows/draft-pr-cleanup.lock.yml +++ b/.github/workflows/draft-pr-cleanup.lock.yml @@ -1596,8 +1596,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1765,6 +1766,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"stale-drafts-triaged","question":"Did the workflow review open draft pull requests and classify stale drafts according to the warning and cleanup policy?"},{"id":"warnings-or-closures-applied","question":"Were the expected labels, comments, and closures applied to stale draft pull requests when appropriate?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index 51073d47135..60249abdcba 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -1731,8 +1731,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1904,6 +1905,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"subissues-planned","question":"Did the workflow analyze the triggering issue or discussion and break the work into actionable sub-issues?"},{"id":"issue-actions-completed","question":"Were the planned sub-issues created successfully, with discussion closure handled correctly when required?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 325ccb8defa..e8dcd020fc7 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -2818,8 +2818,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/stale-pr-cleanup.lock.yml b/.github/workflows/stale-pr-cleanup.lock.yml index b13c8de2d8d..5c280f43672 100644 --- a/.github/workflows/stale-pr-cleanup.lock.yml +++ b/.github/workflows/stale-pr-cleanup.lock.yml @@ -1591,8 +1591,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1754,6 +1755,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"stale-prs-triaged","question":"Did the workflow review pull requests open for 30 or more days and classify them according to the cleanup policy?"},{"id":"actions-taken-or-noop","question":"Were the appropriate stale PR comments, labels, and closures applied when needed, or was noop used correctly when no action was required?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index e4616091ae7..eddff1c7677 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -1807,8 +1807,9 @@ jobs: evals: needs: - activation - - safe_outputs - if: always() && needs.safe_outputs.result != 'skipped' + - agent + - detection + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest permissions: contents: read @@ -1974,6 +1975,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"draft-prepared","question":"Did the workflow analyze recent releases and merged pull requests and prepare a weekly blog post draft?"},{"id":"pr-created-or-explained","question":"Was a blog post pull request created when a draft was ready, or did the agent clearly explain why no pull request was needed?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/pkg/workflow/evals_job.go b/pkg/workflow/evals_job.go index 148f1afb7e9..72aca454f38 100644 --- a/pkg/workflow/evals_job.go +++ b/pkg/workflow/evals_job.go @@ -16,8 +16,8 @@ func evalsBranchName(workflowID string) string { return WorkflowStateBranchName(constants.EvalsBranchPrefix, workflowID) } -// buildEvalsJob creates a separate evals job that runs after the safe_outputs job -// (or directly after the agent job if safe_outputs is not configured). +// buildEvalsJob creates a separate evals job that runs after the agent job (and detection +// job when enabled), allowing it to run in parallel with safe_outputs. // The job downloads the agent artifact to access output files, runs a BinEval // multi-question evaluation via an agentic engine, and uploads evals.jsonl as an artifact. // Returns nil if evals are not declared in the workflow frontmatter. @@ -53,26 +53,19 @@ func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) { steps = append(steps, c.buildEvalsJobSteps(data)...) // Determine job dependencies. - // Evals runs after safe_outputs when it is configured; otherwise directly after agent. - var needs []string - if data.SafeOutputs != nil { - needs = []string{string(constants.SafeOutputsJobName), string(constants.ActivationJobName)} - } else { - needs = []string{string(constants.AgentJobName), string(constants.ActivationJobName)} + // Evals always depends on agent and activation, and additionally on detection when the detection job is enabled. + // This allows evals to run in parallel with safe_outputs. + needs := []string{string(constants.AgentJobName), string(constants.ActivationJobName)} + if IsDetectionJobEnabled(data.SafeOutputs) { + needs = append(needs, string(constants.DetectionJobName)) } evalsJobLog.Printf("Evals job dependencies resolved: needs=%v", needs) - // Evals job condition: always run but skip if the upstream job was skipped. + // Evals job condition: always run but skip if the agent job was skipped. // This matches the detection job pattern so conclusion still sees a non-skipped evals result. - var upstreamJobName string - if data.SafeOutputs != nil { - upstreamJobName = string(constants.SafeOutputsJobName) - } else { - upstreamJobName = string(constants.AgentJobName) - } alwaysFunc := BuildFunctionCall("always") upstreamNotSkipped := BuildNotEquals( - BuildPropertyAccess(fmt.Sprintf("needs.%s.result", upstreamJobName)), + BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.AgentJobName)), BuildStringLiteral("skipped"), ) jobConditionNode := BuildAnd(alwaysFunc, upstreamNotSkipped) diff --git a/pkg/workflow/evals_job_test.go b/pkg/workflow/evals_job_test.go new file mode 100644 index 00000000000..25e59df92eb --- /dev/null +++ b/pkg/workflow/evals_job_test.go @@ -0,0 +1,66 @@ +package workflow + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/github/gh-aw/pkg/constants" +) + +func TestBuildEvalsJobNeedsWithoutDetection(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "copilot", + Evals: &EvalsConfig{ + Questions: []EvalDefinition{ + {ID: "q1", Question: "Does it build?"}, + }, + }, + SafeOutputs: &SafeOutputsConfig{}, + } + + job, err := compiler.buildEvalsJob(data) + require.NoError(t, err) + require.NotNil(t, job) + + assert.ElementsMatch(t, []string{ + string(constants.AgentJobName), + string(constants.ActivationJobName), + }, job.Needs) + assert.NotContains(t, job.Needs, string(constants.SafeOutputsJobName)) + assert.NotContains(t, job.Needs, string(constants.DetectionJobName)) + assert.Contains(t, job.If, "needs.agent.result") + assert.NotContains(t, job.If, "needs.safe_outputs.result") +} + +func TestBuildEvalsJobNeedsWithDetection(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "copilot", + Evals: &EvalsConfig{ + Questions: []EvalDefinition{ + {ID: "q1", Question: "Does it build?"}, + }, + }, + SafeOutputs: &SafeOutputsConfig{ + ThreatDetection: &ThreatDetectionConfig{}, + }, + } + + job, err := compiler.buildEvalsJob(data) + require.NoError(t, err) + require.NotNil(t, job) + + assert.ElementsMatch(t, []string{ + string(constants.AgentJobName), + string(constants.ActivationJobName), + string(constants.DetectionJobName), + }, job.Needs) + assert.NotContains(t, job.Needs, string(constants.SafeOutputsJobName)) + assert.Contains(t, job.If, "needs.agent.result") + assert.NotContains(t, job.If, "needs.safe_outputs.result") +}