From 5a8f0c060ec745d45002414969ec8fc69c2ed361 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 15 Jul 2026 19:04:35 +0000
Subject: [PATCH 1/2] feat: add daily-evals-report workflow for evals feature
analysis
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
.github/workflows/daily-evals-report.lock.yml | 1816 +++++++++++++++++
.github/workflows/daily-evals-report.md | 242 +++
2 files changed, 2058 insertions(+)
create mode 100644 .github/workflows/daily-evals-report.lock.yml
create mode 100644 .github/workflows/daily-evals-report.md
diff --git a/.github/workflows/daily-evals-report.lock.yml b/.github/workflows/daily-evals-report.lock.yml
new file mode 100644
index 00000000000..b33b0dba826
--- /dev/null
+++ b/.github/workflows/daily-evals-report.lock.yml
@@ -0,0 +1,1816 @@
+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"87ee75709c6a01a3c0c2fc6649a108cd998eb5249924ebabc26352361e7d4cc6","body_hash":"eb2526b7c3336226685ae44ca18da78791e20e13de2f2ee4495b427493eddfbe","strict":true,"agent_id":"codex","agent_model":"gpt-5.4","engine_versions":{"codex":"0.144.4"}}
+# gh-aw-manifest: {"version":1,"secrets":["CODEX_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"924ae3a1cded613372ab5595356fb5720e22ba16","version":"v6.5.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"f9f3042f7e2789586610d6e8b85c8f03e5195baf","version":"v7.2.0"},{"repo":"docker/setup-buildx-action","sha":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5","version":"v4.1.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.31","digest":"sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.31@sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31","digest":"sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31@sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.31","digest":"sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.31@sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.31","digest":"sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.31@sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.1","digest":"sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.1@sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]}
+# This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md
+#
+# ___ _ _
+# / _ \ | | (_)
+# | |_| | __ _ ___ _ __ | |_ _ ___
+# | _ |/ _` |/ _ \ '_ \| __| |/ __|
+# | | | | (_| | __/ | | | |_| | (__
+# \_| |_/\__, |\___|_| |_|\__|_|\___|
+# __/ |
+# _ _ |___/
+# | | | | / _| |
+# | | | | ___ _ __ _ __| |_| | _____ ____
+# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___|
+# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \
+# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/
+#
+#
+# To update this file, edit the corresponding .md file and run:
+# gh aw compile
+# Not all edits will cause changes to this file.
+#
+# For more information: https://github.github.com/gh-aw/introduction/overview/
+#
+# Daily data science report analyzing evals feature adoption, per-question pass rates, and quality trends across agentic workflows
+#
+# Resolved workflow manifest:
+# Imports:
+# - shared/daily-audit-discussion.md
+# - shared/meta-analysis-base.md
+# - shared/otlp.md
+# - shared/reporting.md
+# - shared/daily-audit-base.md
+#
+# Secrets used:
+# - CODEX_API_KEY
+# - COPILOT_GITHUB_TOKEN
+# - GH_AW_GITHUB_MCP_SERVER_TOKEN
+# - GH_AW_GITHUB_TOKEN
+# - GH_AW_OTEL_GRAFANA_AUTHORIZATION
+# - GH_AW_OTEL_GRAFANA_ENDPOINT
+# - GH_AW_OTEL_SENTRY_AUTHORIZATION
+# - GH_AW_OTEL_SENTRY_ENDPOINT
+# - GITHUB_TOKEN
+# - OPENAI_API_KEY
+#
+# Custom actions used:
+# - actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+# - actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+# - actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+# - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9)
+# - actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+# - docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
+# - docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
+#
+# Container images used:
+# - ghcr.io/github/gh-aw-firewall/agent:0.27.31@sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1
+# - ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31@sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63
+# - ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.31@sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673
+# - ghcr.io/github/gh-aw-firewall/squid:0.27.31@sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494
+# - ghcr.io/github/gh-aw-mcpg:v0.4.1@sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32
+# - ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b
+# - ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4
+
+name: "Daily Evals Feature Report"
+on:
+ schedule:
+ - cron: "11 7 * * *" # Friendly format: daily around 8:00 (scattered)
+ workflow_dispatch:
+ inputs:
+ aw_context:
+ default: ""
+ description: "Agent caller context (used internally by Agentic Workflows)."
+ required: false
+ type: string
+
+permissions: {}
+
+concurrency:
+ group: "gh-aw-${{ github.workflow }}"
+
+run-name: "Daily Evals Feature Report"
+
+env:
+ OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.GH_AW_OTEL_SENTRY_ENDPOINT }}
+ OTEL_SERVICE_NAME: gh-aw.daily-evals-report
+ OTEL_RESOURCE_ATTRIBUTES: 'gh-aw.workflow.name=Daily%20Evals%20Feature%20Report,gh-aw.repository=${{ github.repository }},gh-aw.run.id=${{ github.run_id }},github.run_id=${{ github.run_id }},gh-aw.engine.id=codex'
+ OTEL_EXPORTER_OTLP_HEADERS: x-sentry-auth=${{ secrets.GH_AW_OTEL_SENTRY_AUTHORIZATION }}
+ GH_AW_OTLP_ALL_HEADERS: x-sentry-auth=${{ secrets.GH_AW_OTEL_SENTRY_AUTHORIZATION }},Authorization=${{ secrets.GH_AW_OTEL_GRAFANA_AUTHORIZATION }}
+ GH_AW_OTLP_ENDPOINTS: '[{"url":"${{ secrets.GH_AW_OTEL_SENTRY_ENDPOINT }}","headers":"x-sentry-auth=${{ secrets.GH_AW_OTEL_SENTRY_AUTHORIZATION }}"},{"url":"${{ secrets.GH_AW_OTEL_GRAFANA_ENDPOINT }}","headers":"Authorization=${{ secrets.GH_AW_OTEL_GRAFANA_AUTHORIZATION }}"}]'
+
+jobs:
+ activation:
+ runs-on: ubuntu-slim
+ permissions:
+ actions: read
+ contents: read
+ env:
+ GH_AW_MAX_DAILY_AI_CREDITS: "3000"
+ GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
+ outputs:
+ comment_id: ""
+ comment_repo: ""
+ daily_ai_credits_exceeded: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }}
+ daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }}
+ daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }}
+ engine_id: ${{ steps.generate_aw_info.outputs.engine_id }}
+ lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }}
+ model: ${{ steps.generate_aw_info.outputs.model }}
+ oauth_token_check_failed: ${{ steps.check-oauth-tokens.outputs.oauth_token_check_failed == 'true' }}
+ secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }}
+ setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }}
+ setup-span-id: ${{ steps.setup.outputs.span-id }}
+ setup-trace-id: ${{ steps.setup.outputs.trace-id }}
+ stale_lock_file_failed: ${{ steps.check-lock-file.outputs.stale_lock_file_failed == 'true' }}
+ steps:
+ - name: Checkout actions folder
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: github/gh-aw
+ sparse-checkout: |
+ actions
+ clean: false
+ persist-credentials: false
+ - name: Setup Scripts
+ id: setup
+ uses: ./actions/setup
+ with:
+ destination: ${{ runner.temp }}/gh-aw/actions
+ job-name: ${{ github.job }}
+ safe-output-artifact-client: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
+ env:
+ GH_AW_SETUP_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-evals-report.lock.yml@${{ github.ref }}
+ GH_AW_INFO_VERSION: "0.144.4"
+ GH_AW_INFO_AWF_VERSION: "v0.27.31"
+ GH_AW_INFO_ENGINE_ID: "codex"
+ - name: Mask OTLP telemetry headers
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh"
+ - name: Generate agentic run info
+ id: generate_aw_info
+ env:
+ GH_AW_INFO_ENGINE_ID: "codex"
+ GH_AW_INFO_ENGINE_NAME: "Codex"
+ GH_AW_INFO_MODEL: "gpt-5.4"
+ GH_AW_INFO_VERSION: "0.144.4"
+ GH_AW_INFO_AGENT_VERSION: "0.144.4"
+ GH_AW_INFO_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_INFO_EXPERIMENTAL: "false"
+ GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true"
+ GH_AW_INFO_STAGED: "false"
+ GH_AW_INFO_ALLOWED_DOMAINS: '["*.grafana.net","*.sentry.io","defaults"]'
+ GH_AW_INFO_FIREWALL_ENABLED: "true"
+ GH_AW_INFO_AWF_VERSION: "v0.27.31"
+ GH_AW_INFO_AWMG_VERSION: ""
+ GH_AW_INFO_FIREWALL_TYPE: "squid"
+ GH_AW_INFO_FRONTMATTER_EMOJI: "๐งช"
+ GH_AW_COMPILED_STRICT: "true"
+ GH_AW_INFO_FEATURES: '{"gh-aw-detection":true}'
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs');
+ await main(core, context);
+ - name: Restore daily AIC usage cache
+ id: restore-daily-aic-cache
+ if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
+ continue-on-error: true
+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ key: agentic-workflow-usage-dailyevalsreport-${{ github.run_id }}
+ restore-keys: agentic-workflow-usage-dailyevalsreport-
+ path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
+ - name: Restore daily AIC usage cache (artifact fallback)
+ id: restore-daily-aic-cache-fallback
+ if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
+ continue-on-error: true
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_RESTORE_DAILY_AIC_CACHE_HIT: ${{ steps.restore-daily-aic-cache.outputs.cache-hit }}
+ GH_AW_RESTORE_DAILY_AIC_CACHE_MATCHED_KEY: ${{ steps.restore-daily-aic-cache.outputs.cache-matched-key }}
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/restore_aic_usage_cache_fallback.cjs');
+ await main();
+ - name: Check daily workflow token guardrail
+ id: daily-effective-workflow-guardrail
+ if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }}
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_ID: "daily-evals-report"
+ GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ GH_AW_WORKFLOW_DISPATCH_AW_CONTEXT: ${{ github.event.inputs.aw_context || '' }}
+ GH_AW_HAS_SLASH_COMMAND: "false"
+ GH_AW_HAS_LABEL_COMMAND: "false"
+ GH_AW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_AW_MAX_DAILY_AI_CREDITS: "3000"
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/check_daily_aic_workflow_guardrail.cjs');
+ await main();
+ - name: Validate CODEX_API_KEY or OPENAI_API_KEY secret
+ id: validate-secret
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_multi_secret.sh" CODEX_API_KEY OPENAI_API_KEY Codex https://github.github.com/gh-aw/reference/engines/#openai-codex
+ env:
+ CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ - name: Check for OAuth tokens
+ id: check-oauth-tokens
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/check_oauth_tokens.sh"
+ env:
+ COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
+ GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}
+ GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}
+ - name: Checkout .github and .agents folders
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+ sparse-checkout: |
+ .github
+ .agents
+ actions/setup
+ .antigravity
+ .claude
+ .codex
+ .gemini
+ .opencode
+ .pi
+ sparse-checkout-cone-mode: true
+ fetch-depth: 1
+ - name: Save agent config folders for base branch restoration
+ env:
+ GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .gemini .github .opencode .pi"
+ GH_AW_AGENT_FILES: "AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc"
+ # poutine:ignore untrusted_checkout_exec
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh"
+ - name: Check workflow lock file
+ id: check-lock-file
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_WORKFLOW_FILE: "daily-evals-report.lock.yml"
+ GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}"
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs');
+ await main();
+ - name: Log runtime features
+ if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }}
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh"
+ - name: Create prompt with built-in context
+ env:
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl
+ GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
+ GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
+ GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
+ GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }}
+ GH_AW_GITHUB_ACTOR: ${{ github.actor }}
+ GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
+ GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
+ GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
+ # poutine:ignore untrusted_checkout_exec
+ run: |
+ bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh"
+ {
+ cat << 'GH_AW_PROMPT_f86f2ea13492f48f_EOF'
+
+ GH_AW_PROMPT_f86f2ea13492f48f_EOF
+ cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md"
+ cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md"
+ cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md"
+ cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md"
+ cat << 'GH_AW_PROMPT_f86f2ea13492f48f_EOF'
+
+ Tools: create_issue, create_discussion, missing_tool, missing_data, noop
+
+ GH_AW_PROMPT_f86f2ea13492f48f_EOF
+ cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md"
+ cat << 'GH_AW_PROMPT_f86f2ea13492f48f_EOF'
+
+ The following GitHub context information is available for this workflow:
+ {{#if github.actor}}
+ - **actor**: __GH_AW_GITHUB_ACTOR__
+ {{/if}}
+ {{#if github.repository}}
+ - **repository**: __GH_AW_GITHUB_REPOSITORY__
+ {{/if}}
+ {{#if github.workspace}}
+ - **workspace**: __GH_AW_GITHUB_WORKSPACE__
+ {{/if}}
+ {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}
+ - **issue-number**: #__GH_AW_EXPR_802A9F6A__
+ {{/if}}
+ {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}
+ - **discussion-number**: #__GH_AW_EXPR_1A3A194A__
+ {{/if}}
+ {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}
+ - **pull-request-number**: #__GH_AW_EXPR_463A214A__
+ {{/if}}
+ {{#if github.event.comment.id || github.aw.context.comment_id}}
+ - **comment-id**: __GH_AW_EXPR_FF1D34CE__
+ {{/if}}
+ {{#if github.run_id}}
+ - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__
+ {{/if}}
+
+
+ GH_AW_PROMPT_f86f2ea13492f48f_EOF
+ cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md"
+ cat << 'GH_AW_PROMPT_f86f2ea13492f48f_EOF'
+
+ **IMPORTANT**: When analyzing agentic workflows, use the `agentic-workflows` tool to read workflow files.
+ {{#runtime-import .github/workflows/shared/reporting.md}}
+ {{#runtime-import .github/workflows/shared/otlp.md}}
+ {{#runtime-import .github/workflows/daily-evals-report.md}}
+ GH_AW_PROMPT_f86f2ea13492f48f_EOF
+ } > "$GH_AW_PROMPT"
+ - name: Interpolate variables and render templates
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ GH_AW_ENGINE_ID: "codex"
+ GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
+ GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs');
+ await main();
+ - name: Substitute placeholders
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
+ GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
+ GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }}
+ GH_AW_EXPR_FF1D34CE: ${{ github.event.comment.id || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').comment_id }}
+ GH_AW_GITHUB_ACTOR: ${{ github.actor }}
+ GH_AW_GITHUB_REPOSITORY: ${{ github.repository }}
+ GH_AW_GITHUB_RUN_ID: ${{ github.run_id }}
+ GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }}
+ GH_AW_MCP_CLI_SERVERS_LIST: "- `agenticworkflows` โ run `agenticworkflows --help` to see available tools\n- `safeoutputs` โ run `safeoutputs --help` to see available tools"
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+
+ const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs');
+
+ // Call the substitution function
+ return await substitutePlaceholders({
+ file: process.env.GH_AW_PROMPT,
+ substitutions: {
+ GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A,
+ GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A,
+ GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A,
+ GH_AW_EXPR_FF1D34CE: process.env.GH_AW_EXPR_FF1D34CE,
+ GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR,
+ GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY,
+ GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID,
+ GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE,
+ GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST
+ }
+ });
+ - name: Validate prompt placeholders
+ env:
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ # poutine:ignore untrusted_checkout_exec
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh"
+ - name: Print prompt
+ env:
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ # poutine:ignore untrusted_checkout_exec
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh"
+ - name: Upload activation artifact
+ if: success()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: activation
+ include-hidden-files: true
+ path: |
+ /tmp/gh-aw/aw_info.json
+ /tmp/gh-aw/models.json
+ /tmp/gh-aw/aw-prompts/prompt.txt
+ /tmp/gh-aw/aw-prompts/prompt-template.txt
+ /tmp/gh-aw/aw-prompts/prompt-import-tree.json
+ /tmp/gh-aw/github_rate_limits.jsonl
+ /tmp/gh-aw/base
+ /tmp/gh-aw/.codex/agents
+ /tmp/gh-aw/.codex/skills
+ if-no-files-found: ignore
+ retention-days: 1
+
+ agent:
+ needs: activation
+ if: needs.activation.outputs.daily_ai_credits_exceeded != 'true'
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ contents: read
+ issues: read
+ pull-requests: read
+ concurrency:
+ group: "gh-aw-codex-${{ github.workflow }}"
+ queue: max
+ env:
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ GH_AW_ASSETS_ALLOWED_EXTS: ""
+ GH_AW_ASSETS_BRANCH: ""
+ GH_AW_ASSETS_MAX_SIZE_KB: 0
+ GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs
+ GH_AW_PROJECT_UTC: "-08:00"
+ GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
+ GH_AW_WORKFLOW_ID_SANITIZED: dailyevalsreport
+ outputs:
+ agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }}
+ ai_credits_rate_limit_error: ${{ steps.parse-mcp-gateway.outputs.ai_credits_rate_limit_error || 'false' }}
+ aic: ${{ steps.parse-mcp-gateway.outputs.aic }}
+ ambient_context: ${{ steps.parse-mcp-gateway.outputs.ambient_context }}
+ checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }}
+ effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }}
+ has_patch: ${{ steps.collect_output.outputs.has_patch }}
+ http_400_response_error: ${{ steps.detect-agent-errors.outputs.http_400_response_error || 'false' }}
+ inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }}
+ mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }}
+ model: ${{ needs.activation.outputs.model }}
+ model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }}
+ output: ${{ steps.collect_output.outputs.output }}
+ output_types: ${{ steps.collect_output.outputs.output_types }}
+ setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }}
+ setup-span-id: ${{ steps.setup.outputs.span-id }}
+ setup-trace-id: ${{ steps.setup.outputs.trace-id }}
+ unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }}
+ steps:
+ - name: Checkout actions folder
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: github/gh-aw
+ sparse-checkout: |
+ actions
+ clean: false
+ persist-credentials: false
+ - name: Setup Scripts
+ id: setup
+ uses: ./actions/setup
+ with:
+ destination: ${{ runner.temp }}/gh-aw/actions
+ job-name: ${{ github.job }}
+ trace-id: ${{ needs.activation.outputs.setup-trace-id }}
+ parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
+ env:
+ GH_AW_SETUP_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-evals-report.lock.yml@${{ github.ref }}
+ GH_AW_INFO_VERSION: "0.144.4"
+ GH_AW_INFO_AWF_VERSION: "v0.27.31"
+ GH_AW_INFO_ENGINE_ID: "codex"
+ - name: Set runtime paths
+ id: set-runtime-paths
+ run: |
+ {
+ echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl"
+ echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json"
+ echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json"
+ } >> "$GITHUB_OUTPUT"
+ - name: Mask OTLP telemetry headers
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh"
+ - name: Checkout repository
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+ - name: Setup Go for CLI build
+ uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+ with:
+ go-version-file: go.mod
+ cache: true
+ - name: Build gh-aw CLI
+ run: |
+ echo "Building gh-aw CLI for linux/amd64..."
+ mkdir -p dist
+ VERSION=$(git describe --tags --always --dirty)
+ CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+ -ldflags "-s -w -X main.version=${VERSION}" \
+ -o dist/gh-aw-linux-amd64 \
+ ./cmd/gh-aw
+ # Copy binary to root for direct execution in user-defined steps
+ cp dist/gh-aw-linux-amd64 ./gh-aw
+ chmod +x ./gh-aw
+ echo "โ Built gh-aw CLI successfully"
+ - name: Setup Docker Buildx
+ uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
+ - name: Build gh-aw Docker image
+ uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
+ with:
+ context: .
+ platforms: linux/amd64
+ push: false
+ load: true
+ tags: localhost/gh-aw:dev
+ build-args: |
+ BINARY=dist/gh-aw-linux-amd64
+ - name: Create gh-aw temp directory
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/create_gh_aw_tmp_dir.sh"
+ - name: Configure gh CLI for GitHub Enterprise
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh"
+ env:
+ GH_TOKEN: ${{ github.token }}
+ - name: Download activation artifact
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: activation
+ path: /tmp/gh-aw
+ - name: Configure Git credentials
+ env:
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ GITHUB_SERVER_URL: ${{ github.server_url }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh"
+ - name: Checkout PR branch
+ id: checkout-pr
+ if: |
+ github.event.pull_request || github.event.issue.pull_request || github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.aw_context || '{}').item_type == 'pull_request'
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs');
+ await main();
+ - name: Setup Node.js
+ uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+ with:
+ node-version: '24'
+ package-manager-cache: false
+ - name: Install Codex CLI
+ run: npm install --ignore-scripts -g @openai/codex@0.144.4
+ - name: Install AWF binary
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.31 --rootless
+ - name: Determine automatic lockdown mode for GitHub MCP Server
+ id: determine-automatic-lockdown
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9)
+ env:
+ GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}
+ GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}
+ with:
+ script: |
+ const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs');
+ await determineAutomaticLockdown(github, context, core);
+ - name: Restore agent config folders from base branch
+ if: steps.checkout-pr.outcome == 'success'
+ env:
+ GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .gemini .github .opencode .pi"
+ GH_AW_AGENT_FILES: "AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc"
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh"
+ - name: Restore inline sub-agents from activation artifact
+ env:
+ GH_AW_SUB_AGENT_DIR: ".codex/agents"
+ GH_AW_SUB_AGENT_EXT: ".md"
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_sub_agents.sh"
+ - name: Restore inline skills from activation artifact
+ env:
+ GH_AW_SKILL_DIR: ".codex/skills"
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh"
+ - name: Download container images
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.31@sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31@sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63 ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.31@sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673 ghcr.io/github/gh-aw-firewall/squid:0.27.31@sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494 ghcr.io/github/gh-aw-mcpg:v0.4.1@sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32 ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4
+ - name: Build and install gh-aw CLI from source
+ run: |
+ gh extension remove aw || true
+ make build
+ gh extension install .
+ gh aw version
+ env:
+ GH_TOKEN: ${{ github.token }}
+ - name: Copy gh-aw binary for MCP server
+ run: |
+ gh aw --version
+ # Copy the gh-aw binary to ${RUNNER_TEMP}/gh-aw for MCP server containerization
+ mkdir -p "${RUNNER_TEMP}/gh-aw"
+ GH_AW_BIN=""
+ GH_AW_BIN=$(command -v gh-aw 2>/dev/null) || true
+ if [ -z "$GH_AW_BIN" ]; then
+ GH_AW_BIN=$(find "${HOME}/.local/share/gh/extensions/gh-aw" -name 'gh-aw' -type f 2>/dev/null | head -1) || true
+ fi
+ if [ -z "$GH_AW_BIN" ] && [ -n "${GH_CONFIG_DIR:-}" ]; then
+ GH_AW_BIN=$(find "${GH_CONFIG_DIR}/extensions/gh-aw" -name 'gh-aw' -type f 2>/dev/null | head -1) || true
+ fi
+ if [ -z "$GH_AW_BIN" ] && [ -f "${GITHUB_WORKSPACE}/gh-aw" ]; then
+ GH_AW_BIN="${GITHUB_WORKSPACE}/gh-aw"
+ fi
+ if [ -n "$GH_AW_BIN" ] && [ -f "$GH_AW_BIN" ]; then
+ cp "$GH_AW_BIN" "${RUNNER_TEMP}/gh-aw/gh-aw"
+ chmod +x "${RUNNER_TEMP}/gh-aw/gh-aw"
+ echo "Copied gh-aw binary to ${RUNNER_TEMP}/gh-aw/gh-aw"
+ else
+ echo "::error::Failed to find gh-aw binary for MCP server"
+ exit 1
+ fi
+ - name: Generate Safe Outputs Config
+ run: |
+ mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs"
+ mkdir -p /tmp/gh-aw/safeoutputs
+ mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs
+ cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_6ae332153deded2a_EOF'
+ {"create_discussion":{"category":"audits","close_older_discussions":true,"expires":168,"fallback_to_issue":true,"max":1,"title_prefix":"[evals] "},"create_issue":{"close_older_issues":true,"expires":168,"labels":["evals","automated-analysis"],"max":1,"title_prefix":"[evals] "},"create_report_incomplete_issue":{},"max_bot_mentions":1,"mentions":{"enabled":false},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}}
+ GH_AW_SAFE_OUTPUTS_CONFIG_6ae332153deded2a_EOF
+ - name: Generate Safe Outputs Tools
+ env:
+ GH_AW_TOOLS_META_JSON: |
+ {
+ "description_suffixes": {
+ "create_discussion": " CONSTRAINTS: Maximum 1 discussion(s) can be created. Title will be prefixed with \"[evals] \". Discussions will be created in category \"audits\".",
+ "create_issue": " CONSTRAINTS: Maximum 1 issue(s) can be created. Title will be prefixed with \"[evals] \". Labels [\"evals\" \"automated-analysis\"] will be automatically added."
+ },
+ "repo_params": {},
+ "dynamic_tools": []
+ }
+ GH_AW_VALIDATION_JSON: |
+ {
+ "create_discussion": {
+ "defaultMax": 1,
+ "fields": {
+ "body": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 65000,
+ "minLength": 64
+ },
+ "category": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 128
+ },
+ "repo": {
+ "type": "string",
+ "maxLength": 256
+ },
+ "title": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 128
+ }
+ }
+ },
+ "create_issue": {
+ "defaultMax": 1,
+ "fields": {
+ "body": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 65000,
+ "minLength": 20
+ },
+ "fields": {
+ "type": "array"
+ },
+ "labels": {
+ "type": "array",
+ "itemType": "string",
+ "itemSanitize": true,
+ "itemMaxLength": 128
+ },
+ "parent": {
+ "issueOrPRNumber": true
+ },
+ "repo": {
+ "type": "string",
+ "maxLength": 256
+ },
+ "temporary_id": {
+ "type": "string"
+ },
+ "title": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 128
+ }
+ }
+ },
+ "mentions": {
+ "enabled": false
+ },
+ "missing_data": {
+ "defaultMax": 20,
+ "fields": {
+ "alternatives": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 256
+ },
+ "context": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 256
+ },
+ "data_type": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 128
+ },
+ "reason": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 256
+ }
+ }
+ },
+ "missing_tool": {
+ "defaultMax": 20,
+ "fields": {
+ "alternatives": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 512
+ },
+ "reason": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 256
+ },
+ "tool": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 128
+ }
+ }
+ },
+ "noop": {
+ "defaultMax": 1,
+ "fields": {
+ "message": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 65000
+ }
+ }
+ },
+ "report_incomplete": {
+ "defaultMax": 5,
+ "fields": {
+ "details": {
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 65000
+ },
+ "reason": {
+ "required": true,
+ "type": "string",
+ "sanitize": true,
+ "maxLength": 1024
+ }
+ }
+ }
+ }
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs');
+ await main();
+ - name: Start MCP Gateway
+ id: start-mcp-gateway
+ env:
+ CODEX_HOME: /tmp/gh-aw/mcp-config
+ GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }}
+ GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
+ GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }}
+ GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ set -eo pipefail
+ mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config"
+
+ # Export gateway environment variables for MCP config and gateway script
+ export MCP_GATEWAY_PORT="8080"
+ export MCP_GATEWAY_DOMAIN="awmg-mcpg"
+ export MCP_GATEWAY_HOST_DOMAIN="localhost"
+ MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=')
+ echo "::add-mask::${MCP_GATEWAY_API_KEY}"
+ export MCP_GATEWAY_API_KEY
+ export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads"
+ mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}"
+ export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288"
+ export DEBUG="*"
+
+ export GH_AW_ENGINE="codex"
+ export GH_AW_MCP_CLI_SERVERS='["agenticworkflows","safeoutputs"]'
+ MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0')
+ MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0')
+ source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh"
+ export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -e CODEX_HOME -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.1'
+
+ cat > "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" << GH_AW_MCP_CONFIG_093f809447f81658_EOF
+ [history]
+ persistence = "none"
+
+ [shell_environment_policy]
+ inherit = "core"
+ include_only = ["^CODEX_API_KEY$", "^GH_AW_ASSETS_ALLOWED_EXTS$", "^GH_AW_ASSETS_BRANCH$", "^GH_AW_ASSETS_MAX_SIZE_KB$", "^GH_AW_SAFE_OUTPUTS$", "^GITHUB_REPOSITORY$", "^GITHUB_SERVER_URL$", "^GITHUB_TOKEN$", "^HOME$", "^OPENAI_API_KEY$", "^PATH$"]
+
+ [mcp_servers.agenticworkflows]
+ container = "localhost/gh-aw:dev"
+ mounts = ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"]
+ env_vars = ["DEBUG", "GH_TOKEN", "GITHUB_TOKEN", "GITHUB_ACTOR", "GITHUB_REPOSITORY"]
+
+ [mcp_servers.agenticworkflows."guard-policies"]
+
+ [mcp_servers.agenticworkflows."guard-policies".write-sink]
+ accept = ["*"]
+
+ [mcp_servers.safeoutputs]
+ container = "ghcr.io/github/gh-aw-node"
+ mounts = ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "${RUNNER_TEMP}/gh-aw/safeoutputs:${RUNNER_TEMP}/gh-aw/safeoutputs:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"]
+ args = ["-w", "$GITHUB_WORKSPACE"]
+ entrypoint = "sh"
+ entrypointArgs = ["-c", "sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh"]
+ env_vars = ["DEBUG", "DEFAULT_BRANCH", "GH_AW_ASSETS_ALLOWED_EXTS", "GH_AW_ASSETS_BRANCH", "GH_AW_ASSETS_MAX_SIZE_KB", "GH_AW_MCP_LOG_DIR", "GH_AW_SAFE_OUTPUTS", "GH_AW_SAFE_OUTPUTS_CONFIG_PATH", "GH_AW_SAFE_OUTPUTS_TOOLS_PATH", "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST", "GITHUB_REPOSITORY", "GITHUB_SHA", "GITHUB_TOKEN", "GITHUB_WORKSPACE", "RUNNER_TEMP"]
+
+ [mcp_servers.safeoutputs."guard-policies"]
+
+ [mcp_servers.safeoutputs."guard-policies".write-sink]
+ accept = ["*"]
+ GH_AW_MCP_CONFIG_093f809447f81658_EOF
+
+ # Generate JSON config for MCP gateway
+ GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node)
+ cat << GH_AW_MCP_CONFIG_8eb75040abf9b800_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs"
+ {
+ "mcpServers": {
+ "agenticworkflows": {
+ "container": "localhost/gh-aw:dev",
+ "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"],
+ "args": ["--network", "host", "-w", "\${GITHUB_WORKSPACE}"],
+ "env": {
+ "DEBUG": "*",
+ "GITHUB_TOKEN": "\${GITHUB_TOKEN}",
+ "GITHUB_ACTOR": "\${GITHUB_ACTOR}",
+ "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}"
+ },
+ "guard-policies": {
+ "write-sink": {
+ "accept": [
+ "*"
+ ],
+ "sink-visibility": ${{ toJSON(steps.determine-automatic-lockdown.outputs.visibility) }}
+ }
+ }
+ },
+ "safeoutputs": {
+ "container": "ghcr.io/github/gh-aw-node",
+ "mounts": ["\${GITHUB_WORKSPACE}:\${GITHUB_WORKSPACE}:rw", "${RUNNER_TEMP}/gh-aw/safeoutputs:${RUNNER_TEMP}/gh-aw/safeoutputs:rw", "/tmp/gh-aw:/tmp/gh-aw:rw"],
+ "args": ["-w", "\${GITHUB_WORKSPACE}"],
+ "entrypoint": "sh",
+ "entrypointArgs": ["-c", "sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh"],
+ "env": {
+ "DEBUG": "*",
+ "DEFAULT_BRANCH": "\${DEFAULT_BRANCH}",
+ "GH_AW_ASSETS_ALLOWED_EXTS": "\${GH_AW_ASSETS_ALLOWED_EXTS}",
+ "GH_AW_ASSETS_BRANCH": "\${GH_AW_ASSETS_BRANCH}",
+ "GH_AW_ASSETS_MAX_SIZE_KB": "\${GH_AW_ASSETS_MAX_SIZE_KB}",
+ "GH_AW_MCP_LOG_DIR": "\${GH_AW_MCP_LOG_DIR}",
+ "GH_AW_SAFE_OUTPUTS": "\${GH_AW_SAFE_OUTPUTS}",
+ "GH_AW_SAFE_OUTPUTS_CONFIG_PATH": "\${GH_AW_SAFE_OUTPUTS_CONFIG_PATH}",
+ "GH_AW_SAFE_OUTPUTS_TOOLS_PATH": "\${GH_AW_SAFE_OUTPUTS_TOOLS_PATH}",
+ "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST": "\${GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST}",
+ "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}",
+ "GITHUB_SHA": "\${GITHUB_SHA}",
+ "GITHUB_TOKEN": "\${GITHUB_TOKEN}",
+ "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}",
+ "RUNNER_TEMP": "\${RUNNER_TEMP}"
+ },
+ "guard-policies": {
+ "write-sink": {
+ "accept": [
+ "*"
+ ],
+ "sink-visibility": ${{ toJSON(steps.determine-automatic-lockdown.outputs.visibility) }}
+ }
+ }
+ }
+ },
+ "gateway": {
+ "port": $MCP_GATEWAY_PORT,
+ "domain": "${MCP_GATEWAY_DOMAIN}",
+ "apiKey": "${MCP_GATEWAY_API_KEY}",
+ "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}",
+ "opentelemetry": {
+ "endpoint": "${OTEL_EXPORTER_OTLP_ENDPOINT}",
+ "traceId": "${GITHUB_AW_OTEL_TRACE_ID}",
+ "spanId": "${GITHUB_AW_OTEL_PARENT_SPAN_ID}"
+ }
+ }
+ }
+ GH_AW_MCP_CONFIG_8eb75040abf9b800_EOF
+
+ # Sync converter output to writable CODEX_HOME for Codex
+ mkdir -p /tmp/gh-aw/mcp-config
+ cat > "/tmp/gh-aw/mcp-config/config.toml" << GH_AW_CODEX_SHELL_POLICY_c096be1fb1ba2c43_EOF
+
+ model_provider = "openai-proxy"
+
+ [model_providers.openai-proxy]
+ name = "OpenAI AWF proxy"
+ base_url = "http://172.30.0.30:10000"
+ env_key = "OPENAI_API_KEY"
+ supports_websockets = false
+ [shell_environment_policy]
+ inherit = "core"
+ include_only = ["^CODEX_API_KEY$", "^GH_AW_ASSETS_ALLOWED_EXTS$", "^GH_AW_ASSETS_BRANCH$", "^GH_AW_ASSETS_MAX_SIZE_KB$", "^GH_AW_SAFE_OUTPUTS$", "^GITHUB_REPOSITORY$", "^GITHUB_SERVER_URL$", "^GITHUB_TOKEN$", "^HOME$", "^OPENAI_API_KEY$", "^PATH$"]
+ GH_AW_CODEX_SHELL_POLICY_c096be1fb1ba2c43_EOF
+ awk '
+ BEGIN { skip_openai_proxy = 0 }
+ /^[[:space:]]*model_provider[[:space:]]*=/ { next }
+ /^\[model_providers\.openai-proxy\][[:space:]]*$/ { skip_openai_proxy = 1; next }
+ /^\[/ { skip_openai_proxy = 0 }
+ !skip_openai_proxy { print }
+ ' "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" >> "/tmp/gh-aw/mcp-config/config.toml"
+ chmod 600 "/tmp/gh-aw/mcp-config/config.toml"
+ mkdir -p "${CODEX_HOME}"
+ if [ "/tmp/gh-aw/mcp-config/config.toml" != "${CODEX_HOME}/config.toml" ]; then cp "/tmp/gh-aw/mcp-config/config.toml" "${CODEX_HOME}/config.toml"; fi
+ chmod 600 "${CODEX_HOME}/config.toml"
+ - name: Mount MCP servers as CLIs
+ id: mount-mcp-clis
+ continue-on-error: true
+ env:
+ MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }}
+ MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }}
+ MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }}
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs');
+ await main();
+ - name: Clean credentials
+ continue-on-error: true
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/clean_git_credentials.sh"
+ - name: Audit pre-agent workspace
+ id: pre_agent_audit
+ continue-on-error: true
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/audit_pre_agent_workspace.sh"
+ - name: Start CLI Proxy
+ env:
+ GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ GITHUB_SERVER_URL: ${{ github.server_url }}
+ GITHUB_API_URL: ${{ github.api_url }}
+ GH_HOST: ${{ env.GH_HOST }}
+ GITHUB_HOST: ${{ env.GITHUB_HOST }}
+ GITHUB_ENTERPRISE_HOST: ${{ env.GITHUB_ENTERPRISE_HOST }}
+ GITHUB_GRAPHQL_URL: ${{ env.GITHUB_GRAPHQL_URL }}
+ GITHUB_COPILOT_BASE_URL: ${{ env.GITHUB_COPILOT_BASE_URL }}
+ GH_AW_NETWORK_ISOLATION: 'true'
+ CLI_PROXY_POLICY: '{"allow-only":{"repos":"${{ steps.determine-automatic-lockdown.outputs.repos }}","min-integrity":"${{ steps.determine-automatic-lockdown.outputs.min_integrity }}"}}'
+ CLI_PROXY_IMAGE: 'ghcr.io/github/gh-aw-mcpg:v0.4.1'
+ run: |
+ bash "${RUNNER_TEMP}/gh-aw/actions/start_cli_proxy.sh"
+ - name: Execute Codex CLI
+ id: agentic_execution
+ run: |
+ set -o pipefail
+ printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt
+ mkdir -p "$CODEX_HOME/logs" && touch /tmp/gh-aw/agent-step-summary.md
+ (umask 177 && touch /tmp/gh-aw/agent-stdio.log)
+ GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }}"
+ printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.31/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.grafana.net\",\"*.sentry.io\",\"172.30.0.1\",\"api.github.com\",\"api.openai.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"chatgpt.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"openai.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"],\"isolation\":true,\"topologyAttach\":[\"awmg-mcpg\",\"awmg-cli-proxy\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.31,squid=sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494,agent=sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1,agent-act=sha256:58fee05c1c54ba5ca1e7056b3aaea30281841d5899093002e2c650710c50540f,api-proxy=sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63,cli-proxy=sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
+ cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json
+ export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json"
+ GH_AW_DOCKER_HOST=""
+ if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
+ GH_AW_DOCKER_HOST="${DOCKER_HOST}"
+ fi
+ if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
+ GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs"
+ fi
+ GH_AW_TOOL_CACHE_MOUNT=""
+ GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"
+ if [ -d "$GH_AW_TOOL_CACHE" ]; then
+ if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then
+ GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro"
+ fi
+ fi
+ # shellcheck disable=SC1003,SC2016,SC2086
+ awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env CODEX_API_KEY --exclude-env GH_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --exclude-env OPENAI_API_KEY --log-level info --skip-pull --difc-proxy-host awmg-cli-proxy:18443 --difc-proxy-ca-cert /tmp/gh-aw/difc-proxy-tls/ca.crt \
+ -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner โ check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/codex_harness.cjs codex exec${GH_AW_MODEL_AGENT_CODEX:+ --model "$GH_AW_MODEL_AGENT_CODEX"} -c web_search="disabled" -c fetch="disabled" --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log
+ env:
+ CODEX_API_KEY: ${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}
+ CODEX_HOME: /tmp/gh-aw/mcp-config
+ GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }}
+ GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml
+ GH_AW_MODEL_AGENT_CODEX: gpt-5.4
+ GH_AW_PHASE: agent
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
+ GH_AW_VERSION: dev
+ GH_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN || github.token }}
+ GITHUB_AW: true
+ GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md
+ GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com
+ GIT_AUTHOR_NAME: github-actions[bot]
+ GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com
+ GIT_COMMITTER_NAME: github-actions[bot]
+ OPENAI_API_KEY: ${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}
+ RUNNER_TEMP: ${{ runner.temp }}
+ RUST_LOG: ${{ runner.debug == 1 && 'trace,hyper_util=info,mio=info,reqwest=info,os_info=info,codex_otel=warn,codex_core=debug,ocodex_exec=debug' || 'warn' }}
+ TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }}
+ - name: Stop CLI Proxy
+ if: always()
+ continue-on-error: true
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/stop_cli_proxy.sh"
+ - name: Detect agent errors
+ if: always()
+ id: detect-agent-errors
+ continue-on-error: true
+ run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs"
+ - name: Configure Git credentials
+ env:
+ GITHUB_REPOSITORY: ${{ github.repository }}
+ GITHUB_SERVER_URL: ${{ github.server_url }}
+ GITHUB_TOKEN: ${{ github.token }}
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh"
+ - name: Stop MCP Gateway
+ if: always()
+ continue-on-error: true
+ env:
+ MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }}
+ MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }}
+ GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }}
+ run: |
+ bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID"
+ - name: Redact secrets in logs
+ if: always()
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs');
+ await main();
+ env:
+ GH_AW_SECRET_NAMES: 'CODEX_API_KEY,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN,OPENAI_API_KEY'
+ SECRET_CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
+ SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}
+ SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}
+ SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ SECRET_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ - name: Append agent step summary
+ if: always()
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/append_agent_step_summary.sh"
+ - name: Copy Safe Outputs
+ if: always()
+ env:
+ GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
+ run: |
+ mkdir -p /tmp/gh-aw
+ cp "$GH_AW_SAFE_OUTPUTS" /tmp/gh-aw/safeoutputs.jsonl 2>/dev/null || true
+ - name: Ingest agent output
+ id: collect_output
+ if: always()
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
+ GH_AW_ALLOWED_DOMAINS: "*.grafana.net,*.sentry.io,172.30.0.1,api.github.com,api.openai.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,chatgpt.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,openai.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com"
+ GH_AW_ALLOWED_GITHUB_REFS: ""
+ GITHUB_SERVER_URL: ${{ github.server_url }}
+ GITHUB_API_URL: ${{ github.api_url }}
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs');
+ await main();
+ - name: Parse agent logs for step summary
+ if: always()
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: /tmp/gh-aw/agent-stdio.log
+ GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_codex_log.cjs');
+ await main();
+ - name: Parse MCP Gateway logs for step summary
+ if: always()
+ id: parse-mcp-gateway
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs');
+ await main();
+ - name: Print firewall logs
+ if: always()
+ continue-on-error: true
+ env:
+ AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/print_firewall_logs.sh" --rootless
+ - name: Parse token usage for step summary
+ if: always()
+ continue-on-error: true
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs');
+ await main();
+ - name: Print AWF reflect summary
+ if: always()
+ continue-on-error: true
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs');
+ await main();
+ - name: Generate observability summary
+ if: always()
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_observability_summary.cjs');
+ await main(core);
+ - name: Write agent output placeholder if missing
+ if: always()
+ run: |
+ if [ ! -f /tmp/gh-aw/agent_output.json ]; then
+ echo '{"items":[]}' > /tmp/gh-aw/agent_output.json
+ fi
+ - name: Upload agent artifacts
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: agent
+ path: |
+ /tmp/gh-aw/aw-prompts/prompt.txt
+ /tmp/gh-aw/mcp-config/logs/
+ /tmp/gh-aw/redacted-urls.log
+ /tmp/gh-aw/mcp-logs/
+ /tmp/gh-aw/agent_usage.json
+ /tmp/gh-aw/agent-stdio.log
+ /tmp/gh-aw/pre-agent-audit.txt
+ /tmp/gh-aw/agent/
+ /tmp/gh-aw/github_rate_limits.jsonl
+ /tmp/gh-aw/otel.jsonl
+ /tmp/gh-aw/otlp-export-errors.jsonl
+ /tmp/gh-aw/safeoutputs.jsonl
+ /tmp/gh-aw/agent_output.json
+ /tmp/gh-aw/aw-*.patch
+ /tmp/gh-aw/aw-*.bundle
+ /tmp/gh-aw/awf-config.json
+ /tmp/gh-aw/sandbox/firewall/logs/
+ /tmp/gh-aw/sandbox/firewall/audit/
+ /tmp/gh-aw/sandbox/firewall/awf-reflect.json
+ if-no-files-found: ignore
+
+ conclusion:
+ needs:
+ - activation
+ - agent
+ - detection
+ - safe_outputs
+ if: >
+ always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' ||
+ needs.activation.outputs.oauth_token_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true' ||
+ needs.activation.outputs.secret_verification_result == 'failed' || needs.activation.outputs.daily_ai_credits_exceeded == 'true')
+ runs-on: ubuntu-slim
+ permissions:
+ contents: read
+ discussions: write
+ issues: write
+ concurrency:
+ group: "gh-aw-conclusion-daily-evals-report"
+ cancel-in-progress: false
+ queue: max
+ env:
+ GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
+ outputs:
+ incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }}
+ noop_message: ${{ steps.noop.outputs.noop_message }}
+ tools_reported: ${{ steps.missing_tool.outputs.tools_reported }}
+ total_count: ${{ steps.missing_tool.outputs.total_count }}
+ steps:
+ - name: Checkout actions folder
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: github/gh-aw
+ sparse-checkout: |
+ actions
+ clean: false
+ persist-credentials: false
+ - name: Setup Scripts
+ id: setup
+ uses: ./actions/setup
+ with:
+ destination: ${{ runner.temp }}/gh-aw/actions
+ job-name: ${{ github.job }}
+ trace-id: ${{ needs.activation.outputs.setup-trace-id }}
+ parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
+ env:
+ GH_AW_SETUP_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-evals-report.lock.yml@${{ github.ref }}
+ GH_AW_INFO_VERSION: "0.144.4"
+ GH_AW_INFO_AWF_VERSION: "v0.27.31"
+ GH_AW_INFO_ENGINE_ID: "codex"
+ - name: Download agent output artifact
+ id: download-agent-output
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: agent
+ path: /tmp/gh-aw/
+ - name: Setup agent output environment variable
+ id: setup-agent-output-env
+ if: steps.download-agent-output.outcome == 'success'
+ run: |
+ mkdir -p /tmp/gh-aw/
+ find "/tmp/gh-aw/" -type f -print
+ echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
+ - name: Download safe outputs items manifest
+ id: download-safe-outputs-manifest
+ if: always()
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: safe-outputs-items
+ path: /tmp/gh-aw/
+ - name: Collect usage artifact files
+ if: always()
+ continue-on-error: true
+ run: |
+ mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection
+ echo "Usage artifact source file status:"
+ for file in /tmp/gh-aw/aw_info.json /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.json /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/evals/evals.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do
+ [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file"
+ done
+ [ -f /tmp/gh-aw/aw_info.json ] && cp /tmp/gh-aw/aw_info.json /tmp/gh-aw/usage/aw_info.json || true
+ [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true
+ [ -f /tmp/gh-aw/agent_usage.json ] && cp /tmp/gh-aw/agent_usage.json /tmp/gh-aw/usage/agent_usage.json || true
+ [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true
+ [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true
+ [ -f /tmp/gh-aw/evals/evals.jsonl ] && cp /tmp/gh-aw/evals/evals.jsonl /tmp/gh-aw/usage/evals.jsonl || true
+ [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true
+ [ -s /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true
+ [ -s /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true
+ [ -s /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true
+ [ -s /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true
+ [ -s /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true
+ [ -s /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true
+ [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl
+ [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl
+ mkdir -p /tmp/gh-aw/usage/activity
+ node "${RUNNER_TEMP}/gh-aw/actions/generate_usage_activity_summary.cjs"
+ find /tmp/gh-aw/usage -type f -print | sort
+ - name: Upload usage artifact
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: usage
+ path: |
+ /tmp/gh-aw/usage/aw_info.json
+ /tmp/gh-aw/usage/aw-info.jsonl
+ /tmp/gh-aw/usage/agent_usage.json
+ /tmp/gh-aw/usage/agent_usage.jsonl
+ /tmp/gh-aw/usage/detection_usage.jsonl
+ /tmp/gh-aw/usage/evals.jsonl
+ /tmp/gh-aw/usage/github_rate_limits.jsonl
+ /tmp/gh-aw/usage/agent/token_usage.jsonl
+ /tmp/gh-aw/usage/detection/token_usage.jsonl
+ /tmp/gh-aw/usage/activity/summary.json
+ if-no-files-found: ignore
+ - name: Restore daily AIC usage cache
+ id: restore-daily-aic-cache-conclusion
+ if: always()
+ continue-on-error: true
+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ key: agentic-workflow-usage-dailyevalsreport-${{ github.run_id }}
+ restore-keys: agentic-workflow-usage-dailyevalsreport-
+ path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
+ - name: Write daily AIC usage cache entry
+ id: write-daily-aic-cache
+ if: always()
+ continue-on-error: true
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ github-token: ${{ github.token }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/write_daily_aic_usage_cache.cjs');
+ await main();
+ - name: Save daily AIC usage cache
+ id: save-daily-aic-cache
+ if: always()
+ continue-on-error: true
+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ key: agentic-workflow-usage-dailyevalsreport-${{ github.run_id }}
+ path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
+ - name: Upload daily AIC usage cache artifact
+ id: upload-daily-aic-cache
+ if: always()
+ continue-on-error: true
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: aic-usage-cache
+ path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl
+ if-no-files-found: ignore
+ retention-days: 7
+ - name: Process no-op messages
+ id: noop
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
+ GH_AW_NOOP_MAX: "1"
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-evals-report.md"
+ GH_AW_TRACKER_ID: "daily-evals-report"
+ GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
+ GH_AW_NOOP_REPORT_AS_ISSUE: "true"
+ GH_AW_AIC: ${{ needs.agent.outputs.aic }}
+ GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}
+ GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }}
+ GH_AW_WORKFLOW_ID: "daily-evals-report"
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs');
+ await main();
+ - name: Log detection run
+ id: detection_runs
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-evals-report.md"
+ GH_AW_TRACKER_ID: "daily-evals-report"
+ GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }}
+ GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }}
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs');
+ await main();
+ - name: Record missing tool
+ id: missing_tool
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
+ GH_AW_MISSING_TOOL_CREATE_ISSUE: "true"
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-evals-report.md"
+ GH_AW_TRACKER_ID: "daily-evals-report"
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs');
+ await main();
+ - name: Record incomplete
+ id: report_incomplete
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
+ GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true"
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-evals-report.md"
+ GH_AW_TRACKER_ID: "daily-evals-report"
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs');
+ await main();
+ - name: Handle agent failure
+ id: handle_agent_failure
+ if: always()
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-evals-report.md"
+ GH_AW_TRACKER_ID: "daily-evals-report"
+ GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }}
+ GH_AW_WORKFLOW_ID: "daily-evals-report"
+ GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "12"
+ GH_AW_ENGINE_ID: "codex"
+ GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }}
+ GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }}
+ GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }}
+ GH_AW_AI_CREDITS_RATE_LIMIT_ERROR: ${{ needs.agent.outputs.ai_credits_rate_limit_error || 'false' }}
+ GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }}
+ GH_AW_AIC: ${{ needs.agent.outputs.aic }}
+ GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}
+ GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }}
+ GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }}
+ GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }}
+ GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }}
+ GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }}
+ GH_AW_HTTP_400_RESPONSE_ERROR: ${{ needs.agent.outputs.http_400_response_error }}
+ GH_AW_ENGINE_API_HOSTS: "api.openai.com"
+ GH_AW_CREATE_DISCUSSION_ERRORS: ${{ needs.safe_outputs.outputs.create_discussion_errors }}
+ GH_AW_CREATE_DISCUSSION_ERROR_COUNT: ${{ needs.safe_outputs.outputs.create_discussion_error_count }}
+ GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }}
+ GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }}
+ GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }}
+ GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }}
+ GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }}
+ GH_AW_DAILY_AI_CREDITS_THRESHOLD: ${{ needs.activation.outputs.daily_ai_credits_threshold }}
+ GH_AW_GROUP_REPORTS: "false"
+ GH_AW_FAILURE_REPORT_AS_ISSUE: "true"
+ GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true"
+ GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true"
+ GH_AW_TIMEOUT_MINUTES: "45"
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs');
+ await main();
+
+ detection:
+ needs:
+ - activation
+ - agent
+ if: always() && needs.agent.result != 'skipped'
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ env:
+ GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
+ outputs:
+ aic: ${{ steps.parse_detection_token_usage.outputs.aic }}
+ detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }}
+ detection_reason: ${{ steps.detection_conclusion.outputs.reason }}
+ detection_success: ${{ steps.detection_conclusion.outputs.success }}
+ steps:
+ - name: Checkout actions folder
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: github/gh-aw
+ sparse-checkout: |
+ actions
+ clean: false
+ persist-credentials: false
+ - name: Setup Scripts
+ id: setup
+ uses: ./actions/setup
+ with:
+ destination: ${{ runner.temp }}/gh-aw/actions
+ job-name: ${{ github.job }}
+ trace-id: ${{ needs.activation.outputs.setup-trace-id }}
+ parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
+ env:
+ GH_AW_SETUP_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-evals-report.lock.yml@${{ github.ref }}
+ GH_AW_INFO_VERSION: "0.144.4"
+ GH_AW_INFO_AWF_VERSION: "v0.27.31"
+ GH_AW_INFO_ENGINE_ID: "codex"
+ - name: Download agent output artifact
+ id: download-agent-output
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: agent
+ path: /tmp/gh-aw/
+ - name: Setup agent output environment variable
+ id: setup-agent-output-env
+ if: steps.download-agent-output.outcome == 'success'
+ run: |
+ mkdir -p /tmp/gh-aw/
+ find "/tmp/gh-aw/" -type f -print
+ echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
+ - name: Checkout repository for patch context
+ if: needs.agent.outputs.has_patch == 'true'
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+ # --- Threat Detection ---
+ - name: Clean stale firewall files from agent artifact
+ run: |
+ rm -rf /tmp/gh-aw/sandbox/firewall/logs
+ rm -rf /tmp/gh-aw/sandbox/firewall/audit
+ - name: Download container images
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.31@sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1 ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31@sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63 ghcr.io/github/gh-aw-firewall/squid:0.27.31@sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494
+ - name: Check if detection needed
+ id: detection_guard
+ if: always()
+ env:
+ OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }}
+ HAS_PATCH: ${{ needs.agent.outputs.has_patch }}
+ run: |
+ if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then
+ echo "run_detection=true" >> "$GITHUB_OUTPUT"
+ echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH"
+ else
+ echo "run_detection=false" >> "$GITHUB_OUTPUT"
+ echo "Detection skipped: no agent outputs or patches to analyze"
+ fi
+ - name: Clear MCP Config for detection
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ run: |
+ rm -f "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json"
+ rm -f "$HOME/.copilot/mcp-config.json"
+ rm -f "$GITHUB_WORKSPACE/.gemini/settings.json"
+ - name: Prepare threat detection files
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ run: |
+ mkdir -p /tmp/gh-aw/threat-detection/aw-prompts
+ rm -f /tmp/gh-aw/agent_usage.json
+ cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true
+ if [ ! -s /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt ]; then
+ echo "::warning::ERR_VALIDATION: Missing or empty detection context prompt at /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt. Ensure the agent artifact includes /tmp/gh-aw/aw-prompts/prompt.txt. Detection will continue with fallback workflow context."
+ fi
+ cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true
+ for f in /tmp/gh-aw/aw-*.patch; do
+ [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true
+ done
+ for f in /tmp/gh-aw/aw-*.bundle; do
+ [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true
+ done
+ echo "Prepared threat detection files:"
+ ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true
+ - name: Setup threat detection
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ WORKFLOW_NAME: "Daily Evals Feature Report"
+ WORKFLOW_DESCRIPTION: "Daily data science report analyzing evals feature adoption, per-question pass rates, and quality trends across agentic workflows"
+ HAS_PATCH: ${{ needs.agent.outputs.has_patch }}
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs');
+ await main();
+ - name: Ensure threat-detection directory and log
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ run: |
+ mkdir -p /tmp/gh-aw/threat-detection
+ touch /tmp/gh-aw/threat-detection/detection.log
+ - name: Install AWF binary
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.31
+ - name: Setup Node.js
+ uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+ with:
+ node-version: '24'
+ package-manager-cache: false
+ - name: Install Codex CLI
+ run: npm install --ignore-scripts -g @openai/codex@0.144.4
+ - name: Prepare Codex config for threat-detect
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ run: |
+ mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" "/tmp/gh-aw/mcp-config" "/tmp/gh-aw/mcp-config/logs/"
+ printf '%s\n' "{\"mcpServers\":{}}" > "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json"
+ # Point Codex at the AWF OpenAI proxy and disable websocket startup.
+ cat > "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" << GH_AW_CODEX_DETECTION_CONFIG_6ed41afb06de5aa1_EOF
+ [history]
+ persistence = "none"
+
+ model_provider = "openai-proxy"
+
+ [model_providers.openai-proxy]
+ name = "OpenAI AWF proxy"
+ base_url = "http://172.30.0.30:10000"
+ api_base = "http://172.30.0.30:10000"
+ wss_base = "ws://172.30.0.30:10000"
+ env_key = "OPENAI_API_KEY"
+ supports_websockets = false
+ GH_AW_CODEX_DETECTION_CONFIG_6ed41afb06de5aa1_EOF
+ cp "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" "/tmp/gh-aw/mcp-config/config.toml"
+ chmod 600 "${RUNNER_TEMP}/gh-aw/mcp-config/config.toml" "/tmp/gh-aw/mcp-config/config.toml"
+ - name: Install threat-detect binary
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ run: |
+ bash "${RUNNER_TEMP}/gh-aw/actions/install_threat_detect_binary.sh" v0.2.2
+ - name: Execute threat detection with AWF
+ id: detection_agentic_execution
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ continue-on-error: true
+ run: |
+ set -o pipefail
+ printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt
+ (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log)
+ GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }}"
+ printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.31/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"172.30.0.1\",\"api.github.com\",\"api.openai.com\",\"chatgpt.com\",\"github.com\",\"host.docker.internal\",\"openai.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5},\"container\":{\"imageTag\":\"0.27.31,squid=sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494,agent=sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1,agent-act=sha256:58fee05c1c54ba5ca1e7056b3aaea30281841d5899093002e2c650710c50540f,api-proxy=sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63,cli-proxy=sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
+ cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json
+ export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json"
+ GH_AW_DOCKER_HOST=""
+ if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
+ GH_AW_DOCKER_HOST="${DOCKER_HOST}"
+ fi
+ if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then
+ _GH_AW_CHROOT_JSON=$(jq -c --arg src "${RUNNER_TEMP}/gh-aw" --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home "${RUNNER_TEMP}/gh-aw/home" '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; }
+ printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
+ printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json"
+ fi
+ GH_AW_TOOL_CACHE_MOUNT=""
+ GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"
+ if [ -d "$GH_AW_TOOL_CACHE" ]; then
+ if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then
+ GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro"
+ fi
+ fi
+ # shellcheck disable=SC1003,SC2016,SC2086
+ sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env CODEX_API_KEY --exclude-env OPENAI_API_KEY --mount /tmp/gh-aw/threat-detection:/tmp/gh-aw/threat-detection:rw --log-level info --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \
+ -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && threat-detect --engine codex --output /tmp/gh-aw/threat-detection/detection_result.json /tmp/gh-aw/threat-detection' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log
+ env:
+ CODEX_API_KEY: ${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}
+ CODEX_HOME: /tmp/gh-aw/mcp-config
+ GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }}
+ GH_AW_MCP_CONFIG: ${{ runner.temp }}/gh-aw/mcp-config/config.toml
+ GH_AW_MODEL_DETECTION_CODEX: ${{ vars.GH_AW_MODEL_DETECTION_CODEX || vars.GH_AW_DEFAULT_MODEL_CODEX || 'gpt-5.4' }}
+ GH_AW_PHASE: detection
+ GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt
+ GH_AW_VERSION: dev
+ GITHUB_AW: true
+ GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md
+ GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com
+ GIT_AUTHOR_NAME: github-actions[bot]
+ GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com
+ GIT_COMMITTER_NAME: github-actions[bot]
+ OPENAI_API_KEY: ${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}
+ RUNNER_TEMP: ${{ runner.temp }}
+ RUST_LOG: ${{ runner.debug == 1 && 'trace,hyper_util=info,mio=info,reqwest=info,os_info=info,codex_otel=warn,codex_core=debug,ocodex_exec=debug' || 'warn' }}
+ TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }}
+ - name: Upload threat detection artifact
+ if: always() && steps.detection_guard.outputs.run_detection == 'true'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: detection
+ path: |
+ /tmp/gh-aw/threat-detection/detection_result.json
+ /tmp/gh-aw/threat-detection/detection.log
+ if-no-files-found: ignore
+ - name: Parse threat detection token usage for step summary
+ id: parse_detection_token_usage
+ if: always()
+ continue-on-error: true
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage
+ with:
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs');
+ await main();
+ - name: Conclude threat detection
+ id: detection_conclusion
+ if: always()
+ continue-on-error: true
+ env:
+ RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }}
+ DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }}
+ GH_AW_DETECTION_CONTINUE_ON_ERROR: "true"
+ run: |
+ bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json
+
+ safe_outputs:
+ needs:
+ - activation
+ - agent
+ - detection
+ if: (!cancelled()) && needs.agent.result != 'skipped' && needs.detection.result == 'success'
+ runs-on: ubuntu-slim
+ permissions:
+ contents: read
+ discussions: write
+ issues: write
+ timeout-minutes: 45
+ env:
+ GH_AW_AGENT_AIC: ${{ needs.agent.outputs.aic }}
+ GH_AW_AIC: ${{ needs.agent.outputs.aic }}
+ GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }}
+ GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/daily-evals-report"
+ GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }}
+ GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }}
+ GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }}
+ GH_AW_ENGINE_ID: "codex"
+ GH_AW_ENGINE_MODEL: "gpt-5.4"
+ GH_AW_PROJECT_UTC: "-08:00"
+ GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }}
+ GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }}
+ GH_AW_TRACKER_ID: "daily-evals-report"
+ GH_AW_WORKFLOW_EMOJI: "๐งช"
+ GH_AW_WORKFLOW_ID: "daily-evals-report"
+ GH_AW_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/daily-evals-report.md"
+ outputs:
+ code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }}
+ code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }}
+ create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }}
+ create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }}
+ created_issue_number: ${{ steps.process_safe_outputs.outputs.created_issue_number }}
+ created_issue_url: ${{ steps.process_safe_outputs.outputs.created_issue_url }}
+ process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }}
+ process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }}
+ steps:
+ - name: Checkout actions folder
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: github/gh-aw
+ sparse-checkout: |
+ actions
+ clean: false
+ persist-credentials: false
+ - name: Setup Scripts
+ id: setup
+ uses: ./actions/setup
+ with:
+ destination: ${{ runner.temp }}/gh-aw/actions
+ job-name: ${{ github.job }}
+ trace-id: ${{ needs.activation.outputs.setup-trace-id }}
+ parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }}
+ env:
+ GH_AW_SETUP_WORKFLOW_NAME: "Daily Evals Feature Report"
+ GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-evals-report.lock.yml@${{ github.ref }}
+ GH_AW_INFO_VERSION: "0.144.4"
+ GH_AW_INFO_AWF_VERSION: "v0.27.31"
+ GH_AW_INFO_ENGINE_ID: "codex"
+ - name: Mask OTLP telemetry headers
+ run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh"
+ - name: Download agent output artifact
+ id: download-agent-output
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: agent
+ path: /tmp/gh-aw/
+ - name: Setup agent output environment variable
+ id: setup-agent-output-env
+ if: steps.download-agent-output.outcome == 'success'
+ run: |
+ mkdir -p /tmp/gh-aw/
+ find "/tmp/gh-aw/" -type f -print
+ echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT"
+ - name: Configure GH_HOST for enterprise compatibility
+ id: ghes-host-config
+ shell: bash
+ run: | # zizmor: ignore[github-env] - GITHUB_SERVER_URL is set by GitHub Actions, not user input.
+ # Derive GH_HOST from GITHUB_SERVER_URL so the gh CLI targets the correct
+ # GitHub instance (GHES/GHEC). On github.com this is a harmless no-op.
+ GH_HOST="${GITHUB_SERVER_URL#https://}"
+ GH_HOST="${GH_HOST#http://}"
+ echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV"
+ - name: Process Safe Outputs
+ id: process_safe_outputs
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
+ GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }}
+ GH_AW_ALLOWED_DOMAINS: "*.grafana.net,*.sentry.io,172.30.0.1,api.github.com,api.openai.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,chatgpt.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,openai.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com"
+ GITHUB_SERVER_URL: ${{ github.server_url }}
+ GITHUB_API_URL: ${{ github.api_url }}
+ GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"category\":\"audits\",\"close_older_discussions\":true,\"expires\":168,\"fallback_to_issue\":true,\"max\":1,\"title_prefix\":\"[evals] \"},\"create_issue\":{\"close_older_issues\":true,\"expires\":168,\"labels\":[\"evals\",\"automated-analysis\"],\"max\":1,\"title_prefix\":\"[evals] \"},\"create_report_incomplete_issue\":{},\"mentions\":{\"enabled\":false},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}"
+ with:
+ github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
+ script: |
+ const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');
+ setupGlobals(core, github, context, exec, io, getOctokit);
+ const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs');
+ await main();
+ - name: Upload Safe Outputs Items
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: safe-outputs-items
+ path: |
+ /tmp/gh-aw/safe-output-items.jsonl
+ /tmp/gh-aw/temporary-id-map.json
+ if-no-files-found: ignore
diff --git a/.github/workflows/daily-evals-report.md b/.github/workflows/daily-evals-report.md
new file mode 100644
index 00000000000..be9335bdc1b
--- /dev/null
+++ b/.github/workflows/daily-evals-report.md
@@ -0,0 +1,242 @@
+---
+private: true
+emoji: "๐งช"
+description: Daily data science report analyzing evals feature adoption, per-question pass rates, and quality trends across agentic workflows
+on:
+ schedule: daily around 8:00
+ workflow_dispatch:
+permissions:
+ contents: read
+ actions: read
+ issues: read
+ pull-requests: read
+max-daily-ai-credits: 3000
+engine:
+ id: codex
+ model: gpt-5.4
+tracker-id: daily-evals-report
+sandbox:
+ agent:
+ sudo: false
+features:
+ gh-aw-detection: true
+timeout-minutes: 45
+imports:
+ - uses: shared/meta-analysis-base.md
+ with:
+ toolsets: [default, actions]
+ - uses: shared/daily-audit-base.md
+ with:
+ title-prefix: "[evals] "
+ expires: 7d
+safe-outputs:
+ mentions: false
+ allowed-github-references: []
+ max-bot-mentions: 1
+ create-issue:
+ title-prefix: "[evals] "
+ labels: [evals, automated-analysis]
+ close-older-issues: true
+ expires: 7d
+ max: 1
+---
+
+{{#runtime-import? .github/shared-instructions.md}}
+
+# Daily Evals Feature Report
+
+You are a data scientist analyzing the `evals` feature in GitHub Agentic Workflows.
+The `evals` feature lets workflow authors declare BinEval-style binary YES/NO questions
+that are automatically evaluated after each run. Results are stored as `evals.jsonl`
+artifacts per run and persisted to dedicated `evals/` git branches.
+
+## Mission
+
+Analyze the last 7 days of workflow runs that use the `evals` feature. Compute per-question
+YES/NO pass rates, identify workflows with failing evals, and produce a data science summary.
+When the evals feature is degraded or broken, produce an actionable issue to guide investigation.
+
+## Context
+
+- Repository: `${{ github.repository }}`
+- Run ID: `${{ github.run_id }}`
+- Window: last 7 full days ending at workflow start (UTC)
+
+## Phase 1: Fetch Evals Runs
+
+Call the `logs` MCP tool once to download runs with evals artifacts from the past 7 days.
+
+**Tool**: `logs`
+**Parameters**:
+```json
+{
+ "workflow_name": "",
+ "count": 100,
+ "start_date": "-7d",
+ "artifacts": ["evals", "usage"]
+}
+```
+
+Logs are saved to `/tmp/gh-aw/aw-mcp/logs/run-/`.
+
+Do **not** enumerate every workflow first. One broad fetch is the right starting point.
+
+## Phase 2: Discover Evals Data
+
+For each downloaded run directory at `/tmp/gh-aw/aw-mcp/logs/run-/`:
+
+1. Check for `evals/evals.jsonl` (or `evals.jsonl` at root if flattened).
+2. Read `aw_info.json` (or `activation/aw_info.json`) to get:
+ - `workflow_name` (or `workflow.name`)
+ - `conclusion` (success / failure / cancelled)
+ - Whether evals were declared (check for an `evals` key or the presence of the evals job in the run)
+3. Note runs where the workflow declares evals but `evals.jsonl` is absent โ these are **evals job failures**.
+
+Cap analysis at **40 runs total**. Prefer runs with evals results first, then evals failures.
+
+## Phase 3: Parse Evals JSONL
+
+For each `evals.jsonl` found, read every JSONL line. Each record contains:
+- `id` โ question ID
+- `question` โ question text
+- `answer` โ `"YES"` or `"NO"`
+- Optionally: `confidence`, `rationale`, `model`
+
+Extract per-run:
+- Workflow name and run ID
+- Per-question answer list
+- Overall pass/fail: **pass** if every question is YES, **fail** if any question is NO
+
+## Phase 4: Aggregate Statistics
+
+### Per-Workflow
+
+For each unique workflow:
+- `runs_with_evals_results` โ runs that produced `evals.jsonl`
+- `runs_evals_job_failed` โ runs that declared evals but produced no `evals.jsonl`
+- `evals_job_success_rate` = `runs_with_evals_results / (runs_with_evals_results + runs_evals_job_failed) * 100`
+- `run_pass_rate` = runs where all questions are YES / `runs_with_evals_results * 100`
+- Per-question: `yes_rate` = YES count / `runs_with_evals_results * 100`
+- `lowest_scoring_question` โ question ID + text with the lowest `yes_rate`
+
+### Cross-Workflow Summary
+
+- `total_workflows_with_evals` โ unique workflow count
+- `total_runs_analyzed` โ total runs checked
+- `total_evals_results` โ runs with `evals.jsonl`
+- `total_evals_failures` โ runs with evals declared but no results
+- `overall_evals_job_success_rate` = `total_evals_results / (total_evals_results + total_evals_failures) * 100`
+- `overall_yes_rate` โ YES answers / total answers across all runs
+- `most_failing_questions` โ top 3 questions (by NO rate) across all workflows
+
+## Phase 5: Classify Feature Health
+
+| Status | Condition |
+|---|---|
+| **HEALTHY** | `overall_evals_job_success_rate >= 80%` AND `overall_yes_rate >= 60%` |
+| **DEGRADED** | `overall_evals_job_success_rate` between 50โ80%, OR `overall_yes_rate` between 30โ60% |
+| **BROKEN** | `overall_evals_job_success_rate < 50%`, OR zero evals runs found, OR evals job not producing any results |
+
+## Phase 6: Generate Output
+
+### HEALTHY or DEGRADED โ Create Issue with Data Science Summary
+
+Create one GitHub issue.
+
+**Title**: `[evals] Daily Evals Feature Report - YYYY-MM-DD`
+
+**Body structure** (use `###` or lower for all headers):
+
+```markdown
+### Executive Summary
+
+[2โ3 sentences: total workflows, runs analyzed, overall health status, headline finding]
+
+> [!NOTE]
+> Status: HEALTHY / DEGRADED / BROKEN โ [one-line reason]
+
+### Key Metrics
+
+| Metric | Value |
+|---|---|
+| Workflows with evals | N |
+| Runs analyzed | N |
+| Runs with evals results | N |
+| Evals job success rate | X% |
+| Overall YES rate | X% |
+
+### Per-Workflow Pass Rates
+
+| Workflow | Runs | Evals Job Success | Run Pass Rate | Lowest-Scoring Question |
+|---|---|---|---|---|
+| workflow-name | N | X% | X% | "question text" (X% YES) |
+
+
+Per-Question Breakdown per Workflow
+
+For each workflow, show a table:
+
+| Question ID | Question | YES | NO | YES Rate |
+|---|---|---|---|---|
+| builds | Does the generated code compile? | 8 | 2 | 80% |
+
+
+
+### Quality Signals
+
+[Top 3 failing questions with YES rate and affected workflows โ these are regressions or areas needing prompt improvement]
+
+### Recommendations
+
+[2โ3 concrete actions, for example:
+- Which workflow prompts to improve based on low YES rates
+- Whether the evals model is appropriate for the question complexity
+- Any evals job failures to investigate]
+
+### References
+
+[Up to 3 run URLs as `[ยง]()` links]
+```
+
+### BROKEN โ Create Actionable Issue
+
+When the evals feature is BROKEN (job success rate < 50% or zero results), the issue body
+must include actionable investigation steps instead of the data science summary.
+
+> [!CAUTION]
+> Status: BROKEN โ the evals job is not producing results.
+
+#### Investigation Checklist
+
+1. Run `gh aw audit ` on the most recent evals-declaring workflow run to inspect the evals job steps.
+2. Check if the `evals` artifact is present: `gh run download -n evals`.
+3. Look for errors in the evals engine execution step: `Parse BinEval results`.
+4. Verify that the `evals` frontmatter config is valid (each question must have `id` and `question`).
+5. Check if the agentic engine has sufficient credit budget: compare `max-ai-credits` vs actual usage.
+6. Check whether the evals branch `evals/` exists and contains recent commits.
+
+#### Affected Runs
+
+List up to 5 run IDs where evals were declared but produced no results, with direct links.
+
+### No Data โ Noop
+
+If no runs with the evals artifact are found in the 7-day window and no workflows declare evals:
+```
+noop("No evals-enabled workflow runs found in the last 7 days. The evals feature may not yet be widely adopted in this repository.")
+```
+
+If workflows declared evals but zero results were produced:
+```
+noop("Evals declared in N workflows but no evals.jsonl artifacts found. Use [evals-alert] issue.")
+```
+(Skip the noop and create the BROKEN issue instead in this case.)
+
+## Token Budget Guidelines
+
+- **One broad `logs` call** โ do not enumerate workflows individually before calling logs.
+- **Cap at 40 runs** โ stop analyzing once the cap is reached.
+- **Summarize, do not transcribe** โ report statistics, not raw JSONL content.
+- **Stop after `create_issue` or `noop`** โ no extra tool calls after publishing.
+
+Begin your analysis now. Download the logs, analyze evals results, and publish the report.
From 56801b8ff6b535eb2ede9de8ea2dac79f0590dd3 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Wed, 15 Jul 2026 19:05:25 +0000
Subject: [PATCH 2/2] fix: clarify 40-run cap prioritization and date format in
evals report prompt
Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com>
---
.github/workflows/daily-evals-report.lock.yml | 2 +-
.github/workflows/daily-evals-report.md | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/daily-evals-report.lock.yml b/.github/workflows/daily-evals-report.lock.yml
index b33b0dba826..9d9a3746b5b 100644
--- a/.github/workflows/daily-evals-report.lock.yml
+++ b/.github/workflows/daily-evals-report.lock.yml
@@ -1,4 +1,4 @@
-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"87ee75709c6a01a3c0c2fc6649a108cd998eb5249924ebabc26352361e7d4cc6","body_hash":"eb2526b7c3336226685ae44ca18da78791e20e13de2f2ee4495b427493eddfbe","strict":true,"agent_id":"codex","agent_model":"gpt-5.4","engine_versions":{"codex":"0.144.4"}}
+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"87ee75709c6a01a3c0c2fc6649a108cd998eb5249924ebabc26352361e7d4cc6","body_hash":"543bd4d4aae81768fa8e9155240cb07e75d66caae693560145030ee7cff85dce","strict":true,"agent_id":"codex","agent_model":"gpt-5.4","engine_versions":{"codex":"0.144.4"}}
# gh-aw-manifest: {"version":1,"secrets":["CODEX_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN","OPENAI_API_KEY"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"924ae3a1cded613372ab5595356fb5720e22ba16","version":"v6.5.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"f9f3042f7e2789586610d6e8b85c8f03e5195baf","version":"v7.2.0"},{"repo":"docker/setup-buildx-action","sha":"d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5","version":"v4.1.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.31","digest":"sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.31@sha256:84d861cb6da723ac10b7a00dddf778be681b8cd74b2091f18ce1d67fe4b3e7a1"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31","digest":"sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.31@sha256:80d982fe7925c640d76cbbfbe94081d2d34f7657b7c37494d8d5488f5dae3c63"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.31","digest":"sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.31@sha256:7c63bc4e57d6eac1be996bb793a5a2d74d40b15a616003f4b6805a457046c673"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.31","digest":"sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.31@sha256:c05a3f086946fab0833e078f46d35571080f187ca72f038958d45aa5cc150494"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.1","digest":"sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.1@sha256:ad2a979c2cd8b50098e84938ca9c9c1580eb8e91526f101a90adfba7859b2c32"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]}
# This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md
#
diff --git a/.github/workflows/daily-evals-report.md b/.github/workflows/daily-evals-report.md
index be9335bdc1b..37bfd6c9591 100644
--- a/.github/workflows/daily-evals-report.md
+++ b/.github/workflows/daily-evals-report.md
@@ -92,7 +92,7 @@ For each downloaded run directory at `/tmp/gh-aw/aw-mcp/logs/run-/`:
- Whether evals were declared (check for an `evals` key or the presence of the evals job in the run)
3. Note runs where the workflow declares evals but `evals.jsonl` is absent โ these are **evals job failures**.
-Cap analysis at **40 runs total**. Prefer runs with evals results first, then evals failures.
+Cap analysis at **40 runs total**. Apply prioritization before capping: first include all runs that produced `evals.jsonl` (up to 40), then fill any remaining capacity with evals-failure runs (declared evals but no results). If there are more than 40 runs with evals results, use the most recent 40 and note in the report that analysis was capped. Evals failures are still recorded even if the 40-run cap is reached from results-only runs โ count them but do not analyze their artifacts.
## Phase 3: Parse Evals JSONL
@@ -143,7 +143,7 @@ For each unique workflow:
Create one GitHub issue.
-**Title**: `[evals] Daily Evals Feature Report - YYYY-MM-DD`
+**Title**: `[evals] Daily Evals Feature Report - YYYY-MM-DD` (date in UTC, ISO 8601 format, e.g. `2026-07-15`)
**Body structure** (use `###` or lower for all headers):