From 58cc89b6ff38e46031ddc0426e4d33843cd491b6 Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Tue, 13 Oct 2026 15:13:31 +0200 Subject: [PATCH] Add BCAL experimental package selection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/bcal-evaluation.yml | 150 +++++++++++++++++-- .github/workflows/summarize-results.yml | 29 ++++ docs/bcal-datasets.md | 20 ++- scripts/Install-BCalTool.ps1 | 2 +- scripts/Resolve-BCalPackageProvenance.ps1 | 125 ++++++++++++++++ src/bcbench/commands/evaluate.py | 28 +++- src/bcbench/results/base.py | 4 +- src/bcbench/results/bceval_export.py | 19 ++- src/bcbench/types.py | 41 +++++ tests/test_bcal_dataset_workflow.py | 62 +++++++- tests/test_bcal_package_provenance.py | 31 ++++ tests/test_bcal_package_provenance_script.py | 130 ++++++++++++++++ tests/test_bcal_version_workflow.py | 27 ++-- tests/test_cli_commands.py | 7 +- tests/test_result_serialization.py | 12 +- tests/test_result_writer.py | 32 +++- 16 files changed, 673 insertions(+), 46 deletions(-) create mode 100644 scripts/Resolve-BCalPackageProvenance.ps1 create mode 100644 tests/test_bcal_package_provenance.py create mode 100644 tests/test_bcal_package_provenance_script.py diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml index 9b7a6c37a..3a3036c5b 100644 --- a/.github/workflows/bcal-evaluation.yml +++ b/.github/workflows/bcal-evaluation.yml @@ -1,5 +1,5 @@ name: Evaluation with bcal -run-name: BCal ${{ inputs.dataset || 'gold' }} (${{ inputs.external-model || 'gpt-55-chat' }}) - ${{ inputs.test-run && 'test' || 'full' }} +run-name: BCal ${{ inputs.dataset || 'gold' }} ${{ inputs.bcal-package-selection == 'experiment-pinned' && format('experiment@{0}', inputs.bcal-version) || inputs.bcal-package-selection == 'official-pinned' && format('official@{0}', inputs.bcal-version) || 'official/latest' }} (${{ inputs.external-model || 'gpt-55-chat' }}) - ${{ inputs.test-run && 'test' || 'full' }} permissions: contents: read actions: write @@ -15,16 +15,17 @@ on: options: - gold - challenge - bcal-version-mode: - description: "Install the latest prerelease or pin an exact BCal NuGet version" + bcal-package-selection: + description: "BCAL package selection" required: false - default: latest-prerelease + default: official-latest type: choice options: - - latest-prerelease - - pinned + - official-latest + - official-pinned + - experiment-pinned bcal-version: - description: "Exact BCal NuGet version for pinned mode; leave empty for latest-prerelease" + description: "Exact BCal NuGet version for pinned selections; leave empty for official-latest" required: false default: "" type: string @@ -47,17 +48,133 @@ on: concurrency: - group: bcal-evaluation-${{ inputs.dataset || 'gold' }}-${{ inputs.test-run && 'test' || 'full' }} + group: bcal-evaluation-${{ inputs.dataset || 'gold' }}-${{ inputs.bcal-package-selection || 'official-latest' }}-${{ inputs.bcal-version || 'latest' }}-${{ inputs.test-run && 'test' || 'full' }} cancel-in-progress: false env: EVALUATION_RESULTS_DIR: evaluation_results NL2AL_DATASET: ${{ inputs.dataset || 'gold' }} - BCAL_VERSION_MODE: ${{ inputs.bcal-version-mode || 'latest-prerelease' }} BCAL_REQUESTED_VERSION: ${{ inputs.bcal-version }} jobs: + validate-bcal-package: + runs-on: ubuntu-latest + environment: + name: ado-read + deployment: false + permissions: + contents: read + id-token: write + outputs: + source: ${{ steps.resolve.outputs.source }} + branch: ${{ steps.provenance.outputs.branch || steps.resolve.outputs.branch }} + feed: ${{ steps.resolve.outputs.feed }} + version-mode: ${{ steps.resolve.outputs.version-mode }} + steps: + - name: Validate and resolve BCal package selection + id: resolve + shell: bash + env: + PACKAGE_SELECTION: ${{ inputs.bcal-package-selection || 'official-latest' }} + REQUESTED_VERSION: ${{ inputs.bcal-version }} + OFFICIAL_FEED_URL: ${{ secrets.BCAL_FEED_URL }} + EXPERIMENT_FEED_URL: ${{ secrets.BCAL_TEST_FEED_URL }} + run: | + set -euo pipefail + + exact_version='^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$' + + case "$PACKAGE_SELECTION" in + official-latest) + channel="official" + version_mode="latest-prerelease" + ;; + official-pinned) + channel="official" + version_mode="pinned" + ;; + experiment-pinned) + channel="experiment" + version_mode="pinned" + ;; + *) + echo "::error::Unknown bcal-package-selection: $PACKAGE_SELECTION" + exit 1 + ;; + esac + + case "$version_mode" in + latest-prerelease) + if [[ -n "$REQUESTED_VERSION" ]]; then + echo "::error::bcal-version must be empty for official-latest." + exit 1 + fi + ;; + pinned) + if [[ ! "$REQUESTED_VERSION" =~ $exact_version ]]; then + echo "::error::Pinned package selections require bcal-version to be an exact NuGet version." + exit 1 + fi + ;; + esac + + if [[ "$OFFICIAL_FEED_URL" != https://* ]]; then + echo "::error::BCAL_FEED_URL must be a non-empty HTTPS Azure Artifacts URL." + exit 1 + fi + + case "$channel" in + official) + branch="master" + feed="DynamicsSMBPackages" + ;; + experiment) + if [[ "$EXPERIMENT_FEED_URL" != https://* ]]; then + echo "::error::BCAL_TEST_FEED_URL must be a non-empty HTTPS Azure Artifacts URL." + exit 1 + fi + if [[ "$OFFICIAL_FEED_URL" == "$EXPERIMENT_FEED_URL" ]]; then + echo "::error::Official and experimental BCAL feed URLs must differ." + exit 1 + fi + branch="" + feed="DynamicsSMBTest@Local" + ;; + esac + + echo "source=$channel" >> "$GITHUB_OUTPUT" + echo "branch=$branch" >> "$GITHUB_OUTPUT" + echo "feed=$feed" >> "$GITHUB_OUTPUT" + echo "version-mode=$version_mode" >> "$GITHUB_OUTPUT" + + - name: Checkout repository for experimental package provenance + if: ${{ steps.resolve.outputs.source == 'experiment' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Azure Login (OIDC) for experimental package provenance + if: ${{ steps.resolve.outputs.source == 'experiment' }} + uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + allow-no-subscriptions: true + + - name: Verify experimental package provenance + if: ${{ steps.resolve.outputs.source == 'experiment' }} + id: provenance + shell: pwsh + env: + BCAL_PACKAGE_ID: ${{ secrets.BCAL_PACKAGE_ID }} + BCAL_REQUESTED_VERSION: ${{ inputs.bcal-version }} + BCAL_TEST_FEED_URL: ${{ secrets.BCAL_TEST_FEED_URL }} + run: | + .\scripts\Resolve-BCalPackageProvenance.ps1 ` + -PackageId "$env:BCAL_PACKAGE_ID" ` + -Version "$env:BCAL_REQUESTED_VERSION" ` + -TestFeedUrl "$env:BCAL_TEST_FEED_URL" + get-entries: + needs: validate-bcal-package uses: $/.github/workflows/get-entries.yml with: test-run: ${{ inputs.test-run || false }} @@ -66,7 +183,7 @@ jobs: evaluate-with-bcal: runs-on: windows-latest - needs: get-entries + needs: [validate-bcal-package, get-entries] outputs: results-dir: ${{ env.EVALUATION_RESULTS_DIR }} env: @@ -98,8 +215,8 @@ jobs: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} package-id: ${{ secrets.BCAL_PACKAGE_ID }} - feed-url: ${{ secrets.BCAL_FEED_URL }} - version-mode: ${{ env.BCAL_VERSION_MODE }} + feed-url: ${{ needs.validate-bcal-package.outputs.source == 'experiment' && secrets.BCAL_TEST_FEED_URL || secrets.BCAL_FEED_URL }} + version-mode: ${{ needs.validate-bcal-package.outputs.version-mode }} version: ${{ env.BCAL_REQUESTED_VERSION }} - name: Build bc-eval LLM API bridge venv @@ -174,6 +291,8 @@ jobs: env: BCAL_LLM_MODEL: ${{ inputs.external-model || 'gpt-55-chat' }} BCAL_PACKAGE_VERSION: ${{ steps.install-bcal.outputs.version }} + BCAL_PACKAGE_CHANNEL: ${{ needs.validate-bcal-package.outputs.source }} + BCAL_PACKAGE_BRANCH: ${{ needs.validate-bcal-package.outputs.branch }} LLM_API_TENANT_ID: ${{ secrets.CAPI_TENANT_ID }} LLM_API_CLIENT_ID: ${{ secrets.CAPI_CLIENT_ID }} shell: pwsh @@ -198,8 +317,8 @@ jobs: retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: - needs: [get-entries, evaluate-with-bcal] - if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.get-entries.outputs.entries != '[]' }} + needs: [validate-bcal-package, get-entries, evaluate-with-bcal] + if: ${{ always() && !cancelled() && needs.validate-bcal-package.result == 'success' && needs.get-entries.result == 'success' && needs.get-entries.outputs.entries != '[]' }} uses: $/.github/workflows/summarize-results.yml permissions: contents: write @@ -211,6 +330,9 @@ jobs: mock: ${{ inputs.test-run || false }} category: nl2al dataset: ${{ inputs.dataset || 'gold' }} + bcal-package-channel: ${{ needs.validate-bcal-package.outputs.source }} + bcal-package-branch: ${{ needs.validate-bcal-package.outputs.branch }} + bcal-package-feed: ${{ needs.validate-bcal-package.outputs.feed }} skip-leaderboard: true expected-entries: ${{ needs.get-entries.outputs.entries }} secrets: inherit diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml index c54f90c08..cf43869a0 100644 --- a/.github/workflows/summarize-results.yml +++ b/.github/workflows/summarize-results.yml @@ -29,6 +29,21 @@ on: required: false type: string default: "" + bcal-package-channel: + description: "Optional BCAL package channel label" + required: false + type: string + default: "" + bcal-package-branch: + description: "Optional BCAL source branch label" + required: false + type: string + default: "" + bcal-package-feed: + description: "Optional BCAL package feed label" + required: false + type: string + default: "" expected-entries: description: "Optional JSON array of selected entry IDs; require complete results before scoring upload" required: false @@ -67,6 +82,9 @@ jobs: EVALUATION_MODEL: ${{ inputs.model }} EVALUATION_GIT_REF: ${{ inputs.git-ref || github.ref_name }} EXPECTED_ENTRIES: ${{ inputs.expected-entries }} + BCAL_PACKAGE_CHANNEL: ${{ inputs.bcal-package-channel }} + BCAL_PACKAGE_BRANCH: ${{ inputs.bcal-package-branch }} + BCAL_PACKAGE_FEED: ${{ inputs.bcal-package-feed }} steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -148,6 +166,17 @@ jobs: BRANCH_TAG="${BRANCH_TAG//./-}" RUN_NAME="${EVALUATION_AGENT} (${EVALUATION_MODEL}) - #${GITHUB_RUN_ID}" RUN_TAGS="${EVALUATION_AGENT},${MODEL_TAG},${BRANCH_TAG}" + if [[ -n "$BCAL_PACKAGE_CHANNEL" ]]; then + CHANNEL_TAG="$(printf '%s' "$BCAL_PACKAGE_CHANNEL" | tr -c '[:alnum:]_-' '-' | sed -E 's/^-+|-+$//g')" + PACKAGE_BRANCH_TAG="$(printf '%s' "$BCAL_PACKAGE_BRANCH" | tr -c '[:alnum:]_-' '-' | sed -E 's/^-+|-+$//g')" + FEED_TAG="$(printf '%s' "$BCAL_PACKAGE_FEED" | tr -c '[:alnum:]_-' '-' | sed -E 's/^-+|-+$//g')" + if [[ -z "$CHANNEL_TAG" || -z "$PACKAGE_BRANCH_TAG" || -z "$FEED_TAG" ]]; then + echo "::error::BCAL package provenance cannot be converted to non-empty run tags." + exit 1 + fi + RUN_NAME="${RUN_NAME} [${BCAL_PACKAGE_CHANNEL}:${BCAL_PACKAGE_BRANCH}]" + RUN_TAGS="${RUN_TAGS},bcal-channel-${CHANNEL_TAG},bcal-branch-${PACKAGE_BRANCH_TAG},bcal-feed-${FEED_TAG}" + fi if [[ -n "$NL2AL_DATASET" ]]; then RUN_NAME="${RUN_NAME} [${NL2AL_DATASET}]" RUN_TAGS="${RUN_TAGS},dataset-${NL2AL_DATASET}" diff --git a/docs/bcal-datasets.md b/docs/bcal-datasets.md index dda574c54..8e513701a 100644 --- a/docs/bcal-datasets.md +++ b/docs/bcal-datasets.md @@ -11,9 +11,19 @@ The **Evaluation with bcal** workflow defaults to `gold` for both scheduled and ## BCAL package version -The workflow defaults to `latest-prerelease`, preserving the existing installation behavior. Manual runs can instead select `pinned` and provide an exact NuGet version in `bcal-version`, including an optional fourth numeric component, prerelease suffix, or build metadata. Empty, floating, and range values are rejected before `dotnet` runs. +The workflow combines channel and version mode into one package selection: -Every matrix job resolves and installs its own package version. The installer reads the structured global-tool list after installation and passes the verified actual NuGet version to evaluation. Raw results record that observed value as `agent_version`, including agent failures and timeouts; they never record `latest`, `unknown`, or an unchecked requested string. Because a feed can publish between matrix jobs, use a pinned version when every case must run with the same package. +- **`official-latest`** (default): installs the latest prerelease from the official feed and records the BCAL source branch as `master`. +- **`official-pinned`**: installs an exact version from the official feed. +- **`experiment-pinned`**: installs an exact package from the configured experimental `@Local` feed and records the originating BC-DeveloperExperience branch or PR. + +Pinned selections require an exact NuGet version in `bcal-version`, including an optional fourth numeric component, prerelease suffix, or build metadata. `official-latest` requires the version field to be empty. Empty pinned versions, floating versions, and ranges are rejected before `dotnet` runs. + +There is no experimental-latest option because the feed contains packages from multiple branches and PRs. `experiment-pinned` requires the `BCAL_TEST_FEED_URL` repository secret to point to the experimental `@Local` feed. + +Before starting the matrix, BC-Bench verifies that the exact experimental package was published directly from BC-DeveloperExperience and derives its branch or `pr/` identity. + +Every official matrix job resolves and installs its own package version; experimental jobs install the exact requested pin. Raw results record the verified installed version as `agent_version`. Local `evaluate bcal` performs the same global-tool discovery. Use `--bcal-version ` or set `BCAL_PACKAGE_VERSION` to require the installed version to match an expected value: @@ -57,6 +67,10 @@ Each named-panel bc-eval row includes: | Metadata | Value | |---|---| +| `agent_version` | Actual installed BCAL NuGet package version | +| `bcal_package_source` | `official` or `experiment` | +| `bcal_package_branch` | `master` or the provenance-derived BC-DeveloperExperience source branch | +| `bcal_package_feed` | `DynamicsSMBPackages` or `DynamicsSMBTest@Local` | | `dataset`, `dataset_version`, `dataset_sha256` | Exact panel identity | | `dataset_mode` | `single_turn` | | `dataset_turn_count` | `1` | @@ -65,6 +79,8 @@ Each named-panel bc-eval row includes: The row tags include the dataset, reviewed version, single-turn mode, evaluation scope, and available area/family/tier classifications. +Do not pool different dataset hashes or BCAL package variants as if they were the same population. Rows without package provenance are older/unclassified results, not automatically official or master. + ## Complete-panel safety The BCAL workflow passes the exact selected matrix entry IDs to result summarization. Before any bc-eval export, summarization compares those IDs with the result artifacts and rejects: diff --git a/scripts/Install-BCalTool.ps1 b/scripts/Install-BCalTool.ps1 index ac750230b..9498e618f 100644 --- a/scripts/Install-BCalTool.ps1 +++ b/scripts/Install-BCalTool.ps1 @@ -31,7 +31,7 @@ if ($VersionMode -eq "pinned") { else { $installArguments += "--prerelease" } -$installArguments += @("--add-source", $FeedUrl) +$installArguments += @("--source", $FeedUrl) for ($attempt = 1; $attempt -le 3; $attempt++) { & dotnet @installArguments diff --git a/scripts/Resolve-BCalPackageProvenance.ps1 b/scripts/Resolve-BCalPackageProvenance.ps1 new file mode 100644 index 000000000..f2b4e5427 --- /dev/null +++ b/scripts/Resolve-BCalPackageProvenance.ps1 @@ -0,0 +1,125 @@ +param( + [Parameter(Mandatory = $true)] + [string]$PackageId, + + [Parameter(Mandatory = $true)] + [string]$Version, + + [Parameter(Mandatory = $true)] + [string]$TestFeedUrl +) + +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $false + +$expectedRepository = "bic/BC-DeveloperExperience" +$exactVersionPattern = '\A[0-9]+\.[0-9]+\.[0-9]+(?:\.[0-9]+)?(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z' + +function Normalize-Branch([string]$Value) { + $branch = $Value.Trim() + if ($branch.StartsWith("refs/heads/", [System.StringComparison]::Ordinal)) { + return $branch.Substring("refs/heads/".Length) + } + if ($branch -match '\Arefs/pull/([0-9]+)/(?:head|merge)\z') { + return "pr/$($Matches[1])" + } + if ($branch.StartsWith("refs/", [System.StringComparison]::Ordinal)) { + return $branch.Substring("refs/".Length) + } + return $branch +} + +if ([string]::IsNullOrWhiteSpace($PackageId)) { + throw "BCal package ID must not be empty." +} +if ($Version -notmatch $exactVersionPattern) { + throw "Experimental BCal provenance requires an exact NuGet version." +} +if ([string]::IsNullOrWhiteSpace($TestFeedUrl)) { + throw "Experimental BCal feed URL must not be empty." +} +$testFeed = [Uri]$TestFeedUrl +$segments = @($testFeed.AbsolutePath.Trim("/") -split "/") +$packagingIndex = -1 +for ($index = 0; $index -lt $segments.Count; $index++) { + if ($segments[$index] -eq "_packaging") { + $packagingIndex = $index + break + } +} +if ($packagingIndex -lt 0 -or $packagingIndex + 1 -ge $segments.Count) { + throw "Cannot derive the Azure Artifacts feed from the experimental BCal feed URL." +} +$feedView = $segments[$packagingIndex + 1] +if ($feedView -notmatch '\A(?[^@]+)@Local\z') { + throw "Experimental BCal feed URL must select an @Local view." +} +$feedName = $Matches.feed + +if ($testFeed.Host -match '\A(?[^.]+)\.pkgs\.visualstudio\.com\z') { + $organization = $Matches.organization +} +elseif ($testFeed.Host -eq "pkgs.dev.azure.com") { + if ($segments.Count -lt 1 -or [string]::IsNullOrWhiteSpace($segments[0])) { + throw "Cannot derive the Azure DevOps organization from the experimental BCal feed URL." + } + $organization = $segments[0] +} +else { + throw "Experimental BCal feed URL is not an Azure Artifacts endpoint." +} +$adoToken = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv +if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($adoToken)) { + throw "Could not obtain an Azure DevOps access token for BCal package provenance." +} +$headers = @{ Authorization = "Bearer $adoToken" } +$apiVersion = "7.1-preview.1" + +$feedsUri = "https://feeds.dev.azure.com/$organization/_apis/packaging/feeds?includeUrls=true&api-version=$apiVersion" +$feeds = Invoke-RestMethod -Uri $feedsUri -Headers $headers -Method Get +$matchingFeeds = @($feeds.value | Where-Object { $_.name -eq $feedName }) +if ($matchingFeeds.Count -ne 1) { + throw "Expected exactly one Azure Artifacts feed named '$feedName'; found $($matchingFeeds.Count)." +} +$feed = $matchingFeeds[0] + +$encodedPackageId = [Uri]::EscapeDataString($PackageId) +$packagesUri = "https://feeds.dev.azure.com/$organization/_apis/packaging/feeds/$($feed.id)/packages?protocolType=NuGet&packageNameQuery=$encodedPackageId&includeAllVersions=true&api-version=$apiVersion" +$packages = Invoke-RestMethod -Uri $packagesUri -Headers $headers -Method Get +$matchingPackages = @($packages.value | Where-Object { $_.name -ieq $PackageId }) +if ($matchingPackages.Count -ne 1) { + throw "Expected exactly one '$PackageId' package in '$feedName'; found $($matchingPackages.Count)." +} +$package = $matchingPackages[0] + +$versionsUri = "https://feeds.dev.azure.com/$organization/_apis/packaging/feeds/$($feed.id)/packages/$($package.id)/versions?includeUrls=true&isListed=true&api-version=$apiVersion" +$versions = Invoke-RestMethod -Uri $versionsUri -Headers $headers -Method Get +$matchingVersions = @($versions.value | Where-Object { $_.version -ieq $Version }) +if ($matchingVersions.Count -ne 1) { + throw "Expected exact BCal version '$Version' in '$feedName'; found $($matchingVersions.Count)." +} +$packageVersion = $matchingVersions[0] +if (@($packageVersion.sourceChain).Count -ne 0) { + throw "BCal version '$Version' is upstreamed into '$feedName', not published directly by a branch build." +} + +$provenanceUri = "https://feeds.dev.azure.com/$organization/_apis/packaging/Feeds/$($feed.id)/Packages/$($package.id)/Versions/$($packageVersion.id)/provenance?api-version=$apiVersion" +$provenance = Invoke-RestMethod -Uri $provenanceUri -Headers $headers -Method Get +if ($provenance.provenance.provenanceSource -ne "InternalBuild") { + throw "BCal version '$Version' does not have internal build provenance." +} +$data = $provenance.provenance.data +if ($data.'Build.Repository.Id' -ne $expectedRepository) { + throw "BCal version '$Version' was built from '$($data.'Build.Repository.Id')', not '$expectedRepository'." +} +$sourceRef = $data.'Build.SourceBranch' +if ([string]::IsNullOrWhiteSpace($sourceRef)) { + throw "BCal version '$Version' has incomplete Azure build provenance." +} + +$actualBranch = Normalize-Branch $sourceRef +if ([string]::IsNullOrWhiteSpace($actualBranch) -or $actualBranch -eq "master" -or $actualBranch.StartsWith("releases/", [System.StringComparison]::Ordinal)) { + throw "BCal version '$Version' came from official branch '$actualBranch', not an experimental branch." +} +Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "branch=$actualBranch" -Encoding utf8 +Write-Output "Verified experimental BCal package $PackageId $Version from $actualBranch." diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 1cc41e451..9e30869e7 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -6,6 +6,7 @@ import typer from bcbench_core.filesystem import prepare_run_dir +from pydantic import ValidationError from bcbench.agent import BCalBackendConfig, get_claude_version, get_copilot_version, get_pr_review_version, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent from bcbench.agent.bcal.version import get_bcal_version, verify_bcal_version @@ -32,7 +33,16 @@ from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.exceptions import AgentError from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult -from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration, NL2ALDataset +from bcbench.types import ( + AgentHarness, + AgentMetrics, + BCalPackageProvenance, + BCalPackageSource, + EvaluationCategory, + EvaluationContext, + ExperimentConfiguration, + NL2ALDataset, +) logger = logging.getLogger(__name__) _config = get_config() @@ -243,6 +253,14 @@ def evaluate_bcal( llm_model: Annotated[str | None, typer.Option(envvar="BCAL_LLM_MODEL", help="Optional model/deployment passed to BCal")] = None, dataset: Annotated[NL2ALDataset, typer.Option(help="NL2AL dataset panel: gold or challenge")] = NL2ALDataset.GOLD, bcal_version: Annotated[str | None, typer.Option(envvar="BCAL_PACKAGE_VERSION", help="Expected installed BCAL NuGet version; verified against dotnet tool list")] = None, + bcal_package_channel: Annotated[ + BCalPackageSource, + typer.Option("--bcal-package-channel", envvar="BCAL_PACKAGE_CHANNEL", help="BCAL package channel: official baseline or experimental package"), + ] = BCalPackageSource.OFFICIAL, + bcal_branch: Annotated[ + str, + typer.Option(envvar="BCAL_PACKAGE_BRANCH", help="BCAL source branch; official packages use master"), + ] = "master", ) -> None: """ Evaluate BCal dotnet tool on single nl2al dataset entry. @@ -257,8 +275,13 @@ def evaluate_bcal( command=llm_command, model=llm_model, ) + try: + bcal_package = BCalPackageProvenance(source=bcal_package_channel, branch=bcal_branch) + except ValidationError as exc: + message = exc.errors()[0]["msg"].removeprefix("Value error, ") + raise typer.BadParameter(message, param_hint="--bcal-branch") from exc - logger.info(f"Running evaluation on entry {entry_id} with BCal") + logger.info(f"Running evaluation on entry {entry_id} with BCal package {bcal_package.source}/{bcal_package.branch}") context = EvaluationContext( entry=entry, @@ -268,6 +291,7 @@ def evaluate_bcal( model=llm_model or "external-command", agent_name=AgentHarness.BCAL, category=category, + bcal_package=bcal_package, dataset=dataset, dataset_version=dataset.version, dataset_sha256=sha256(dataset_path.read_bytes()).hexdigest(), diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 97ee7fe3a..9fd7bf4f8 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, model_validator from bcbench.dataset import BaseDatasetEntry -from bcbench.types import AnyAgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration, NL2ALDataset +from bcbench.types import AnyAgentMetrics, BCalPackageProvenance, EvaluationCategory, EvaluationContext, ExperimentConfiguration, NL2ALDataset logger = logging.getLogger(__name__) @@ -22,6 +22,7 @@ class BaseEvaluationResult(BaseModel): agent_name: str category: EvaluationCategory agent_version: str | None = None + bcal_package: BCalPackageProvenance | None = None dataset: NL2ALDataset | None = None dataset_version: str | None = None dataset_sha256: str | None = None @@ -51,6 +52,7 @@ def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dic "category": context.category, "agent_name": context.agent_name, "agent_version": context.agent_version, + "bcal_package": context.bcal_package, "dataset": context.dataset, "dataset_version": context.dataset_version, "dataset_sha256": context.dataset_sha256, diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index 7a7ed2645..687a3ebc6 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -11,21 +11,27 @@ from bcbench.dataset import BaseDatasetEntry, NL2ALEntry from bcbench.results.base import BaseEvaluationResult from bcbench.results.summary import get_benchmark_version -from bcbench.types import EvaluationCategory, ExpectedOutput, ExperimentConfiguration, NL2ALDataset +from bcbench.types import BCalPackageProvenance, EvaluationCategory, ExpectedOutput, ExperimentConfiguration, NL2ALDataset logger = logging.getLogger(__name__) -def _experiment_metadata(experiment: ExperimentConfiguration | None, git_ref: str | None, benchmark_version: str) -> dict[str, Any]: +def _experiment_metadata( + experiment: ExperimentConfiguration | None, + bcal_package: BCalPackageProvenance | None, + git_ref: str | None, + benchmark_version: str, +) -> dict[str, Any]: """Metadata identifying whether a run is a baseline or an experiment, and its configuration. bc-eval promotes the ``EvalRunType`` key to the Kusto ``EvalRunType``/``testJobType`` fields when ``--eval-run-type`` is left at its default, so no extra CLI flag is needed. """ - is_experiment: bool = experiment is not None and not experiment.is_empty() + has_configuration_experiment = experiment is not None and not experiment.is_empty() + is_experiment: bool = has_configuration_experiment or (bcal_package is not None and bcal_package.is_experiment) return { "EvalRunType": "experiment" if is_experiment else "baseline", - "experiment": experiment.model_dump(mode="json") if is_experiment and experiment else None, + "experiment": experiment.model_dump(mode="json") if has_configuration_experiment and experiment else None, "git_branch": git_ref, "benchmark_version": benchmark_version, } @@ -121,6 +127,9 @@ def write_bceval_results( metadata: dict[str, Any] = { "model": result.model, "agent_version": result.agent_version, + "bcal_package_source": result.bcal_package.source if result.bcal_package else None, + "bcal_package_branch": result.bcal_package.branch if result.bcal_package else None, + "bcal_package_feed": result.bcal_package.feed_name if result.bcal_package else None, **result.export_metadata, **dataset_metadata, "prompt_tokens": (result.metrics.prompt_tokens if result.metrics else None) or 0, @@ -134,7 +143,7 @@ def write_bceval_results( "project": result.project, "error_message": result.error_message, "tool_usage": (result.metrics.tool_usage if result.metrics and result.metrics.tool_usage else None) or 0, - **_experiment_metadata(result.experiment, git_ref, benchmark_version), + **_experiment_metadata(result.experiment, result.bcal_package, git_ref, benchmark_version), } rows.append( diff --git a/src/bcbench/types.py b/src/bcbench/types.py index 759be99df..283729e56 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -26,6 +26,8 @@ "AgentMetrics", "AgentMetricsContract", "AnyAgentMetrics", + "BCalPackageProvenance", + "BCalPackageSource", "Checklist", "ChecklistAssertion", "ChecklistLevel", @@ -70,6 +72,44 @@ class Checklist(TypedDict): type RepoSlug = Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+$")] +class BCalPackageSource(StrEnum): + OFFICIAL = "official" + EXPERIMENT = "experiment" + + @property + def feed_name(self) -> Literal["DynamicsSMBPackages", "DynamicsSMBTest@Local"]: + match self: + case BCalPackageSource.OFFICIAL: + return "DynamicsSMBPackages" + case BCalPackageSource.EXPERIMENT: + return "DynamicsSMBTest@Local" + + raise ValueError(f"Unknown BCal package source: {self}") + + +class BCalPackageProvenance(BaseModel): + model_config = ConfigDict(frozen=True) + + source: BCalPackageSource + branch: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + + @model_validator(mode="after") + def validate_source_branch(self) -> BCalPackageProvenance: + if self.source is BCalPackageSource.OFFICIAL and self.branch != "master": + raise ValueError("Official BCAL packages must use branch 'master'") + if self.source is BCalPackageSource.EXPERIMENT and self.branch.casefold() == "master": + raise ValueError("Experimental BCAL packages must name a non-master branch") + return self + + @property + def feed_name(self) -> Literal["DynamicsSMBPackages", "DynamicsSMBTest@Local"]: + return self.source.feed_name + + @property + def is_experiment(self) -> bool: + return self.source is BCalPackageSource.EXPERIMENT + + class AgentMetrics(BaseModel): """Metrics collected during agent execution. @@ -727,6 +767,7 @@ class EvaluationContext[E: BaseDatasetEntry]: container: ContainerConfig | None = None agent_version: str | None = None + bcal_package: BCalPackageProvenance | None = None dataset: NL2ALDataset | None = None dataset_version: str | None = None dataset_sha256: str | None = None diff --git a/tests/test_bcal_dataset_workflow.py b/tests/test_bcal_dataset_workflow.py index c5d0649c1..1836a35e7 100644 --- a/tests/test_bcal_dataset_workflow.py +++ b/tests/test_bcal_dataset_workflow.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Any import pytest import yaml @@ -16,6 +17,10 @@ def _workflow(name: str): return yaml.safe_load(_workflow_text(name)) +def _step(workflow: dict[Any, Any], job: str, name: str) -> dict[str, Any]: + return next(step for step in workflow["jobs"][job]["steps"] if step.get("name") == name) + + def test_bcal_dispatch_exposes_only_gold_and_challenge(): workflow = _workflow("bcal-evaluation.yml") dataset = workflow[True]["workflow_dispatch"]["inputs"]["dataset"] @@ -26,11 +31,47 @@ def test_bcal_dispatch_exposes_only_gold_and_challenge(): def test_bcal_run_identity_includes_selected_dataset(): - workflow = _workflow_text("bcal-evaluation.yml") + workflow = _workflow("bcal-evaluation.yml") - assert "run-name: BCal ${{ inputs.dataset || 'gold' }}" in workflow - assert "group: bcal-evaluation-${{ inputs.dataset || 'gold' }}-" in workflow - assert "dataset: ${{ inputs.dataset || 'gold' }}" in workflow + assert "${{ inputs.dataset || 'gold' }}" in workflow["run-name"] + assert ( + "${{ inputs.bcal-package-selection == 'experiment-pinned' && format('experiment@{0}', inputs.bcal-version) || inputs.bcal-package-selection == 'official-pinned' && format('official@{0}', inputs.bcal-version) || 'official/latest' }}" + in workflow["run-name"] + ) + assert workflow["concurrency"]["group"] == ( + "bcal-evaluation-${{ inputs.dataset || 'gold' }}-${{ inputs.bcal-package-selection || 'official-latest' }}-${{ inputs.bcal-version || 'latest' }}-${{ inputs.test-run && 'test' || 'full' }}" + ) + assert workflow["jobs"]["get-entries"]["with"]["dataset"] == "${{ inputs.dataset || 'gold' }}" + + +def test_bcal_package_selection_defaults_to_official_latest_and_derives_branch_provenance(): + workflow = _workflow("bcal-evaluation.yml") + inputs = workflow[True]["workflow_dispatch"]["inputs"] + validation = _step(workflow, "validate-bcal-package", "Validate and resolve BCal package selection") + provenance = _step(workflow, "validate-bcal-package", "Verify experimental package provenance") + + assert inputs["bcal-package-selection"] == { + "description": "BCAL package selection", + "required": False, + "default": "official-latest", + "type": "choice", + "options": ["official-latest", "official-pinned", "experiment-pinned"], + } + assert "bcal-package-channel" not in inputs + assert "bcal-version-mode" not in inputs + assert "bcal-branch" not in inputs + assert "BCAL_TEST_FEED_URL" not in workflow["env"] + assert validation["env"]["EXPERIMENT_FEED_URL"] == "${{ secrets.BCAL_TEST_FEED_URL }}" + assert "pkgs.dev.azure.com" not in _workflow_text("bcal-evaluation.yml") + assert 'channel="experiment"' in validation["run"] + assert 'version_mode="pinned"' in validation["run"] + assert 'echo "version-mode=$version_mode" >> "$GITHUB_OUTPUT"' in validation["run"] + assert 'feed="DynamicsSMBTest@Local"' in validation["run"] + assert provenance["if"] == "${{ steps.resolve.outputs.source == 'experiment' }}" + assert ".\\scripts\\Resolve-BCalPackageProvenance.ps1" in provenance["run"] + assert "-ExpectedBranch" not in provenance["run"] + assert '-TestFeedUrl "$env:BCAL_TEST_FEED_URL"' in provenance["run"] + assert workflow["jobs"]["validate-bcal-package"]["outputs"]["branch"] == "${{ steps.provenance.outputs.branch || steps.resolve.outputs.branch }}" @pytest.mark.parametrize("workflow_name", ["get-entries.yml", "summarize-results.yml"]) @@ -64,20 +105,25 @@ def test_bcal_keeps_existing_installation_and_single_turn_execution(): workflow = _workflow_text("bcal-evaluation.yml") assert ".github/actions/install-bcal" in workflow - assert "bcal-version-mode" in workflow + assert "bcal-package-selection" in workflow + assert "bcal-version-mode" not in workflow assert "scenario" not in workflow.lower() def test_summary_runs_after_non_cancelled_matrix_failures_with_exact_ids(): job = _workflow("bcal-evaluation.yml")["jobs"]["summarize-results"] - assert set(job["needs"]) == {"get-entries", "evaluate-with-bcal"} + assert set(job["needs"]) == {"validate-bcal-package", "get-entries", "evaluate-with-bcal"} assert "always()" in job["if"] assert "!cancelled()" in job["if"] + assert "needs.validate-bcal-package.result == 'success'" in job["if"] assert "needs.get-entries.result == 'success'" in job["if"] assert "needs.get-entries.outputs.entries != '[]'" in job["if"] assert job["with"]["expected-entries"] == "${{ needs.get-entries.outputs.entries }}" assert job["with"]["dataset"] == "${{ inputs.dataset || 'gold' }}" + assert job["with"]["bcal-package-channel"] == "${{ needs.validate-bcal-package.outputs.source }}" + assert job["with"]["bcal-package-branch"] == "${{ needs.validate-bcal-package.outputs.branch }}" + assert job["with"]["bcal-package-feed"] == "${{ needs.validate-bcal-package.outputs.feed }}" def test_summary_validates_coverage_and_tags_selected_dataset(): @@ -90,3 +136,7 @@ def test_summary_validates_coverage_and_tags_selected_dataset(): assert "if: always()" in workflow assert 'RUN_NAME="${RUN_NAME} [${NL2AL_DATASET}]"' in workflow assert 'RUN_TAGS="${RUN_TAGS},dataset-${NL2AL_DATASET}"' in workflow + assert 'RUN_NAME="${RUN_NAME} [${BCAL_PACKAGE_CHANNEL}:${BCAL_PACKAGE_BRANCH}]"' in workflow + assert "bcal-channel-${CHANNEL_TAG}" in workflow + assert "bcal-branch-${PACKAGE_BRANCH_TAG}" in workflow + assert "bcal-feed-${FEED_TAG}" in workflow diff --git a/tests/test_bcal_package_provenance.py b/tests/test_bcal_package_provenance.py new file mode 100644 index 000000000..f6a18fe3c --- /dev/null +++ b/tests/test_bcal_package_provenance.py @@ -0,0 +1,31 @@ +import pytest +from pydantic import ValidationError + +from bcbench.types import BCalPackageProvenance, BCalPackageSource + + +def test_official_package_is_master_from_the_official_feed(): + package = BCalPackageProvenance(source=BCalPackageSource.OFFICIAL, branch="master") + + assert package.feed_name == "DynamicsSMBPackages" + assert not package.is_experiment + + +def test_experimental_package_uses_the_test_local_view(): + package = BCalPackageProvenance(source=BCalPackageSource.EXPERIMENT, branch="private/user/feature+experiment") + + assert package.feed_name == "DynamicsSMBTest@Local" + assert package.is_experiment + + +@pytest.mark.parametrize( + ("source", "branch"), + [ + (BCalPackageSource.OFFICIAL, "main"), + (BCalPackageSource.EXPERIMENT, "master"), + (BCalPackageSource.EXPERIMENT, ""), + ], +) +def test_invalid_source_branch_combinations_are_rejected(source, branch): + with pytest.raises(ValidationError): + BCalPackageProvenance(source=source, branch=branch) diff --git a/tests/test_bcal_package_provenance_script.py b/tests/test_bcal_package_provenance_script.py new file mode 100644 index 000000000..207f27ce9 --- /dev/null +++ b/tests/test_bcal_package_provenance_script.py @@ -0,0 +1,130 @@ +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +PWSH = shutil.which("pwsh") +SCRIPT = Path(__file__).parents[1] / "scripts" / "Resolve-BCalPackageProvenance.ps1" + + +def _resolve_package(*, source_ref="refs/heads/feature/test", source_chain=None, repository="bic/BC-DeveloperExperience"): + assert PWSH is not None + case = { + "source_chain": [] if source_chain is None else source_chain, + "source_ref": source_ref, + "repository": repository, + } + result = subprocess.run( + [ + PWSH, + "-NoProfile", + "-NonInteractive", + "-Command", + r""" + $ErrorActionPreference = "Stop" + $case = $env:BCAL_PROVENANCE_TEST_CASE | ConvertFrom-Json -AsHashtable + $global:BcalOutputs = [System.Collections.Generic.List[string]]::new() + function az { + $global:LASTEXITCODE = 0 + "test-token" + } + function Invoke-RestMethod { + param([string]$Uri, [hashtable]$Headers, [string]$Method) + if ($Uri -match '/feeds\?') { + return @{ value = @(@{ name = "ExperimentFeed"; id = "feed-id" }) } + } + if ($Uri -match '/packages\?') { + return @{ value = @(@{ name = "Microsoft.BusinessCentral.BCal.CLI"; id = "package-id" }) } + } + if ($Uri -match '/versions\?') { + return @{ + value = @( + @{ + id = "version-id" + version = "30.0.43.223-beta" + sourceChain = $case.source_chain + publishDate = "2026-10-01T10:35:28Z" + } + ) + } + } + if ($Uri -match '/provenance\?') { + return @{ + provenance = @{ + provenanceSource = "InternalBuild" + data = @{ + "Build.Repository.Id" = $case.repository + "Build.SourceBranch" = $case.source_ref + } + } + } + } + throw "Unexpected URI: $Uri" + } + function Add-Content { + param([string]$LiteralPath, [string]$Value, [string]$Encoding) + if ($LiteralPath -ne "bcal-provenance-test-output") { throw "Unexpected output path." } + $global:BcalOutputs.Add($Value) + } + $env:GITHUB_OUTPUT = "bcal-provenance-test-output" + $failure = "" + try { + & $env:BCAL_PROVENANCE_TEST_SCRIPT ` + -PackageId "Microsoft.BusinessCentral.BCal.CLI" ` + -Version "30.0.43.223-beta" ` + -TestFeedUrl "https://example.pkgs.visualstudio.com/_packaging/ExperimentFeed@Local/nuget/v3/index.json" | Out-Null + } + catch { + $failure = $_.Exception.Message + } + @{ outputs = $global:BcalOutputs.ToArray(); error = $failure } | ConvertTo-Json -Compress -Depth 8 + """, + ], + env={ + **os.environ, + "PATH": "", + "BCAL_PROVENANCE_TEST_CASE": json.dumps(case), + "BCAL_PROVENANCE_TEST_SCRIPT": str(SCRIPT), + }, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.splitlines()[-1]) + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell is required to test BCal package provenance") +def test_experimental_package_branch_is_verified_and_published(): + result = _resolve_package() + + assert result["error"] == "" + assert result["outputs"] == ["branch=feature/test"] + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell is required to test BCal package provenance") +def test_pr_package_uses_stable_pr_label(): + result = _resolve_package(source_ref="refs/pull/123/merge") + + assert result["error"] == "" + assert result["outputs"] == ["branch=pr/123"] + + +@pytest.mark.skipif(PWSH is None, reason="PowerShell is required to test BCal package provenance") +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"source_ref": "refs/heads/master"}, "not an experimental branch"), + ({"source_chain": ["official-upstream"]}, "not published directly"), + ({"repository": "other/repository"}, "not 'bic/BC-DeveloperExperience'"), + ], +) +def test_unverified_branch_package_is_rejected(overrides, message): + result = _resolve_package(**overrides) + + assert message in result["error"] + assert result["outputs"] == [] diff --git a/tests/test_bcal_version_workflow.py b/tests/test_bcal_version_workflow.py index c713a4a45..b0e16f41a 100644 --- a/tests/test_bcal_version_workflow.py +++ b/tests/test_bcal_version_workflow.py @@ -25,21 +25,22 @@ def _step(workflow: dict[Any, Any], name: str) -> dict[str, Any]: return next(step for step in workflow["jobs"]["evaluate-with-bcal"]["steps"] if step.get("name") == name) -def test_bcal_version_selection_defaults_to_latest_prerelease_for_dispatch_and_schedule() -> None: +def test_bcal_package_selection_encodes_valid_channel_and_version_modes() -> None: workflow = _workflow() inputs = workflow[True]["workflow_dispatch"]["inputs"] - mode = inputs["bcal-version-mode"] + selection = inputs["bcal-package-selection"] version = inputs["bcal-version"] - assert mode["type"] == "choice" - assert mode["default"] == "latest-prerelease" - assert mode["options"] == ["latest-prerelease", "pinned"] - assert mode["required"] is False + assert selection["type"] == "choice" + assert selection["default"] == "official-latest" + assert selection["options"] == ["official-latest", "official-pinned", "experiment-pinned"] + assert selection["required"] is False assert version["type"] == "string" assert version["default"] == "" assert version["required"] is False - assert workflow["env"]["BCAL_VERSION_MODE"] == "${{ inputs.bcal-version-mode || 'latest-prerelease' }}" + assert "BCAL_VERSION_MODE" not in workflow["env"] assert workflow["env"]["BCAL_REQUESTED_VERSION"] == "${{ inputs.bcal-version }}" + assert workflow["jobs"]["validate-bcal-package"]["outputs"]["version-mode"] == "${{ steps.resolve.outputs.version-mode }}" def test_bcal_installed_version_output_flows_to_evaluation_metadata() -> None: @@ -59,6 +60,8 @@ def test_bcal_installed_version_output_flows_to_evaluation_metadata() -> None: assert '-FeedUrl "$env:BCAL_FEED_URL"' in action_install["run"] assert "${{ inputs." not in action_install["run"] assert evaluate["env"]["BCAL_PACKAGE_VERSION"] == "${{ steps.install-bcal.outputs.version }}" + assert evaluate["env"]["BCAL_PACKAGE_CHANNEL"] == "${{ needs.validate-bcal-package.outputs.source }}" + assert evaluate["env"]["BCAL_PACKAGE_BRANCH"] == "${{ needs.validate-bcal-package.outputs.branch }}" assert "[string]::IsNullOrWhiteSpace($env:BCAL_PACKAGE_VERSION)" in evaluate["run"] assert evaluate["run"].index("BCAL_PACKAGE_VERSION") < evaluate["run"].index("bcbench evaluate bcal") @@ -71,14 +74,14 @@ def test_bcal_matrix_resolves_versions_per_job_and_preserves_feed_authentication login, credential_provider, action_install = action["runs"]["steps"] assert "resolve-bcal-version" not in workflow["jobs"] - assert matrix["needs"] == "get-entries" + assert set(matrix["needs"]) == {"validate-bcal-package", "get-entries"} assert len([step for step in matrix["steps"] if step.get("uses") == "$/.github/actions/install-bcal"]) == 1 assert install["with"] == { "client-id": "${{ secrets.AZURE_CLIENT_ID }}", "tenant-id": "${{ secrets.AZURE_TENANT_ID }}", "package-id": "${{ secrets.BCAL_PACKAGE_ID }}", - "feed-url": "${{ secrets.BCAL_FEED_URL }}", - "version-mode": "${{ env.BCAL_VERSION_MODE }}", + "feed-url": "${{ needs.validate-bcal-package.outputs.source == 'experiment' && secrets.BCAL_TEST_FEED_URL || secrets.BCAL_FEED_URL }}", + "version-mode": "${{ needs.validate-bcal-package.outputs.version-mode }}", "version": "${{ env.BCAL_REQUESTED_VERSION }}", } assert action["runs"]["using"] == "composite" @@ -209,7 +212,7 @@ def test_bcal_install_uses_selected_mode_and_records_actual_version(mode: str, v assert result["error"] == "" assert result["calls"] == [ - ["tool", "install", "--global", PACKAGE_ID, *version_arguments, "--add-source", FEED_URL], + ["tool", "install", "--global", PACKAGE_ID, *version_arguments, "--source", FEED_URL], ["tool", "list", PACKAGE_ID, "--global", "--format", "json"], ] assert result["outputs"] == [f"version={actual_version}"] @@ -305,7 +308,7 @@ def test_bcal_transient_install_failure_retries_same_exact_pin() -> None: result = _install_bcal_tool(mode="pinned", version="1.2.3-preview.4", install_exit_codes=[1, 1, 0]) assert result["error"] == "" - assert result["calls"][:3] == [["tool", "install", "--global", PACKAGE_ID, "--version", "1.2.3-preview.4", "--add-source", FEED_URL]] * 3 + assert result["calls"][:3] == [["tool", "install", "--global", PACKAGE_ID, "--version", "1.2.3-preview.4", "--source", FEED_URL]] * 3 assert result["calls"][3] == ["tool", "list", PACKAGE_ID, "--global", "--format", "json"] assert result["outputs"] == ["version=1.2.3-preview.4"] assert result["sleeps"] == [20, 20] diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 064169078..b8911b6c2 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -16,7 +16,7 @@ from bcbench.commands import evaluate as evaluate_commands from bcbench.commands import run as run_commands from bcbench.dataset.dataset_entry import _BugFixTestGenBase -from bcbench.types import AgentMetrics, EvaluationCategory +from bcbench.types import AgentMetrics, BCalPackageProvenance, BCalPackageSource, EvaluationCategory from tests.conftest import ( create_bugfix_result, create_dataset_entry, @@ -144,6 +144,7 @@ def execute(self, context, _agent_runner): assert captured["context"].model == expected_model assert captured["context"].agent_version == "18.0.1.2-beta" + assert captured["context"].bcal_package == BCalPackageProvenance(source=BCalPackageSource.OFFICIAL, branch="master") def _option_names(command: click.Command) -> set[str]: @@ -160,6 +161,10 @@ def test_bcal_commands_only_expose_external_command_options(command_group): assert {"--llm-command", "--llm-model"} <= options assert options.isdisjoint({"--backend", "--endpoint", "--deployment"}) + if command_group == "evaluate": + assert {"--bcal-package-channel", "--bcal-branch"} <= options + else: + assert options.isdisjoint({"--bcal-package-channel", "--bcal-branch"}) @pytest.fixture diff --git a/tests/test_result_serialization.py b/tests/test_result_serialization.py index 099568b89..7879c92d3 100644 --- a/tests/test_result_serialization.py +++ b/tests/test_result_serialization.py @@ -4,7 +4,7 @@ from bcbench.results.base import BaseEvaluationResult from bcbench.results.summary import EvaluationResultSummary -from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics +from bcbench.types import AgentHarness, AgentMetrics, BCalPackageProvenance, BCalPackageSource, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics from tests.conftest import create_bugfix_result, create_codereview_result, create_testgen_result @@ -185,6 +185,16 @@ def test_some_experiment_saves_as_dict(self, tmp_path, sample_result_bug_fix): assert data["experiment"]["custom_instructions"] is True assert data["experiment"]["mcp_servers"] is None + def test_bcal_package_provenance_round_trips(self, tmp_path, sample_result_bug_fix): + sample_result_bug_fix.bcal_package = BCalPackageProvenance(source=BCalPackageSource.EXPERIMENT, branch="private/user/change") + sample_result_bug_fix.save(tmp_path, "result.jsonl") + + payload = json.loads((tmp_path / "result.jsonl").read_text(encoding="utf-8")) + loaded = BaseEvaluationResult.from_json(payload) + + assert payload["bcal_package"] == {"source": "experiment", "branch": "private/user/change"} + assert loaded.bcal_package == sample_result_bug_fix.bcal_package + def test_tool_usage_saves_as_dict(self, tmp_path): tool_usage = {"bash": 5, "view": 3, "search": 2} result = create_bugfix_result( diff --git a/tests/test_result_writer.py b/tests/test_result_writer.py index 5dc8a0b0d..63573d3f0 100644 --- a/tests/test_result_writer.py +++ b/tests/test_result_writer.py @@ -6,7 +6,7 @@ from bcbench.dataset.codereview import CodeReviewEntry from bcbench.dataset.dataset_entry import BugFixEntry, _BugFixTestGenBase from bcbench.results.bceval_export import write_bceval_results -from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics +from bcbench.types import AgentHarness, AgentMetrics, BCalPackageProvenance, BCalPackageSource, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics from tests.conftest import VALID_INSTANCE_ID, create_bugfix_result, create_codereview_entry, create_codereview_result @@ -328,3 +328,33 @@ def test_marks_run_as_experiment_with_config(self, tmp_path, sample_dataset_file assert data["metadata"]["experiment"]["al_lsp_enabled"] is True assert data["metadata"]["experiment"]["custom_instructions"] is True assert data["metadata"]["git_branch"] == "feat/al-mcp" + + @pytest.mark.parametrize( + ("package", "run_type", "feed"), + [ + (BCalPackageProvenance(source=BCalPackageSource.OFFICIAL, branch="master"), "baseline", "DynamicsSMBPackages"), + (BCalPackageProvenance(source=BCalPackageSource.EXPERIMENT, branch="private/user/change"), "experiment", "DynamicsSMBTest@Local"), + ], + ) + def test_bcal_package_provenance_is_exported_and_classifies_the_run(self, tmp_path, sample_dataset_file, problem_statement_dir, package, run_type, feed): + result = create_bugfix_result(metrics=AgentMetrics(execution_time=1.0), experiment=ExperimentConfiguration()) + result.bcal_package = package + + with ( + patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), + patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file), + ): + write_bceval_results( + results=[result], + out_dir=tmp_path, + run_id="bcal-package", + output_filename="results.jsonl", + category=EvaluationCategory.BUG_FIX, + ) + + metadata = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metadata"] + assert metadata["bcal_package_source"] == package.source + assert metadata["bcal_package_branch"] == package.branch + assert metadata["bcal_package_feed"] == feed + assert metadata["EvalRunType"] == run_type + assert metadata["experiment"] is None