Skip to content

Commit 90fc515

Browse files
azure-sdksmw-ms
andauthored
Sync eng/common directory with azure-sdk-tools for PR 13218 (#44381)
* Add script for tool description eval workflow * Fix param casing * respond to comments --------- Co-authored-by: Summer Warren <[email protected]>
1 parent ca69f0c commit 90fc515

File tree

1 file changed

+112
-0
lines changed

1 file changed

+112
-0
lines changed
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
<#
2+
.SYNOPSIS
3+
Evaluates tool and prompt descriptions and produces a Markdown results report using Tool Description Evaluator from Mcp repo.
4+
5+
.DESCRIPTION
6+
This script builds and runs the ToolDescriptionEvaluator (.NET) against a set of tool and prompt definitions.
7+
It restores and compiles the evaluator, executes it with the provided JSON inputs, and emits a `results.md`
8+
report. The script supports configuration via parameters and environment variables for the embedding model
9+
service used during evaluation.
10+
11+
.LINK
12+
https://github.com/microsoft/mcp/tree/main/eng/tools/ToolDescriptionEvaluator
13+
14+
.PARAMETER EvaluatorPath
15+
The path to the evaluator project root (or its `src` directory) that will be restored, built, and executed.
16+
17+
.PARAMETER ToolsFilePath
18+
The path to the JSON file containing tool definitions to be evaluated.
19+
20+
.PARAMETER PromptsFilePath
21+
The path to the JSON file containing prompt definitions to be evaluated.
22+
23+
.PARAMETER OutputFilePath
24+
The target file path where the generated `results.md` will be moved after the evaluator runs.
25+
26+
.PARAMETER AoaiEndpoint
27+
The full endpoint URL for the embedding model (e.g., Azure OpenAI embeddings) used by the evaluator.
28+
29+
.PARAMETER TextEmbeddingApiKey
30+
The API key used to authenticate with the embedding endpoint. Prefer providing this via a secure
31+
secret store or environment variable rather than hard-coding.
32+
33+
.NOTES
34+
- The evaluator emits `results.md` in the evaluator folder; this script moves it to `OutputFilePath`.
35+
- Requires .NET SDK available on PATH.
36+
- Set-StrictMode is enabled.
37+
38+
.EXAMPLE
39+
.\Invoke-ToolDescriptionEvaluator.ps1 `
40+
-EvaluatorPath "C:\work\mcp\eng\tools\ToolDescriptionEvaluator\src" `
41+
-ToolsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-tools.json" `
42+
-PromptsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-prompts.json" `
43+
-OutputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" `
44+
-AoaiEndpoint "https://<your-endpoint>/openai/deployments/text-embedding-3-large/embeddings?api-version=2023-05-15" `
45+
-TextEmbeddingApiKey (Get-Secret -Name 'TextEmbeddingApiKey')
46+
47+
Runs the evaluator with the specified tools and prompts files, then moves the generated results to the output path.
48+
#>
49+
param (
50+
[Parameter(Mandatory = $true)]
51+
[string] $EvaluatorPath,
52+
53+
[Parameter(Mandatory = $true)]
54+
[string] $ToolsFilePath,
55+
56+
[Parameter(Mandatory = $true)]
57+
[string] $PromptsFilePath,
58+
59+
[Parameter(Mandatory = $true)]
60+
[string] $OutputFilePath,
61+
62+
# Environment Variables
63+
[Parameter(Mandatory = $true)]
64+
[string] $AoaiEndpoint,
65+
66+
[Parameter(Mandatory = $true)]
67+
[string] $TextEmbeddingApiKey
68+
)
69+
70+
Set-StrictMode -Version 3
71+
72+
# Validate input paths
73+
$pathsToCheck = @{
74+
"EvaluatorPath" = $EvaluatorPath
75+
"ToolsFilePath" = $ToolsFilePath
76+
"PromptsFilePath" = $PromptsFilePath
77+
"OutputFilePath" = $OutputFilePath
78+
}
79+
80+
foreach ($p in $pathsToCheck.GetEnumerator()) {
81+
if (-not (Test-Path -Path $p.Value)) {
82+
throw "Path does not exist for parameter '$($p.Key)': $($p.Value)"
83+
}
84+
}
85+
86+
# Build & run the evaluator
87+
Write-Host "Changing directory to evaluator: $EvaluatorPath"
88+
Push-Location $EvaluatorPath
89+
try {
90+
$env:AOAI_ENDPOINT = $AoaiEndpoint
91+
$env:TEXT_EMBEDDING_API_KEY = $TextEmbeddingApiKey
92+
93+
Write-Host "Running Tool..."
94+
dotnet run --configuration Release -- --tools-file "$ToolsFilePath" --prompts-file "$PromptsFilePath"
95+
}
96+
finally {
97+
Pop-Location
98+
Remove-Item Env:\AOAI_ENDPOINT -ErrorAction SilentlyContinue
99+
Remove-Item Env:\TEXT_EMBEDDING_API_KEY -ErrorAction SilentlyContinue
100+
}
101+
102+
# The tool emits results.md in the evaluator folder
103+
$generatedName = 'results.md'
104+
$EvaluatorRoot = Split-Path $EvaluatorPath -Parent
105+
$generatedPath = Join-Path -Path $EvaluatorRoot -ChildPath $generatedName
106+
if (-not (Test-Path -Path $generatedPath -PathType Leaf)) {
107+
throw "Expected output file not found: $generatedPath"
108+
}
109+
110+
Write-Host "Moving Results File: $generatedPath -> $OutputFilePath"
111+
Move-Item -Path $generatedPath -Destination $OutputFilePath -Force
112+
Write-Host "Successfully moved results file to $OutputFilePath"

0 commit comments

Comments
 (0)