1+ <#
2+ . SYNOPSIS
3+ Evaluates tool and prompt descriptions and produces a Markdown results report using Tool Description Evaluator from Mcp repo.
4+
5+ . DESCRIPTION
6+ This script builds and runs the ToolDescriptionEvaluator (.NET) against a set of tool and prompt definitions.
7+ It restores and compiles the evaluator, executes it with the provided JSON inputs, and emits a `results.md`
8+ report. The script supports configuration via parameters and environment variables for the embedding model
9+ service used during evaluation.
10+
11+ . LINK
12+ https://github.com/microsoft/mcp/tree/main/eng/tools/ToolDescriptionEvaluator
13+
14+ . PARAMETER EvaluatorPath
15+ The path to the evaluator project root (or its `src` directory) that will be restored, built, and executed.
16+
17+ . PARAMETER ToolsFilePath
18+ The path to the JSON file containing tool definitions to be evaluated.
19+
20+ . PARAMETER PromptsFilePath
21+ The path to the JSON file containing prompt definitions to be evaluated.
22+
23+ . PARAMETER OutputFilePath
24+ The target file path where the generated `results.md` will be moved after the evaluator runs.
25+
26+ . PARAMETER AoaiEndpoint
27+ The full endpoint URL for the embedding model (e.g., Azure OpenAI embeddings) used by the evaluator.
28+
29+ . PARAMETER TextEmbeddingApiKey
30+ The API key used to authenticate with the embedding endpoint. Prefer providing this via a secure
31+ secret store or environment variable rather than hard-coding.
32+
33+ . NOTES
34+ - The evaluator emits `results.md` in the evaluator folder; this script moves it to `OutputFilePath`.
35+ - Requires .NET SDK available on PATH.
36+ - Set-StrictMode is enabled.
37+
38+ . EXAMPLE
39+ .\Invoke-ToolDescriptionEvaluator.ps1 `
40+ -EvaluatorPath "C:\work\mcp\eng\tools\ToolDescriptionEvaluator\src" `
41+ -ToolsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-tools.json" `
42+ -PromptsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-prompts.json" `
43+ -OutputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" `
44+ -AoaiEndpoint "https://<your-endpoint>/openai/deployments/text-embedding-3-large/embeddings?api-version=2023-05-15" `
45+ -TextEmbeddingApiKey (Get-Secret -Name 'TextEmbeddingApiKey')
46+
47+ Runs the evaluator with the specified tools and prompts files, then moves the generated results to the output path.
48+ #>
49+ param (
50+ [Parameter (Mandatory = $true )]
51+ [string ] $EvaluatorPath ,
52+
53+ [Parameter (Mandatory = $true )]
54+ [string ] $ToolsFilePath ,
55+
56+ [Parameter (Mandatory = $true )]
57+ [string ] $PromptsFilePath ,
58+
59+ [Parameter (Mandatory = $true )]
60+ [string ] $OutputFilePath ,
61+
62+ # Environment Variables
63+ [Parameter (Mandatory = $true )]
64+ [string ] $AoaiEndpoint ,
65+
66+ [Parameter (Mandatory = $true )]
67+ [string ] $TextEmbeddingApiKey
68+ )
69+
70+ Set-StrictMode - Version 3
71+
72+ # Validate input paths
73+ $pathsToCheck = @ {
74+ " EvaluatorPath" = $EvaluatorPath
75+ " ToolsFilePath" = $ToolsFilePath
76+ " PromptsFilePath" = $PromptsFilePath
77+ " OutputFilePath" = $OutputFilePath
78+ }
79+
80+ foreach ($p in $pathsToCheck.GetEnumerator ()) {
81+ if (-not (Test-Path - Path $p.Value )) {
82+ throw " Path does not exist for parameter '$ ( $p.Key ) ': $ ( $p.Value ) "
83+ }
84+ }
85+
86+ # Build & run the evaluator
87+ Write-Host " Changing directory to evaluator: $EvaluatorPath "
88+ Push-Location $EvaluatorPath
89+ try {
90+ $env: AOAI_ENDPOINT = $AoaiEndpoint
91+ $env: TEXT_EMBEDDING_API_KEY = $TextEmbeddingApiKey
92+
93+ Write-Host " Running Tool..."
94+ dotnet run -- configuration Release -- -- tools- file " $ToolsFilePath " -- prompts- file " $PromptsFilePath "
95+ }
96+ finally {
97+ Pop-Location
98+ Remove-Item Env:\AOAI_ENDPOINT - ErrorAction SilentlyContinue
99+ Remove-Item Env:\TEXT_EMBEDDING_API_KEY - ErrorAction SilentlyContinue
100+ }
101+
102+ # The tool emits results.md in the evaluator folder
103+ $generatedName = ' results.md'
104+ $EvaluatorRoot = Split-Path $EvaluatorPath - Parent
105+ $generatedPath = Join-Path - Path $EvaluatorRoot - ChildPath $generatedName
106+ if (-not (Test-Path - Path $generatedPath - PathType Leaf)) {
107+ throw " Expected output file not found: $generatedPath "
108+ }
109+
110+ Write-Host " Moving Results File: $generatedPath -> $OutputFilePath "
111+ Move-Item - Path $generatedPath - Destination $OutputFilePath - Force
112+ Write-Host " Successfully moved results file to $OutputFilePath "
0 commit comments