diff --git a/.changeset/add-models-command.md b/.changeset/add-models-command.md new file mode 100644 index 00000000000..922c062e661 --- /dev/null +++ b/.changeset/add-models-command.md @@ -0,0 +1,5 @@ +--- +"gh-aw": minor +--- + +Add `gh aw models`, listing catalog model pricing, built-in alias resolution order, and models observed in local automation artifacts, with `--json` output. diff --git a/cmd/gh-aw/main.go b/cmd/gh-aw/main.go index d8294ce4cee..63a591399f7 100644 --- a/cmd/gh-aw/main.go +++ b/cmd/gh-aw/main.go @@ -556,7 +556,7 @@ type commandSet struct { addCmd, addWizardCmd, updateCmd, deployCmd, trialCmd, initCmd, statusCmd, listCmd *cobra.Command mcpCmd, logsCmd, auditCmd, viewCmd, healthCmd, outcomesCmd, mcpServerCmd, prCmd, secretsCmd *cobra.Command fixCmd, upgradeCmd, completionCmd, hashCmd, projectCmd, doctorCmd, checksCmd, validateCmd, lintCmd *cobra.Command - domainsCmd, experimentsCmd, forecastCmd, envCmd *cobra.Command + domainsCmd, experimentsCmd, forecastCmd, envCmd, modelsCmd *cobra.Command } func fixPathForCommand(s string) string { @@ -736,6 +736,7 @@ func createCommandSet() commandSet { experimentsCmd: cli.NewExperimentsCommand(), forecastCmd: cli.NewForecastCommand(), envCmd: cli.NewEnvCommand(), + modelsCmd: cli.NewModelsCommand(), } cli.RegisterEngineFlagCompletion(cmds.initCmd) return cmds @@ -850,7 +851,7 @@ func assignCommandGroups(cmds commandSet) { runCmd.GroupID, enableCmd.GroupID, disableCmd.GroupID, cmds.trialCmd.GroupID = "execution", "execution", "execution", "execution" cmds.logsCmd.GroupID, cmds.auditCmd.GroupID, cmds.viewCmd.GroupID = "analysis", "analysis", "analysis" cmds.healthCmd.GroupID, cmds.outcomesCmd.GroupID, cmds.checksCmd.GroupID = "analysis", "analysis", "analysis" - cmds.statusCmd.GroupID, cmds.listCmd.GroupID, cmds.experimentsCmd.GroupID, cmds.forecastCmd.GroupID = "analysis", "analysis", "analysis", "analysis" + cmds.statusCmd.GroupID, cmds.listCmd.GroupID, cmds.experimentsCmd.GroupID, cmds.forecastCmd.GroupID, cmds.modelsCmd.GroupID = "analysis", "analysis", "analysis", "analysis", "analysis" cmds.mcpServerCmd.GroupID, cmds.prCmd.GroupID, cmds.completionCmd.GroupID, cmds.hashCmd.GroupID, cmds.projectCmd.GroupID = "utilities", "utilities", "utilities", "utilities", "utilities" } @@ -860,7 +861,7 @@ func addCommandsToRoot(cmds commandSet) { runCmd, removeCmd, cmds.statusCmd, cmds.listCmd, enableCmd, disableCmd, cmds.logsCmd, cmds.auditCmd, cmds.viewCmd, cmds.healthCmd, cmds.outcomesCmd, cmds.checksCmd, cmds.mcpCmd, cmds.mcpServerCmd, cmds.prCmd, versionCmd, cmds.secretsCmd, cmds.fixCmd, cmds.validateCmd, cmds.lintCmd, cmds.completionCmd, cmds.hashCmd, cmds.projectCmd, cmds.doctorCmd, - cmds.domainsCmd, cmds.experimentsCmd, cmds.forecastCmd, cmds.envCmd, + cmds.domainsCmd, cmds.experimentsCmd, cmds.forecastCmd, cmds.modelsCmd, cmds.envCmd, ) } diff --git a/docs/adr/55148-models-command-for-catalog-alias-and-observed-models.md b/docs/adr/55148-models-command-for-catalog-alias-and-observed-models.md new file mode 100644 index 00000000000..2236434583e --- /dev/null +++ b/docs/adr/55148-models-command-for-catalog-alias-and-observed-models.md @@ -0,0 +1,54 @@ +# ADR-55148: `gh aw models` Command for Catalog Pricing, Aliases, and Observed Models + +**Date**: 2026-08-23 +**Status**: Draft +**Deciders**: pelikhan, copilot-swe-agent + +--- + +### Context + +Choosing a model for a workflow currently requires reading three disconnected sources: the embedded models catalog (`pkg/cli/model_costs.go`, providing per-token pricing), the built-in alias map in `pkg/workflow` (which alias resolves to which ordered list of concrete model IDs), and downloaded run artifacts (`summary.json`, per-run token usage files, and `awf-reflect.json`) that show which models automation has actually used. Agents and maintainers had no single command that answers "what can I pick, what does it cost, and what is actually in use here?", so model selection relied on grepping the repository or reading raw JSON artifacts. + +### Decision + +We add a dedicated read-only `gh aw models` CLI command in the analysis command group (`pkg/cli/models_command.go`) that renders three sections — catalog pricing, alias resolution order, and models observed in local automation artifacts — with `--json` for machine consumption. + +Observed-model discovery aggregates three artifact sources under a shared record keyed by normalized `provider/model`, merging provenance labels and occurrence counts. Because `summary.json` is generated from the sibling `run-*` directories, run IDs recorded in the summary are skipped when walking run directories, so a run's requests are counted once rather than twice. + +Catalog membership is provider-scoped: an observation with a known provider is matched against `provider/model` catalog IDs only, and the bare model-name index is consulted only for observations with no provider, so an unrelated `other/gpt-5.4` is not reported as catalog-backed just because `gpt-5.4` exists under another provider. + +For the optional artifact refresh (`--refresh-observed`, on by default), the command reuses `DownloadWorkflowLogs` with a new `SuppressRender` option. That option stops the logs orchestrator after artifacts and the summary file are written, so the refresh does not emit its own report onto stdout and `gh aw models --json` stays a single valid JSON document. + +### Alternatives Considered + +#### Alternative 1: Extend `gh aw logs` or `gh aw audit` With a Models View + +Add a `--models` flag to an existing analysis command instead of introducing a new one. This avoids growing the command surface, but both commands are run-centric (they take run selectors, dates, and artifact filters), whereas catalog pricing and alias resolution are static repository data unrelated to any run. Overloading them would make the flag semantics conditional on unrelated options and would still require the same suppression work for JSON output. Rejected as a worse fit for the data being reported. + +#### Alternative 2: Capture Rendered Output Instead of Adding `SuppressRender` + +Redirect `os.Stdout` around the refresh call and discard whatever the logs orchestrator prints. This keeps the change local to the new command, but process-global stdout swapping is not concurrency-safe, hides genuine errors, and silently discards warnings that the orchestrator writes intentionally. Rejected in favour of separating downloading from rendering with an explicit option. + +#### Alternative 3: Read Only `summary.json` for Observed Models + +Restrict discovery to the aggregated summary file, avoiding both the run-directory walk and the de-duplication problem entirely. This loses models seen only in `awf-reflect.json` endpoint lists (models the sandbox exposed but the summary never attributed) and produces nothing at all when a logs directory contains run artifacts without a generated summary. Rejected because endpoint-level model availability is a primary reason to run the command. + +### Consequences + +#### Positive +- One command answers model selection questions that previously required reading three separate sources. +- `--json` output is a single valid document, so agents can pipe it directly into `jq`. +- `SuppressRender` gives any future caller a supported way to download artifacts without emitting a report, instead of each caller inventing its own stdout suppression. + +#### Negative +- Observed-model results depend on artifact layout conventions (`run-` directories, `summary.json` `run_id` fields, `awf-reflect.json` endpoint shape); a change to any of those degrades the observed section until the collectors are updated. +- The default refresh performs network calls, so the command is slower than a purely local report unless `--refresh-observed=false` is passed. + +#### Neutral +- The command is read-only: it never writes workflow files and only writes artifacts through the existing logs download path. +- Provider-scoped catalog matching means observations from providers absent from the embedded catalog are reported as not-in-catalog rather than being matched by model name alone. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* diff --git a/docs/src/content/docs/setup/cli.md b/docs/src/content/docs/setup/cli.md index 65f02938198..de10f1dac98 100644 --- a/docs/src/content/docs/setup/cli.md +++ b/docs/src/content/docs/setup/cli.md @@ -636,6 +636,22 @@ gh aw outcomes history --repo owner/repo --json # JSON output for another re **Options:** `--limit`, `--source`, `--json/-j`, `--repo/-r` +#### `models` + +List model catalog pricing, built-in aliases and their resolution order, and models observed in local automation artifacts. + +```bash wrap +gh aw models # Catalog, aliases, and observed models +gh aw models --json # JSON output +gh aw models --logs-dir .github/aw/logs # Read observed models from another logs directory +gh aw models --refresh-count 50 # Inspect more recent runs when refreshing +gh aw models --refresh-observed=false # Skip the artifact refresh (local data only) +``` + +Observed models are aggregated from `summary.json` token usage, per-run token usage artifacts, and `awf-reflect.json` endpoint model lists. By default the command first refreshes those artifacts from recent runs; the refresh writes no report of its own, so `--json` output stays a single JSON document. + +**Options:** `--json/-j`, `--logs-dir`, `--refresh-observed`, `--refresh-count`, `--repo/-r` + #### `health` Display workflow health metrics and success rates. diff --git a/pkg/cli/logs_orchestrator.go b/pkg/cli/logs_orchestrator.go index 3a7afcaff4f..301afd590a2 100644 --- a/pkg/cli/logs_orchestrator.go +++ b/pkg/cli/logs_orchestrator.go @@ -323,5 +323,6 @@ func DownloadWorkflowLogs(ctx context.Context, opts LogsDownloadOptions) error { endDate: opts.EndDate, checkStaleness: true, countLimitReached: countLimitReached, + suppressRender: opts.SuppressRender, }) } diff --git a/pkg/cli/logs_orchestrator_render.go b/pkg/cli/logs_orchestrator_render.go index dd7b4bd2c5e..e0c5b3b9188 100644 --- a/pkg/cli/logs_orchestrator_render.go +++ b/pkg/cli/logs_orchestrator_render.go @@ -70,6 +70,9 @@ func renderLogsOutput(processedRuns []ProcessedRun, opts renderLogsOutputOptions } // Render output based on format preference. + if opts.suppressRender { + return nil + } switch opts.format { case "tsv": if opts.verbose { diff --git a/pkg/cli/logs_orchestrator_types.go b/pkg/cli/logs_orchestrator_types.go index ee0e84b1adc..3fbdf2bd3c9 100644 --- a/pkg/cli/logs_orchestrator_types.go +++ b/pkg/cli/logs_orchestrator_types.go @@ -35,6 +35,11 @@ type LogsDownloadOptions struct { ArtifactSets []string After string ReportFile string + // SuppressRender downloads and processes runs (including writing the summary + // file) without emitting any report to stdout. Callers that only need the + // downloaded artifacts, and that own stdout themselves, set this so their own + // output is not interleaved with the logs report. + SuppressRender bool } // StdinLogsOptions holds parameters for DownloadWorkflowLogsFromStdin. @@ -108,4 +113,7 @@ type renderLogsOutputOptions struct { // scope dateRangeCoverageWarning to the cause it actually describes, rather // than firing for timeout-driven continuations too. countLimitReached bool + // suppressRender skips all report rendering after the summary file has been + // written, for callers that only want the downloaded artifacts. + suppressRender bool } diff --git a/pkg/cli/logs_output_hint_test.go b/pkg/cli/logs_output_hint_test.go index bdf2d598c82..eb189dea3af 100644 --- a/pkg/cli/logs_output_hint_test.go +++ b/pkg/cli/logs_output_hint_test.go @@ -94,3 +94,30 @@ func TestRenderLogsOutputStaleWarningGatedByCheckStaleness(t *testing.T) { assert.NotContains(t, stdout, "No start_date/end_date was specified") }) } + +func TestRenderLogsOutputSuppressRenderWritesNothing(t *testing.T) { + processedRuns := []ProcessedRun{{ + Run: WorkflowRun{ + DatabaseID: 1, + Status: "completed", + WorkflowName: "logs", + CreatedAt: time.Now(), + }, + }} + + for _, format := range []string{"", "console", "tsv", "markdown", "pretty"} { + t.Run(format, func(t *testing.T) { + stdout, stderr := captureOutput(t, func() error { + return renderLogsOutput(processedRuns, renderLogsOutputOptions{ + outputDir: t.TempDir(), + format: format, + artifactFilter: []string{"usage"}, + suppressRender: true, + }) + }) + + assert.Empty(t, stdout, "suppressed rendering must not write to stdout") + assert.Empty(t, stderr, "suppressed rendering must not write to stderr") + }) + } +} diff --git a/pkg/cli/models_command.go b/pkg/cli/models_command.go new file mode 100644 index 00000000000..6d33a0c6c69 --- /dev/null +++ b/pkg/cli/models_command.go @@ -0,0 +1,474 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/github/gh-aw/pkg/console" + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/modelsdev" + "github.com/github/gh-aw/pkg/workflow" + "github.com/spf13/cobra" +) + +// NewModelsCommand creates the models command. +func NewModelsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "models", + Short: "List model catalog pricing, aliases, and observed automation models", + Long: `List model data to help pick a model alias or explicit model name. + +Outputs three dense sections: +- Catalog models and per-token pricing from the embedded models catalog +- Built-in model aliases and their resolution order +- Models observed in local automation logs and AWF reflect artifacts + +By default, the command attempts a lightweight log refresh focused on firewall artifacts +so recent awf-reflect data can be discovered before reporting.`, + Example: ` ` + string(constants.CLIExtensionPrefix) + ` models + ` + string(constants.CLIExtensionPrefix) + ` models --json + ` + string(constants.CLIExtensionPrefix) + ` models --logs-dir .github/aw/logs + ` + string(constants.CLIExtensionPrefix) + ` models --refresh-count 50 + ` + string(constants.CLIExtensionPrefix) + ` models --refresh-observed=false`, + RunE: func(cmd *cobra.Command, _ []string) error { + return runModelsCommand(cmd) + }, + } + + addJSONFlag(cmd) + cmd.Flags().String("logs-dir", defaultLogsOutputDir, "Directory containing downloaded logs/artifacts") + cmd.Flags().Bool("refresh-observed", true, "Attempt to refresh local observed-model artifacts before reporting") + cmd.Flags().Int("refresh-count", 20, "Maximum number of recent runs to inspect when refreshing observed models") + addRepoFlag(cmd) + return cmd +} + +type modelCatalogRow struct { + Provider string `json:"provider" console:"header:Provider"` + Model string `json:"model" console:"header:Model"` + Input string `json:"input" console:"header:Input USD/token"` + Output string `json:"output" console:"header:Output USD/token"` + CacheRead string `json:"cache_read" console:"header:Cache Read USD/token"` + CacheWrite string `json:"cache_write" console:"header:Cache Write USD/token"` + Reasoning string `json:"reasoning,omitempty" console:"header:Reasoning USD/token,omitempty"` +} + +type modelAliasRow struct { + Alias string `json:"alias" console:"header:Alias"` + Targets string `json:"targets" console:"header:Resolution Order"` +} + +type observedModelRow struct { + Provider string `json:"provider" console:"header:Provider"` + Model string `json:"model" console:"header:Model"` + Sources string `json:"sources" console:"header:Sources"` + Occurrences int `json:"occurrences" console:"header:Seen"` + InCatalog bool `json:"in_catalog" console:"header:Catalog"` + AliasHints string `json:"alias_hints,omitempty" console:"header:Alias Hints,omitempty"` +} + +type modelsReport struct { + Catalog []modelCatalogRow `json:"catalog"` + Aliases []modelAliasRow `json:"aliases"` + Observed []observedModelRow `json:"observed"` + Warnings []string `json:"warnings,omitempty"` +} + +const maxAliasHints = 6 + +func runModelsCommand(cmd *cobra.Command) error { + jsonOutput, _ := cmd.Flags().GetBool("json") + logsDir, _ := cmd.Flags().GetString("logs-dir") + refreshObserved, _ := cmd.Flags().GetBool("refresh-observed") + refreshCount, _ := cmd.Flags().GetInt("refresh-count") + repoOverride, _ := cmd.Flags().GetString("repo") + + warnings := make([]string, 0) + if refreshObserved { + if err := refreshObservedArtifacts(cmd.Context(), logsDir, refreshCount, repoOverride); err != nil { + warnings = append(warnings, "observed-model refresh failed: "+err.Error()) + } + } + + catalogRows := buildModelCatalogRows() + aliasRows, aliasMap := buildModelAliasRows() + observedRows, observedWarnings := collectObservedModelRows(logsDir, aliasMap) + warnings = append(warnings, observedWarnings...) + + report := modelsReport{ + Catalog: catalogRows, + Aliases: aliasRows, + Observed: observedRows, + Warnings: warnings, + } + + if jsonOutput { + jsonBytes, err := marshalIndentJSONOrWrap(report, "models report") + if err != nil { + return err + } + fmt.Fprintln(os.Stdout, string(jsonBytes)) + return nil + } + + fmt.Fprintln(os.Stdout, "Catalog Models") + fmt.Fprint(os.Stdout, console.RenderStruct(catalogRows)) + fmt.Fprintln(os.Stdout) + fmt.Fprintln(os.Stdout, "Model Aliases") + fmt.Fprint(os.Stdout, console.RenderStruct(aliasRows)) + fmt.Fprintln(os.Stdout) + fmt.Fprintln(os.Stdout, "Observed Models") + if len(observedRows) == 0 { + fmt.Fprintln(os.Stdout, "No observed models found in local logs/artifacts.") + } else { + fmt.Fprint(os.Stdout, console.RenderStruct(observedRows)) + } + for _, warning := range warnings { + fmt.Fprintln(os.Stderr, warning) + } + return nil +} + +func refreshObservedArtifacts(ctx context.Context, logsDir string, refreshCount int, repoOverride string) error { + if refreshCount <= 0 { + refreshCount = 20 + } + return DownloadWorkflowLogs(ctx, LogsDownloadOptions{ + Count: refreshCount, + OutputDir: logsDir, + RepoOverride: repoOverride, + ArtifactSets: []string{string(ArtifactSetFirewall), string(ArtifactSetUsage)}, + SuppressRender: true, + }) +} + +func buildModelCatalogRows() []modelCatalogRow { + initModelPrices() + rows := make([]modelCatalogRow, 0, len(modelPriceRecords)) + for _, record := range modelPriceRecords { + rows = append(rows, modelCatalogRow{ + Provider: record.provider, + Model: record.model, + Input: formatCost(record.pricing["input"]), + Output: formatCost(record.pricing["output"]), + CacheRead: formatCost(record.pricing["cache_read"]), + CacheWrite: formatCost(record.pricing["cache_write"]), + Reasoning: formatCost(record.pricing["reasoning"]), + }) + } + slices.SortFunc(rows, func(a, b modelCatalogRow) int { + if cmp := strings.Compare(a.Provider, b.Provider); cmp != 0 { + return cmp + } + return strings.Compare(a.Model, b.Model) + }) + return rows +} + +func buildModelAliasRows() ([]modelAliasRow, map[string][]string) { + aliasMap := workflow.BuiltinModelAliases() + aliases := make([]string, 0, len(aliasMap)) + for alias := range aliasMap { + aliases = append(aliases, alias) + } + slices.Sort(aliases) + + rows := make([]modelAliasRow, 0, len(aliases)) + for _, alias := range aliases { + rows = append(rows, modelAliasRow{Alias: alias, Targets: strings.Join(aliasMap[alias], ", ")}) + } + return rows, aliasMap +} + +type observedModelRecord struct { + provider string + model string + sources map[string]struct{} + occurrences int +} + +func collectObservedModelRows(logsDir string, aliasMap map[string][]string) ([]observedModelRow, []string) { + warnings := make([]string, 0) + records := make(map[string]*observedModelRecord) + catalogIndex := makeCatalogIndex() + + addObserved := func(provider, model, source string, occurrences int) { + normalizedProvider := modelsdev.NormalizeProvider(provider) + trimmedModel := strings.TrimSpace(model) + if trimmedModel == "" { + return + } + normalizedModel := strings.ToLower(trimmedModel) + key := path.Join(normalizedProvider, normalizedModel) + record := records[key] + if record == nil { + record = &observedModelRecord{provider: normalizedProvider, model: normalizedModel, sources: map[string]struct{}{}} + records[key] = record + } + record.sources[source] = struct{}{} + if occurrences <= 0 { + occurrences = 1 + } + record.occurrences += occurrences + } + + // summary.json is generated from the same run-* directories that live alongside + // it, so runs already represented in the summary are skipped when walking the + // run directories to avoid counting their requests twice. + summarizedRuns, summaryErr := collectObservedFromSummary(logsDir, addObserved) + warnings = appendObservedCollectionWarning(warnings, summaryErr, "summary.json") + warnings = appendObservedCollectionWarning(warnings, collectObservedFromRunDirs(logsDir, summarizedRuns, addObserved), "run directories") + warnings = appendObservedCollectionWarning(warnings, collectObservedFromAWFReflect(logsDir, addObserved), "awf-reflect artifacts") + + return buildObservedModelRows(records, catalogIndex, aliasMap), warnings +} + +// buildObservedModelRows converts collected observations into sorted report rows, +// ordered by descending occurrence count then provider and model. +func buildObservedModelRows(records map[string]*observedModelRecord, catalogIndex catalogIndex, aliasMap map[string][]string) []observedModelRow { + rows := make([]observedModelRow, 0, len(records)) + for _, record := range records { + sourceList := make([]string, 0, len(record.sources)) + for source := range record.sources { + sourceList = append(sourceList, source) + } + slices.Sort(sourceList) + + rows = append(rows, observedModelRow{ + Provider: record.provider, + Model: record.model, + Sources: strings.Join(sourceList, ", "), + Occurrences: record.occurrences, + InCatalog: modelExistsInCatalog(catalogIndex, record.provider, record.model), + AliasHints: inferAliasHints(record.provider, record.model, aliasMap), + }) + } + slices.SortFunc(rows, func(a, b observedModelRow) int { + if a.Occurrences != b.Occurrences { + if a.Occurrences > b.Occurrences { + return -1 + } + return 1 + } + if cmp := strings.Compare(a.Provider, b.Provider); cmp != 0 { + return cmp + } + return strings.Compare(a.Model, b.Model) + }) + return rows +} + +func appendObservedCollectionWarning(warnings []string, err error, scope string) []string { + if err != nil { + return append(warnings, "failed to parse "+scope+" for observed models: "+err.Error()) + } + return warnings +} + +func collectObservedFromSummary(logsDir string, addObserved func(provider, model, source string, occurrences int)) (map[int64]struct{}, error) { + summarizedRuns := make(map[int64]struct{}) + summaryPath := filepath.Join(logsDir, "summary.json") + content, err := os.ReadFile(summaryPath) + if err != nil { + if os.IsNotExist(err) { + return summarizedRuns, nil + } + return summarizedRuns, err + } + var payload struct { + Runs []struct { + RunID int64 `json:"run_id"` + TokenUsageSummary *struct { + ByModel map[string]*struct { + Provider string `json:"provider"` + Requests int `json:"requests"` + } `json:"by_model"` + } `json:"token_usage_summary"` + } `json:"runs"` + } + if err := json.Unmarshal(content, &payload); err != nil { + return summarizedRuns, err + } + for _, run := range payload.Runs { + if run.RunID > 0 { + summarizedRuns[run.RunID] = struct{}{} + } + if run.TokenUsageSummary == nil { + continue + } + for model, usage := range run.TokenUsageSummary.ByModel { + provider := "" + requests := 1 + if usage != nil { + provider = usage.Provider + if usage.Requests > 0 { + requests = usage.Requests + } + } + addObserved(provider, model, "summary", requests) + } + } + return summarizedRuns, nil +} + +func collectObservedFromRunDirs(logsDir string, summarizedRuns map[int64]struct{}, addObserved func(provider, model, source string, occurrences int)) error { + entries, err := os.ReadDir(logsDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "run-") { + continue + } + if runID, parseErr := strconv.ParseInt(strings.TrimPrefix(entry.Name(), "run-"), 10, 64); parseErr == nil { + if _, alreadyCounted := summarizedRuns[runID]; alreadyCounted { + continue + } + } + runDir := filepath.Join(logsDir, entry.Name()) + summary, err := analyzeTokenUsage(runDir, false) + if err != nil || summary == nil { + continue + } + for model, usage := range summary.ByModel { + provider := "" + requests := 1 + if usage != nil { + provider = usage.Provider + if usage.Requests > 0 { + requests = usage.Requests + } + } + addObserved(provider, model, "token-usage", requests) + } + } + return nil +} + +func collectObservedFromAWFReflect(logsDir string, addObserved func(provider, model, source string, occurrences int)) error { + walkErr := filepath.WalkDir(logsDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() || d.Name() != "awf-reflect.json" { + return nil + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return nil + } + var payload struct { + Endpoints []struct { + Provider string `json:"provider"` + Models []string `json:"models"` + } `json:"endpoints"` + } + if unmarshalErr := json.Unmarshal(content, &payload); unmarshalErr != nil { + return nil + } + for _, endpoint := range payload.Endpoints { + for _, model := range endpoint.Models { + addObserved(endpoint.Provider, model, "awf-reflect", 1) + } + } + return nil + }) + if walkErr != nil && !os.IsNotExist(walkErr) { + return walkErr + } + return nil +} + +func inferAliasHints(provider, model string, aliasMap map[string][]string) string { + modelIDs := []string{path.Join(provider, model)} + switch provider { + case "github-copilot": + modelIDs = append(modelIDs, path.Join("copilot", model), path.Join("github", model), path.Join("github_models", model)) + case "copilot", "github", "github_models": + modelIDs = append(modelIDs, path.Join("github-copilot", model)) + } + matches := make([]string, 0) + for alias, entries := range aliasMap { + for _, entry := range entries { + if !strings.Contains(entry, "/") { + continue + } + for _, modelID := range modelIDs { + if wildcardMatch(entry, modelID) { + matches = append(matches, alias) + break + } + } + } + } + if len(matches) == 0 { + return "" + } + slices.Sort(matches) + if len(matches) > maxAliasHints { + matches = matches[:maxAliasHints] + } + return strings.Join(matches, ", ") +} + +func wildcardMatch(pattern, value string) bool { + matched, err := filepath.Match(pattern, value) + if err != nil { + return false + } + return matched +} + +// catalogIndex holds catalog model identifiers for observed-model lookups. +// fullIDs is keyed by "provider/model" and is used for provider-scoped +// observations; bareModels is keyed by model name alone and is only consulted +// for observations whose provider is unknown. +type catalogIndex struct { + fullIDs map[string]struct{} + bareModels map[string]struct{} +} + +func makeCatalogIndex() catalogIndex { + initModelPrices() + index := catalogIndex{ + fullIDs: make(map[string]struct{}, len(modelPriceRecords)), + bareModels: make(map[string]struct{}, len(modelPriceRecords)), + } + for _, record := range modelPriceRecords { + index.fullIDs[modelsdev.NormalizeComparableModelID(record.id)] = struct{}{} + index.bareModels[modelsdev.NormalizeComparableModelID(record.model)] = struct{}{} + } + return index +} + +func modelExistsInCatalog(index catalogIndex, provider, model string) bool { + // A model that already carries its own provider scope is matched as-is. + if strings.Contains(model, "/") { + _, ok := index.fullIDs[modelsdev.NormalizeComparableModelID(model)] + return ok + } + if provider != "" { + _, ok := index.fullIDs[modelsdev.NormalizeComparableModelID(path.Join(provider, model))] + return ok + } + _, ok := index.bareModels[modelsdev.NormalizeComparableModelID(model)] + return ok +} + +func formatCost(v float64) string { + if v == 0 { + return "" + } + return fmt.Sprintf("%.9g", v) +} diff --git a/pkg/cli/models_command_integration_test.go b/pkg/cli/models_command_integration_test.go new file mode 100644 index 00000000000..3c6994c9746 --- /dev/null +++ b/pkg/cli/models_command_integration_test.go @@ -0,0 +1,176 @@ +//go:build integration + +package cli + +import ( + "bytes" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// captureModelsCommandStdout runs the models command with the given arguments and +// returns everything it wrote to stdout. +func captureModelsCommandStdout(t *testing.T, args ...string) string { + t.Helper() + + oldStdout := os.Stdout + reader, writer, err := os.Pipe() + require.NoError(t, err) + os.Stdout = writer + + // The catalog table is larger than the pipe buffer, so drain it concurrently. + outputChan := make(chan string, 1) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, reader) + outputChan <- buf.String() + }() + + cmd := NewModelsCommand() + cmd.SetArgs(args) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + execErr := cmd.Execute() + + require.NoError(t, writer.Close()) + os.Stdout = oldStdout + require.NoError(t, execErr) + + return <-outputChan +} + +// writeModelsLogsFixture creates a logs directory containing a summary, a per-run +// token usage artifact for a run absent from the summary, and an awf-reflect file. +func writeModelsLogsFixture(t *testing.T) string { + t.Helper() + + logsDir := t.TempDir() + + summaryPayload := map[string]any{ + "runs": []any{ + map[string]any{ + "run_id": 111, + "token_usage_summary": map[string]any{ + "by_model": map[string]any{ + "claude-sonnet-4.6": map[string]any{ + "provider": "github-copilot", + "requests": 3, + }, + }, + }, + }, + }, + } + summaryBytes, err := json.Marshal(summaryPayload) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(logsDir, "summary.json"), summaryBytes, 0o644)) + + // Run 111 is already represented in summary.json and must not be counted twice. + summarizedUsageDir := filepath.Join(logsDir, "run-111", "usage", "agent") + require.NoError(t, os.MkdirAll(summarizedUsageDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(summarizedUsageDir, "token_usage.jsonl"), + []byte(`{"provider":"github-copilot","model":"claude-sonnet-4.6","input_tokens":10,"output_tokens":2}`+"\n"), + 0o644, + )) + + // Run 222 is not in the summary, so its token usage is a new observation. + newUsageDir := filepath.Join(logsDir, "run-222", "usage", "agent") + require.NoError(t, os.MkdirAll(newUsageDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(newUsageDir, "token_usage.jsonl"), + []byte(`{"provider":"openai","model":"gpt-5.4","input_tokens":20,"output_tokens":5}`+"\n"), + 0o644, + )) + + reflectPath := filepath.Join(logsDir, "run-222", "sandbox", "firewall", "awf-reflect.json") + require.NoError(t, os.MkdirAll(filepath.Dir(reflectPath), 0o755)) + reflectPayload := map[string]any{ + "endpoints": []any{ + map[string]any{ + "provider": "copilot", + "models": []string{"claude-sonnet-4.6"}, + }, + }, + } + reflectBytes, err := json.Marshal(reflectPayload) + require.NoError(t, err) + require.NoError(t, os.WriteFile(reflectPath, reflectBytes, 0o644)) + + return logsDir +} + +// TestModelsCommandJSONOutputIsSingleDocument runs `gh aw models --json` end to end +// against a fixture logs directory and verifies stdout is exactly one JSON payload +// containing catalog, alias, and observed model data. +func TestModelsCommandJSONOutputIsSingleDocument(t *testing.T) { + logsDir := writeModelsLogsFixture(t) + + output := captureModelsCommandStdout(t, "--json", "--refresh-observed=false", "--logs-dir", logsDir) + + decoder := json.NewDecoder(strings.NewReader(output)) + var report modelsReport + require.NoError(t, decoder.Decode(&report), "stdout should contain a valid JSON report") + + // Any trailing content would mean a second payload was printed alongside the report. + remaining, err := io.ReadAll(decoder.Buffered()) + require.NoError(t, err) + assert.Empty(t, strings.TrimSpace(string(remaining)), "stdout should contain a single JSON document") + + require.NotEmpty(t, report.Catalog) + require.NotEmpty(t, report.Aliases) + assert.Empty(t, report.Warnings) + + observed := make(map[string]observedModelRow, len(report.Observed)) + for _, row := range report.Observed { + observed[row.Provider+"/"+row.Model] = row + } + + sonnet, ok := observed["github-copilot/claude-sonnet-4.6"] + require.True(t, ok, "observed models should include the summary model") + assert.Contains(t, sonnet.Sources, "summary") + assert.Contains(t, sonnet.Sources, "awf-reflect") + assert.NotContains(t, sonnet.Sources, "token-usage", "run-111 is already covered by summary.json") + assert.Equal(t, 4, sonnet.Occurrences, "summary requests plus one awf-reflect sighting") + assert.True(t, sonnet.InCatalog) + assert.NotEmpty(t, sonnet.AliasHints) + + gpt, ok := observed["openai/gpt-5.4"] + require.True(t, ok, "observed models should include the unsummarized run") + assert.Contains(t, gpt.Sources, "token-usage") + assert.True(t, gpt.InCatalog) +} + +// TestModelsCommandConsoleOutputSections runs the command without --json and verifies +// the human-readable report renders all three sections. +func TestModelsCommandConsoleOutputSections(t *testing.T) { + logsDir := writeModelsLogsFixture(t) + + output := captureModelsCommandStdout(t, "--refresh-observed=false", "--logs-dir", logsDir) + + assert.Contains(t, output, "Catalog Models") + assert.Contains(t, output, "Model Aliases") + assert.Contains(t, output, "Observed Models") + assert.Contains(t, output, "claude-sonnet-4.6") + assert.NotContains(t, output, "No observed models found") +} + +// TestModelsCommandWithEmptyLogsDir verifies the command still reports catalog and +// alias data when no automation artifacts are available. +func TestModelsCommandWithEmptyLogsDir(t *testing.T) { + output := captureModelsCommandStdout(t, "--json", "--refresh-observed=false", "--logs-dir", filepath.Join(t.TempDir(), "missing")) + + var report modelsReport + require.NoError(t, json.Unmarshal([]byte(output), &report)) + assert.NotEmpty(t, report.Catalog) + assert.NotEmpty(t, report.Aliases) + assert.Empty(t, report.Observed) + assert.Empty(t, report.Warnings) +} diff --git a/pkg/cli/models_command_test.go b/pkg/cli/models_command_test.go new file mode 100644 index 00000000000..698535854a9 --- /dev/null +++ b/pkg/cli/models_command_test.go @@ -0,0 +1,152 @@ +//go:build !integration + +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildModelCatalogRowsSorted(t *testing.T) { + t.Parallel() + + rows := buildModelCatalogRows() + require.NotEmpty(t, rows) + + for i := 1; i < len(rows); i++ { + prev := rows[i-1] + curr := rows[i] + if prev.Provider == curr.Provider { + assert.LessOrEqual(t, prev.Model, curr.Model) + continue + } + assert.LessOrEqual(t, prev.Provider, curr.Provider) + } +} + +func TestCollectObservedModelRowsFromSummaryAndReflect(t *testing.T) { + t.Parallel() + + logsDir := t.TempDir() + summaryPath := filepath.Join(logsDir, "summary.json") + reflectPath := filepath.Join(logsDir, "run-123", "sandbox", "firewall", "awf-reflect.json") + require.NoError(t, os.MkdirAll(filepath.Dir(reflectPath), 0o755)) + + summaryPayload := map[string]any{ + "runs": []any{ + map[string]any{ + "token_usage_summary": map[string]any{ + "by_model": map[string]any{ + "claude-sonnet-4.6": map[string]any{ + "provider": "github-copilot", + "requests": 7, + }, + }, + }, + }, + }, + } + summaryBytes, err := json.Marshal(summaryPayload) + require.NoError(t, err) + require.NoError(t, os.WriteFile(summaryPath, summaryBytes, 0o644)) + + reflectPayload := map[string]any{ + "endpoints": []any{ + map[string]any{ + "provider": "copilot", + "models": []string{"gpt-5.4", "claude-sonnet-4.6"}, + }, + }, + } + reflectBytes, err := json.Marshal(reflectPayload) + require.NoError(t, err) + require.NoError(t, os.WriteFile(reflectPath, reflectBytes, 0o644)) + + _, aliasMap := buildModelAliasRows() + rows, warnings := collectObservedModelRows(logsDir, aliasMap) + require.Empty(t, warnings) + require.NotEmpty(t, rows) + + lookup := make(map[string]observedModelRow, len(rows)) + for _, row := range rows { + lookup[row.Provider+"/"+row.Model] = row + } + + sonnet, ok := lookup["github-copilot/claude-sonnet-4.6"] + require.True(t, ok) + assert.Contains(t, sonnet.Sources, "summary") + assert.Contains(t, sonnet.Sources, "awf-reflect") + assert.GreaterOrEqual(t, sonnet.Occurrences, 8) + assert.True(t, sonnet.InCatalog) + + gpt, ok := lookup["github-copilot/gpt-5.4"] + require.True(t, ok) + assert.Contains(t, gpt.Sources, "awf-reflect") + assert.True(t, gpt.InCatalog) +} + +func TestInferAliasHints(t *testing.T) { + t.Parallel() + + _, aliasMap := buildModelAliasRows() + hints := inferAliasHints("github-copilot", "claude-sonnet-4.6", aliasMap) + assert.NotEmpty(t, hints) + assert.Contains(t, hints, "sonnet") +} + +func TestCollectObservedModelRowsSkipsRunsAlreadyInSummary(t *testing.T) { + t.Parallel() + + logsDir := t.TempDir() + usageDir := filepath.Join(logsDir, "run-123", "usage", "agent") + require.NoError(t, os.MkdirAll(usageDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(usageDir, "token_usage.jsonl"), + []byte(`{"provider":"github-copilot","model":"claude-sonnet-4.6","input_tokens":10,"output_tokens":2}`+"\n"), + 0o644, + )) + + summaryPayload := map[string]any{ + "runs": []any{ + map[string]any{ + "run_id": 123, + "token_usage_summary": map[string]any{ + "by_model": map[string]any{ + "claude-sonnet-4.6": map[string]any{ + "provider": "github-copilot", + "requests": 1, + }, + }, + }, + }, + }, + } + summaryBytes, err := json.Marshal(summaryPayload) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(logsDir, "summary.json"), summaryBytes, 0o644)) + + _, aliasMap := buildModelAliasRows() + rows, warnings := collectObservedModelRows(logsDir, aliasMap) + require.Empty(t, warnings) + require.Len(t, rows, 1) + assert.Equal(t, "summary", rows[0].Sources) + assert.Equal(t, 1, rows[0].Occurrences) +} + +func TestModelExistsInCatalogIsProviderScoped(t *testing.T) { + t.Parallel() + + index := makeCatalogIndex() + + assert.True(t, modelExistsInCatalog(index, "openai", "gpt-5.4")) + assert.True(t, modelExistsInCatalog(index, "", "gpt-5.4")) + assert.True(t, modelExistsInCatalog(index, "", "openai/gpt-5.4")) + assert.False(t, modelExistsInCatalog(index, "other", "gpt-5.4")) + assert.False(t, modelExistsInCatalog(index, "", "other/gpt-5.4")) + assert.False(t, modelExistsInCatalog(index, "openai", "not-a-real-model")) +}