Label forecast output as AI Credits and cache AIC across runs

.devcontainer/devcontainer.json

@@ -13,6 +13,17 @@
     },
     "codespaces": {
       "repositories": {
+        "gh-aw": {
+          "permissions": {
+            "actions": "write",
+            "checks": "write",
+            "contents": "write",
+            "discussions": "write",
+            "issues": "write",
+            "pull-requests": "write",
+            "workflows": "write"
+          }
+        },
         "github/gh-aw": {
           "permissions": {
             "actions": "write",

docs/src/content/docs/specs/forecast-specification.md

@@ -386,6 +386,7 @@ Effective token counts are obtained from locally-cached run summaries when avail
 - **R-SAMP-010**: MUST attempt to load the cached `run_summary.json` for each sampled run using the default logs output directory (`.github/aw/logs`).
 - **R-SAMP-011**: MUST extract the `TotalEffectiveTokens` field from the cached `TokenUsage` summary when present.
 - **R-SAMP-012**: If no cached summary exists or the ET field is zero, the run's ET contribution MUST be treated as zero and the run MUST still be counted in `sampled_runs`.  The implementation SHOULD log a debug-level warning.
+- **R-SAMP-013**: When no `run_summary.json` is available, the implementation SHOULD consult a forecast-specific `forecast_aic.json` cache in the run directory before downloading the usage artifact, and after computing a positive AIC from a freshly downloaded artifact it SHOULD persist that value to `forecast_aic.json` (version-gated by CLI version) so repeated forecast runs reuse the cached AIC without re-scanning or re-parsing artifacts. A dedicated file is used because `run_summary.json` is a "fully processed" marker for `gh aw logs`/`audit`.
 
 This lightweight approach avoids re-downloading artifacts while still providing accurate ET observations for runs that have already been processed locally by `gh aw logs`.
 

pkg/cli/audit_report.go

@@ -632,6 +632,7 @@ func describeFile(filename string) string {
 		"log.md":                        "Human-readable agent session summary",
 		"firewall.md":                   "Firewall log analysis report",
 		"run_summary.json":              "Cached summary of workflow run analysis",
+		forecastAICCacheFileName:        "Cached AI Credits (AIC) value for forecasting",
 		"prompt.txt":                    "Input prompt for AI agent",
 	}
 

pkg/cli/forecast_cache.go

@@ -0,0 +1,79 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/github/gh-aw/pkg/constants"
+)
+
+// forecastAICCacheFileName is a forecast-specific cache file written to each run
+// folder alongside the downloaded usage artifact. It stores only the computed AI
+// Credits (AIC) value so that repeated forecast runs can reload the metric without
+// re-scanning the run directory or re-parsing the usage artifact.
+//
+// A dedicated file is used (rather than the shared run_summary.json) because
+// run_summary.json acts as a "fully processed" marker for `gh aw logs`/`audit`;
+// writing a partial summary there would make those commands serve incomplete data.
+const forecastAICCacheFileName = "forecast_aic.json"
+
+// forecastAICCache is the on-disk payload cached per run for forecasting.
+type forecastAICCache struct {
+	CLIVersion string    `json:"cli_version"` // CLI version used to compute the AIC (cache invalidation key)
+	RunID      int64     `json:"run_id"`      // Workflow run database ID
+	AIC        float64   `json:"ai_credits"`  // Total AI Credits consumed by the run
+	CachedAt   time.Time `json:"cached_at"`   // When this cache entry was written
+}
+
+// loadForecastAICCache returns the cached AIC for a run when a valid, version-matching
+// forecast cache file exists. The second return value reports whether a usable cache
+// entry was found. Stale entries (version mismatch, mismatched run ID, or non-positive
+// AIC) are treated as misses so the caller recomputes.
+func loadForecastAICCache(dir string, runID int64) (float64, bool) {
+	path := filepath.Join(dir, forecastAICCacheFileName)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return 0, false
+	}
+	var c forecastAICCache
+	if err := json.Unmarshal(data, &c); err != nil {
+		return 0, false
+	}
+	if c.CLIVersion != GetVersion() || c.RunID != runID || c.AIC <= 0 {
+		return 0, false
+	}
+	return c.AIC, true
+}
+
+// saveForecastAICCache writes the computed AIC for a run to the forecast cache file.
+// It is best-effort: any error (including a non-positive AIC) is silently ignored so a
+// cache-write failure never fails the forecast. The run directory is expected to already
+// exist (it holds the downloaded usage artifact), but is created defensively.
+func saveForecastAICCache(dir string, runID int64, aic float64) {
+	if aic <= 0 {
+		return
+	}
+	c := forecastAICCache{
+		CLIVersion: GetVersion(),
+		RunID:      runID,
+		AIC:        aic,
+		CachedAt:   time.Now().UTC(),
+	}
+	data, err := json.MarshalIndent(&c, "", "  ")
+	if err != nil {
+		forecastRunLog.Printf("Failed to marshal forecast AIC cache for run %d: %v", runID, err)
+		return
+	}
+	if err := os.MkdirAll(dir, constants.DirPermPublic); err != nil {
+		forecastRunLog.Printf("Failed to create dir for forecast AIC cache for run %d: %v", runID, err)
+		return
+	}
+	path := filepath.Join(dir, forecastAICCacheFileName)
+	if err := os.WriteFile(path, data, constants.FilePermPublic); err != nil {
+		forecastRunLog.Printf("Failed to write forecast AIC cache for run %d: %v", runID, err)
+		return
+	}
+	forecastRunLog.Printf("Wrote forecast AIC cache for run %d: aic=%.3f (path=%s)", runID, aic, path)
+}

pkg/cli/forecast_cache_test.go

@@ -0,0 +1,67 @@
+//go:build !integration
+
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestForecastAICCache_RoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	const runID int64 = 12345
+
+	// Miss when no cache file exists.
+	if _, ok := loadForecastAICCache(dir, runID); ok {
+		t.Fatalf("expected cache miss for empty dir")
+	}
+
+	// Save then load returns the same value.
+	saveForecastAICCache(dir, runID, 42.5)
+	got, ok := loadForecastAICCache(dir, runID)
+	require.True(t, ok, "expected cache hit after save")
+	assert.InDelta(t, 42.5, got, 1e-9)
+
+	// The cache file records the current CLI version.
+	data, err := os.ReadFile(filepath.Join(dir, forecastAICCacheFileName))
+	require.NoError(t, err)
+	var c forecastAICCache
+	require.NoError(t, json.Unmarshal(data, &c))
+	assert.Equal(t, GetVersion(), c.CLIVersion)
+	assert.Equal(t, runID, c.RunID)
+}
+
+func TestForecastAICCache_NonPositiveNotWritten(t *testing.T) {
+	dir := t.TempDir()
+	saveForecastAICCache(dir, 1, 0)
+	saveForecastAICCache(dir, 1, -5)
+	if _, err := os.Stat(filepath.Join(dir, forecastAICCacheFileName)); !os.IsNotExist(err) {
+		t.Fatalf("expected no cache file for non-positive AIC")
+	}
+}
+
+func TestForecastAICCache_InvalidatedOnVersionMismatch(t *testing.T) {
+	dir := t.TempDir()
+	const runID int64 = 999
+	c := forecastAICCache{CLIVersion: "some-old-version", RunID: runID, AIC: 10}
+	data, err := json.MarshalIndent(&c, "", "  ")
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, forecastAICCacheFileName), data, 0o644))
+
+	if _, ok := loadForecastAICCache(dir, runID); ok {
+		t.Fatalf("expected cache miss on CLI version mismatch")
+	}
+}
+
+func TestForecastAICCache_MismatchedRunID(t *testing.T) {
+	dir := t.TempDir()
+	saveForecastAICCache(dir, 100, 7.0)
+	if _, ok := loadForecastAICCache(dir, 200); ok {
+		t.Fatalf("expected cache miss when run ID differs")
+	}
+}

pkg/cli/forecast_compute.go

@@ -248,14 +248,18 @@ func parallelLoadRunAICs(ctx context.Context, runs []WorkflowRun, config Forecas
 	return aicMap
 }
 
-// loadCachedRunAIC looks up a locally-cached RunSummary for the given
-// run ID and returns the TotalAIC from its TokenUsage summary.
-// Returns 0 when no cache exists or the cache does not contain AIC data.
-// This avoids re-downloading aw_info.json artifacts for runs already processed by
-// `gh aw logs` while still providing accurate AIC observations for the simulation.
+// loadCachedRunAIC looks up a locally-cached AIC value for the given run ID.
+// It checks, in order: (1) a fully-processed run_summary.json written by `gh aw logs`,
+// (2) a forecast-specific forecast_aic.json written by a previous forecast run, and
+// only on a miss (3) downloads the usage artifact, computes the AIC, and persists it to
+// the forecast cache for next time.
+// Returns 0 when no cache exists or the run has no AIC data.
+// This avoids re-downloading and re-parsing aw_info.json/usage artifacts for runs already
+// seen while still providing accurate AIC observations for the simulation.
 //
-// Cache location: <defaultLogsOutputDir>/run-<runID>/run_summary.json
-// (defaultLogsOutputDir is ".github/aw/logs" — defined in logs_models.go)
+// Cache locations (under <defaultLogsOutputDir>/run-<runID>/, i.e. ".github/aw/logs"):
+//   - run_summary.json   (shared cache produced by `gh aw logs`)
+//   - forecast_aic.json  (forecast-only cache produced by this function)
 func loadCachedRunAIC(ctx context.Context, runID int64, verbose bool) float64 {
 	dir := filepath.Join(defaultLogsOutputDir, fmt.Sprintf("run-%d", runID))
 	summary, ok := loadRunSummary(dir, verbose)
@@ -267,6 +271,14 @@ func loadCachedRunAIC(ctx context.Context, runID int64, verbose bool) float64 {
 		forecastRunLog.Printf("AIC cache stale/empty for run %d: cached_total_aic=%.3f, token_file_recompute_required=true", runID, summary.TokenUsage.TotalAIC)
 	}
 
+	// Second fast path: a forecast-specific AIC cache written by a previous forecast run.
+	// This lets repeated forecast runs reuse the computed AIC without re-scanning the run
+	// directory or re-parsing the usage artifact.
+	if aic, ok := loadForecastAICCache(dir, runID); ok {
+		forecastRunLog.Printf("AIC forecast-cache hit for run %d: aic=%.3f (from %s)", runID, aic, forecastAICCacheFileName)
+		return aic
+	}
+
 	forecastRunLog.Printf("AIC cache miss for run %d; downloading usage artifact to %s", runID, dir)
 	if verbose {
 		fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Downloading usage artifact for run %d…", runID)))
@@ -301,6 +313,9 @@ func loadCachedRunAIC(ctx context.Context, runID int64, verbose bool) float64 {
 		return 0
 	}
 	forecastRunLog.Printf("AIC from usage artifact for run %d: aic=%.3f", runID, tokenUsage.TotalAIC)
+	// Persist the computed AIC so subsequent forecast runs hit the fast forecast cache
+	// instead of re-scanning the directory and re-parsing the usage artifact.
+	saveForecastAICCache(dir, runID, tokenUsage.TotalAIC)
 	return tokenUsage.TotalAIC
 }
 

pkg/cli/forecast_render.go

@@ -31,6 +31,8 @@ func renderForecastTable(output ForecastResult, config ForecastConfig) error {
 
 	fmt.Fprintln(os.Stderr, console.FormatInfoMessage(
 		fmt.Sprintf("Workflow Forecast — weekly & monthly projections (based on last %d days of history)", config.Days)))
+	fmt.Fprintln(os.Stderr, console.FormatInfoMessage(
+		"Cost/projection figures are AI Credits (AIC) — the gh-aw cost metric."))
 	fmt.Fprintln(os.Stderr, "")
 
 	anyUnreliable := false
@@ -98,7 +100,7 @@ func renderForecastTable(output ForecastResult, config ForecastConfig) error {
 	}
 
 	fmt.Fprintln(os.Stderr, console.FormatInfoMessage(
-		fmt.Sprintf("P50/Run = per-run median AIC; P95/Run = 95th-percentile per-run AIC; Weekly/Monthly = projected P50 from %d-trial Monte Carlo simulation.", monteCarloIterations)))
+		fmt.Sprintf("AIC = AI Credits. P50 AIC/Run = per-run median AIC; P95 AIC/Run = 95th-percentile per-run AIC; Weekly/Monthly AIC = projected P50 from %d-trial Monte Carlo simulation.", monteCarloIterations)))
 	if anyUnreliable {
 		fmt.Fprintln(os.Stderr, console.FormatWarningMessage(
 			fmt.Sprintf("* Fewer than %d sampled runs — confidence intervals may be unreliable.", minObservationsForReliableForecast)))

pkg/cli/forecast_test.go

@@ -412,9 +412,18 @@ func TestRenderForecastTable_ZeroMonteCarloRangeRendersDash(t *testing.T) {
 	out, readErr := io.ReadAll(reader)
 	require.NoError(t, readErr)
 	assert.NotContains(t, string(out), "-–-")
+	assert.Contains(t, string(out), "Cost/projection figures are AI Credits (AIC)")
 }
 
 func TestLoadCachedRunAIC_UsageArtifactFirst(t *testing.T) {
+	originalDir, err := os.Getwd()
+	require.NoError(t, err)
+	tmpDir := t.TempDir()
+	require.NoError(t, os.Chdir(tmpDir))
+	t.Cleanup(func() {
+		_ = os.Chdir(originalDir)
+	})
+
 	originalDownload := forecastDownloadRunArtifacts
 	originalAnalyze := forecastAnalyzeTokenUsage
 	t.Cleanup(func() {
@@ -440,6 +449,14 @@ func TestLoadCachedRunAIC_UsageArtifactFirst(t *testing.T) {
 }
 
 func TestLoadCachedRunAIC_MissingUsageReturnsZero(t *testing.T) {
+	originalDir, err := os.Getwd()
+	require.NoError(t, err)
+	tmpDir := t.TempDir()
+	require.NoError(t, os.Chdir(tmpDir))
+	t.Cleanup(func() {
+		_ = os.Chdir(originalDir)
+	})
+
 	originalDownload := forecastDownloadRunArtifacts
 	originalAnalyze := forecastAnalyzeTokenUsage
 	t.Cleanup(func() {

pkg/cli/forecast_types.go

@@ -146,10 +146,10 @@ type forecastTableRow struct {
 	Workflow    string `json:"workflow"     console:"header:Workflow"`
 	Engines     string `json:"engines"      console:"header:Engines"`
 	Runs        int    `json:"runs"         console:"header:Runs"`
-	P50PerRun   string `json:"p50_per_run"  console:"header:P50/Run"`
-	P95PerRun   string `json:"p95_per_run"  console:"header:P95/Run"`
-	WeeklyP50   string `json:"weekly_p50"   console:"header:Weekly (P50)"`
-	MonthlyP50  string `json:"monthly_p50"  console:"header:Monthly (P50)"`
+	P50PerRun   string `json:"p50_per_run"  console:"header:P50 AIC/Run"`
+	P95PerRun   string `json:"p95_per_run"  console:"header:P95 AIC/Run"`
+	WeeklyP50   string `json:"weekly_p50"   console:"header:Weekly AIC (P50)"`
+	MonthlyP50  string `json:"monthly_p50"  console:"header:Monthly AIC (P50)"`
 	SuccessRate string `json:"success_rate" console:"header:Success Rate"`
 	Triggers    string `json:"triggers"     console:"header:Triggers"`
 }

pkg/linters/internal/filecheck/filecheck_test.go

@@ -4,6 +4,7 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
+	"go/types"
 	"testing"
 
 	"golang.org/x/tools/go/analysis"
@@ -32,7 +33,11 @@ func regular() {}
 		t.Fatalf("ParseFile(regular) error = %v", err)
 	}
 
-	idx := BuildGeneratedIndex(&analysis.Pass{Fset: fset, Files: []*ast.File{generatedFile, regularFile}})
+	idx := BuildGeneratedIndex(&analysis.Pass{
+		Fset:  fset,
+		Files: []*ast.File{generatedFile, regularFile},
+		Pkg:   types.NewPackage("github.com/github/gh-aw/pkg/linters/internal/filecheck/testdata", "p"),
+	})
 	if !IsGeneratedFile(generatedFilename, idx) {
 		t.Fatalf("expected %q to be marked generated", generatedFilename)
 	}
@@ -74,7 +79,11 @@ func regular() {}
 		t.Fatalf("ParseFile(regular) error = %v", err)
 	}
 
-	idx := BuildGeneratedIndex(&analysis.Pass{Fset: fset, Files: []*ast.File{regularFile}})
+	idx := BuildGeneratedIndex(&analysis.Pass{
+		Fset:  fset,
+		Files: []*ast.File{regularFile},
+		Pkg:   types.NewPackage("github.com/github/gh-aw/pkg/linters/internal/filecheck/testdata", "p"),
+	})
 	if IsGeneratedFile(regularFilename, idx) {
 		t.Fatalf("did not expect %q to be marked generated", regularFilename)
 	}

pkg/linters/internal/nolint/nolint_test.go

@@ -4,6 +4,7 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
+	"go/types"
 	"testing"
 
 	"golang.org/x/tools/go/analysis"
@@ -35,7 +36,11 @@ var _ = 5
 		t.Fatalf("ParseFile() error = %v", err)
 	}
 
-	shared := BuildDirectiveIndex(&analysis.Pass{Fset: fset, Files: []*ast.File{file}})
+	shared := BuildDirectiveIndex(&analysis.Pass{
+		Fset:  fset,
+		Files: []*ast.File{file},
+		Pkg:   types.NewPackage("github.com/github/gh-aw/pkg/linters/internal/nolint/testdata", "p"),
+	})
 	if !HasDirectiveForLinter(token.Position{Filename: filename, Line: 4}, shared, "tolowerequalfold") {
 		t.Fatalf("expected previous-line shared directive match")
 	}