adding Skillwalker evals for skills, agents, etc.

.gitignore

@@ -15,6 +15,10 @@ tests/harness
 tests/duckdb.node
 tests/libduckdb.dylib
 
+# skillwalker eval files
+output/
+analytics/
+
 # ralph loop progress tracking
 progress.txt
 deferred.txt

evals/architectural-analysis/prompts/prompt-analyze-go-project.md

@@ -0,0 +1 @@
+/architectural-analysis internal/runner

evals/architectural-analysis/prompts/skill-call-adr-graphql-switch.md

@@ -0,0 +1 @@
+create an ADR for our decision to switch from REST to GraphQL

evals/architectural-analysis/prompts/skill-call-analyze-payments-module.md

@@ -0,0 +1 @@
+analyze the architecture of the src/payments module

evals/architectural-analysis/prompts/skill-call-architectural-debt-refactor.md

@@ -0,0 +1 @@
+I need to understand the architectural debt in lib/auth before we refactor it

evals/architectural-analysis/prompts/skill-call-architecture-book-recommendation.md

@@ -0,0 +1 @@
+what's the best software architecture book for beginners

evals/architectural-analysis/prompts/skill-call-concurrency-data-flow.md

@@ -0,0 +1 @@
+assess the concurrency patterns and data flow in our worker pipeline

evals/architectural-analysis/prompts/skill-call-debug-webhook-errors.md

@@ -0,0 +1 @@
+the payment webhook is returning 500 errors intermittently, figure out why

evals/architectural-analysis/prompts/skill-call-document-notification-system.md

@@ -0,0 +1 @@
+write documentation for how the notification system works

evals/architectural-analysis/prompts/skill-call-evaluate-api-gateway-coupling.md

@@ -0,0 +1 @@
+evaluate the structural health and coupling in our api gateway directory

evals/architectural-analysis/prompts/skill-call-mermaid-architecture-diagram.md

@@ -0,0 +1 @@
+write an architecture diagram in Mermaid for the README

evals/architectural-analysis/prompts/skill-call-module-boundaries-dependencies.md

@@ -0,0 +1 @@
+review the module boundaries and dependencies in src/core

evals/architectural-analysis/prompts/skill-call-review-branch-code-changes.md

@@ -0,0 +1 @@
+review the code changes on this branch for quality issues

evals/architectural-analysis/prompts/skill-call-setup-directory-structure.md

@@ -0,0 +1 @@
+help me set up the project directory structure for a new microservice

evals/architectural-analysis/prompts/skill-call-test-plan-auth-module.md

@@ -0,0 +1 @@
+create a test plan for the user authentication module

evals/architectural-analysis/rubrics/architectural-analysis-quality.md

@@ -0,0 +1,26 @@
+## Rubric: architectural-analysis of go-project scaffold
+
+### Presence — things the analysis must identify
+- The analysis notices the fan-in concurrency pattern in runner.go where three goroutines send results through a shared channel
+- The analysis calls out that the goroutines (pi.Compute, fibonacci.Compute, sqrt.Compute) have no way to propagate errors back — if one panics, the runner will deadlock waiting on the channel
+- The analysis picks up on the tight coupling between runner.go and the three computation packages, since it directly imports and calls all three
+- The analysis mentions the types package and how it acts as the shared contract between runner and the computation modules
+- The analysis identifies that there's no context cancellation wired into the goroutines — the context.Background() passed into Run doesn't get forwarded to the computations
+- The analysis includes an executive summary that highlights the most critical findings
+- The analysis produces a risk assessment that rates findings by severity or impact
+
+### Specificity — the analysis must be concrete
+- Findings reference actual file paths like internal/runner/runner.go or internal/fibonacci/fibonacci.go, not just vague package names
+- When discussing the concurrency pattern, the analysis points to the specific channel operations (the make(chan types.Result, 3) and the for range 3 loop)
+- The coupling discussion names the specific imports in runner.go that create the dependency fan-out
+
+### Depth — the analysis must be actionable
+- At least one recommendation sketches out what a fix would look like — something like an interface, a registry pattern, or passing context down to the goroutines
+- The analysis explains why the missing error propagation is a real risk, not just a style nit — e.g., what actually happens if a goroutine panics
+- Architectural recommendations connect back to specific findings rather than offering generic advice like "reduce coupling"
+
+### Absence — the analysis must not do these things
+- The analysis doesn't invent concurrency bugs that aren't there — the channel is buffered correctly and the receive loop count matches the send count, so there's no actual deadlock in the happy path
+- The analysis doesn't claim there are data races when the goroutines don't actually share any mutable state
+- The analysis doesn't hallucinate files or packages that don't exist in the scaffold
+- The analysis doesn't confuse this with a web service or API — it's a CLI calculator, and the analysis should treat it as such

evals/architectural-analysis/scaffolds/go-project/cmd/calculator/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/testdouble/calculator/internal/runner"
+)
+
+func main() {
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
+
+	sum, err := runner.Run(context.Background())
+	if err != nil {
+		slog.Error("calculation failed", "error", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Final sum: %s\n", sum.Text('f', 50))
+}

evals/architectural-analysis/scaffolds/go-project/go.mod

@@ -0,0 +1,11 @@
+module github.com/testdouble/calculator
+
+go 1.22
+
+require github.com/stretchr/testify v1.9.0
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

evals/architectural-analysis/scaffolds/go-project/go.sum

@@ -0,0 +1,10 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

evals/architectural-analysis/scaffolds/go-project/internal/fibonacci/fibonacci.go

@@ -0,0 +1,50 @@
+package fibonacci
+
+import (
+	"log/slog"
+	"math/big"
+
+	"github.com/testdouble/calculator/internal/types"
+)
+
+// Compute calculates the fibonacci sum and sends the result on the channel.
+func Compute(ch chan<- types.Result) {
+	slog.Info("fibonacci: task started")
+	value := Calculate()
+	slog.Info("fibonacci: task complete")
+	ch <- types.Result{TaskName: "fibonacci", Value: value}
+}
+
+// Calculate computes fib(1) through fib(100) and returns their sum as a *big.Float.
+func Calculate() *big.Float {
+	sum := new(big.Int)
+
+	for i := 1; i <= 100; i++ {
+		sum.Add(sum, Fib(i))
+
+		if i%10 == 0 {
+			slog.Info("fibonacci: progress", "iteration", i, "total", 100)
+		}
+	}
+
+	return new(big.Float).SetPrec(512).SetInt(sum)
+}
+
+// Fib returns the nth Fibonacci number using iterative computation.
+func Fib(n int) *big.Int {
+	if n <= 0 {
+		return big.NewInt(0)
+	}
+	if n == 1 || n == 2 {
+		return big.NewInt(1)
+	}
+
+	a := big.NewInt(1)
+	b := big.NewInt(1)
+
+	for i := 3; i <= n; i++ {
+		a, b = b, new(big.Int).Add(a, b)
+	}
+
+	return b
+}

evals/architectural-analysis/scaffolds/go-project/internal/fibonacci/fibonacci_test.go

@@ -0,0 +1,30 @@
+package fibonacci
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFibKnownValues(t *testing.T) {
+	assert.Equal(t, big.NewInt(1), Fib(1))
+	assert.Equal(t, big.NewInt(1), Fib(2))
+	assert.Equal(t, big.NewInt(55), Fib(10))
+	assert.Equal(t, big.NewInt(6765), Fib(20))
+}
+
+func TestFibHundred(t *testing.T) {
+	expected := new(big.Int)
+	expected.SetString("354224848179261915075", 10)
+	assert.Equal(t, expected, Fib(100))
+}
+
+func TestFibZero(t *testing.T) {
+	assert.Equal(t, big.NewInt(0), Fib(0))
+}
+
+func TestCalculatePositive(t *testing.T) {
+	result := Calculate()
+	assert.True(t, result.Sign() > 0, "fibonacci sum should be positive")
+}

evals/architectural-analysis/scaffolds/go-project/internal/pi/pi.go

@@ -0,0 +1,69 @@
+package pi
+
+import (
+	"log/slog"
+	"math/big"
+
+	"github.com/testdouble/calculator/internal/types"
+)
+
+// Compute calculates pi and sends the result on the channel.
+func Compute(ch chan<- types.Result) {
+	slog.Info("pi: task started")
+	value := Calculate()
+	slog.Info("pi: task complete")
+	ch <- types.Result{TaskName: "pi", Value: value}
+}
+
+// Calculate computes pi to 100 digits of precision using Machin's formula:
+// pi/4 = 4*arctan(1/5) - arctan(1/239)
+func Calculate() *big.Float {
+	prec := uint(512)
+
+	// 4 * arctan(1/5)
+	one := new(big.Float).SetPrec(prec).SetInt64(1)
+	five := new(big.Float).SetPrec(prec).SetInt64(5)
+	term1 := arctan(new(big.Float).SetPrec(prec).Quo(one, five), 100, prec)
+	four := new(big.Float).SetPrec(prec).SetInt64(4)
+	term1.Mul(four, term1)
+
+	// arctan(1/239)
+	twoThirtyNine := new(big.Float).SetPrec(prec).SetInt64(239)
+	term2 := arctan(new(big.Float).SetPrec(prec).Quo(one, twoThirtyNine), 100, prec)
+
+	// pi/4 = term1 - term2
+	piOverFour := new(big.Float).SetPrec(prec).Sub(term1, term2)
+
+	// pi = 4 * pi/4
+	pi := new(big.Float).SetPrec(prec).Mul(four, piOverFour)
+	return pi
+}
+
+// arctan computes arctan(x) using the Taylor series:
+// arctan(x) = x - x^3/3 + x^5/5 - x^7/7 + ...
+func arctan(x *big.Float, iterations int, prec uint) *big.Float {
+	result := new(big.Float).SetPrec(prec).SetInt64(0)
+	xSquared := new(big.Float).SetPrec(prec).Mul(x, x)
+	power := new(big.Float).SetPrec(prec).Copy(x)
+	sign := int64(1)
+
+	for i := 0; i < iterations; i++ {
+		denominator := new(big.Float).SetPrec(prec).SetInt64(int64(2*i + 1))
+		term := new(big.Float).SetPrec(prec).Quo(power, denominator)
+
+		if sign > 0 {
+			result.Add(result, term)
+		} else {
+			result.Sub(result, term)
+		}
+
+		sign = -sign
+		power.Mul(power, xSquared)
+
+		if (i+1)%10 == 0 {
+			slog.Info("pi: progress", "iteration", i+1, "total", iterations)
+		}
+	}
+
+	return result
+}

evals/architectural-analysis/scaffolds/go-project/internal/pi/pi_test.go

@@ -0,0 +1,20 @@
+package pi
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPiPrecision(t *testing.T) {
+	result := Calculate()
+	piStr := result.Text('f', 20)
+	assert.True(t, strings.HasPrefix(piStr, "3.14159265358979"),
+		"pi should start with 3.14159265358979, got: %s", piStr)
+}
+
+func TestPiPositive(t *testing.T) {
+	result := Calculate()
+	assert.True(t, result.Sign() > 0, "pi should be positive")
+}

evals/architectural-analysis/scaffolds/go-project/internal/runner/runner.go

@@ -0,0 +1,36 @@
+package runner
+
+import (
+	"context"
+	"log/slog"
+	"math/big"
+
+	"github.com/testdouble/calculator/internal/fibonacci"
+	"github.com/testdouble/calculator/internal/pi"
+	"github.com/testdouble/calculator/internal/sqrt"
+	"github.com/testdouble/calculator/internal/types"
+)
+
+// Run launches all computational tasks concurrently, collects their results,
+// and returns the final sum.
+func Run(ctx context.Context) (*big.Float, error) {
+	ch := make(chan types.Result, 3)
+
+	go pi.Compute(ch)
+	go fibonacci.Compute(ch)
+	go sqrt.Compute(ch)
+
+	sum := new(big.Float).SetPrec(512).SetInt64(0)
+
+	for range 3 {
+		result := <-ch
+		slog.Info("received result",
+			"task", result.TaskName,
+			"value", result.Value.Text('f', 10),
+		)
+		sum.Add(sum, result.Value)
+	}
+
+	slog.Info("final sum computed", "value", sum.Text('f', 10))
+	return sum, nil
+}

evals/architectural-analysis/scaffolds/go-project/internal/runner/runner_test.go

@@ -0,0 +1,16 @@
+package runner
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunReturnsResult(t *testing.T) {
+	result, err := Run(context.Background())
+	require.NoError(t, err)
+	assert.NotNil(t, result)
+	assert.True(t, result.Sign() > 0, "final sum should be positive")
+}

evals/architectural-analysis/scaffolds/go-project/internal/sqrt/sqrt.go

@@ -0,0 +1,38 @@
+package sqrt
+
+import (
+	"log/slog"
+	"math"
+	"math/big"
+
+	"github.com/testdouble/calculator/internal/types"
+)
+
+// Compute calculates the square root sum and sends the result on the channel.
+func Compute(ch chan<- types.Result) {
+	slog.Info("sqrt: task started")
+	value := Calculate()
+	slog.Info("sqrt: task complete")
+	ch <- types.Result{TaskName: "sqrt", Value: value}
+}
+
+// Calculate computes sqrt(1) through sqrt(100) and returns their sum as a *big.Float.
+func Calculate() *big.Float {
+	sum := new(big.Float).SetPrec(512).SetFloat64(0)
+
+	for i := 1; i <= 100; i++ {
+		val := SqrtOf(i)
+		sum.Add(sum, new(big.Float).SetPrec(512).SetFloat64(val))
+
+		if i%10 == 0 {
+			slog.Info("sqrt: progress", "iteration", i, "total", 100)
+		}
+	}
+
+	return sum
+}
+
+// SqrtOf returns the square root of n.
+func SqrtOf(n int) float64 {
+	return math.Sqrt(float64(n))
+}