Changing to use the embedded copilot CLI

.gitattributes

@@ -3,6 +3,6 @@
 .squad/agents/*/history.md merge=union
 .squad/log/** merge=union
 .squad/orchestration-log/** merge=union
-
 # Set the default behavior, in case people don't have core.autocrlf set.
 * text=auto
+*.zst filter=lfs diff=lfs merge=lfs -text

.github/workflows/docker-ci.yml

@@ -27,6 +27,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          lfs: true
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/go-ci.yml

@@ -23,6 +23,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          lfs: true
 
       - name: Setup Go Environment
         uses: actions/setup-go@v5
@@ -111,6 +113,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          lfs: true
 
       - name: Setup Go Environment
         uses: actions/setup-go@v5

.github/workflows/release.yml

@@ -74,6 +74,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          lfs: true
 
       - name: Setup Go Environment
         uses: actions/setup-go@v5
@@ -170,6 +172,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          lfs: true
 
       - name: Setup Go Environment
         uses: actions/setup-go@v5

.github/workflows/waza-eval.yml

@@ -90,6 +90,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+        with:
+          lfs: true
 
       - name: Setup Go Environment
         uses: actions/setup-go@v5

README.md

@@ -22,8 +22,18 @@ Or download binaries directly from the [latest release](https://github.com/micro
 
 Requires Go 1.26+:
 
+NOTE, due to the use of LFS artifacts you cannot install waza using `go install`. To install waza, outside of a normal release, you'll need to clone the repository:
+
 ```bash
-go install github.com/microsoft/waza/cmd/waza@latest
+git clone https://github.com/microsoft/waza.git
+cd waza
+
+# ensure git LFS-based artifacts are available (for embedded copilot binaries)
+git lfs install
+git lfs pull
+
+go build -o waza ./cmd/waza
+./waza <waza command line>
 ```
 
 ### Azure Developer CLI (azd) Extension

cmd/waza/cmd_new_task_test.go

@@ -173,10 +173,7 @@ func TestNewTaskFromPromptCommand_CopilotInitErrorReturned(t *testing.T) {
 
 func TestNewTaskFromPromptCommand_DiscoverErrorReturned(t *testing.T) {
 	ctrl := gomock.NewController(t)
-	client := NewMockCopilotClient(ctrl)
-
-	client.EXPECT().Start(gomock.Any())
-	client.EXPECT().Stop().Return(nil)
+	client := newClientMock(ctrl)
 
 	rootDir := t.TempDir()
 	calledRoot := ""
@@ -203,16 +200,14 @@ func TestNewTaskFromPromptCommand_DiscoverErrorReturned(t *testing.T) {
 
 func TestNewTaskFromPromptCommand_DiscoveredSkillsPassedToCopilotSession(t *testing.T) {
 	ctrl := gomock.NewController(t)
-	client := NewMockCopilotClient(ctrl)
+	client := newClientMock(ctrl)
 
 	skillDir := t.TempDir()
 
-	client.EXPECT().Start(gomock.Any())
 	client.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, cfg *copilot.SessionConfig) (execution.CopilotSession, error) {
 		assert.Contains(t, cfg.SkillDirectories, skillDir)
 		return nil, errors.New("create failed")
 	})
-	client.EXPECT().Stop().Return(nil)
 
 	cmd := newTaskFromPromptCmd(&newTaskFromPromptCmdOptions{
 		NewTaskList: func(*ux.TaskListOptions) taskList { return &fakeTaskList{runAll: true} },
@@ -236,7 +231,7 @@ func TestNewTaskFromPromptCommand_DiscoveredSkillsPassedToCopilotSession(t *test
 
 func TestNewTaskFromPromptCommand_EndToEndCreatesTaskFile(t *testing.T) {
 	ctrl := gomock.NewController(t)
-	client := NewMockCopilotClient(ctrl)
+	client := newClientMock(ctrl)
 	session := NewMockCopilotSession(ctrl)
 
 	sessionID := "session-end-to-end"
@@ -251,7 +246,6 @@ func TestNewTaskFromPromptCommand_EndToEndCreatesTaskFile(t *testing.T) {
 	require.NoError(t, os.MkdirAll(filepath.Dir(logPath), 0o755))
 	require.NoError(t, os.WriteFile(logPath, fixtureBytes, 0o644))
 
-	client.EXPECT().Start(gomock.Any())
 	client.EXPECT().CreateSession(gomock.Any(), gomock.Any()).Return(session, nil)
 
 	session.EXPECT().On(gomock.Any()).Return(func() {}).Times(3)
@@ -260,7 +254,6 @@ func TestNewTaskFromPromptCommand_EndToEndCreatesTaskFile(t *testing.T) {
 	session.EXPECT().Disconnect().Return(nil)
 
 	client.EXPECT().DeleteSession(gomock.Any(), sessionID).Return(nil)
-	client.EXPECT().Stop().Return(nil)
 
 	outputPath := filepath.Join(t.TempDir(), "nested", "generated-task.yaml")
 
@@ -316,3 +309,16 @@ func TestNewTaskFromPromptCommand_EndToEndCreatesTaskFile(t *testing.T) {
 
 	require.Equal(t, expected, actual)
 }
+
+func newClientMock(ctrl *gomock.Controller) *MockCopilotClient {
+	clientMock := NewMockCopilotClient(ctrl)
+
+	// This is the basic sequence of calls that occurs anytime a copilot engine is initialized
+	clientMock.EXPECT().Start(gomock.Any()).Times(1)
+	clientMock.EXPECT().Stop().Times(1)
+	clientMock.EXPECT().GetAuthStatus(gomock.Any()).Return(&copilot.GetAuthStatusResponse{
+		IsAuthenticated: true,
+	}, nil).Times(1)
+
+	return clientMock
+}

cmd/waza/cmd_run.go

@@ -91,8 +91,9 @@ With no arguments, uses workspace detection to find eval.yaml automatically:
 
 You can also specify a skill name to run its eval:
   waza run code-explainer`,
-		Args: cobra.MaximumNArgs(1),
-		RunE: runCommandE,
+		Args:          cobra.MaximumNArgs(1),
+		RunE:          runCommandE,
+		SilenceErrors: true,
 	}
 
 	cmd.Flags().StringVar(&contextDir, "context-dir", "", "Context directory for fixtures (default: ./fixtures relative to spec)")
@@ -219,7 +220,7 @@ func runCommandE(cmd *cobra.Command, args []string) error {
 				skillFolders = append(skillFolders, ds.Dir)
 			}
 
-			slog.Info("Workspace skills added", "skills", skillFolders, "base", skillsPath)
+			slog.Debug("Workspace skills added", "skills", skillFolders, "base", skillsPath)
 		}
 	}
 

cmd/waza/cmd_run_test.go

@@ -2360,35 +2360,33 @@ func testWazaRun(t *testing.T, cwd string, args []string) (evalNames []string, s
 
 	// This helper derives which evals were selected by inspecting EvaluationOutcome.BenchName in the output folder.
 	ctrl := gomock.NewController(t)
-	client := NewMockCopilotClient(ctrl)
-	sess := NewMockCopilotSession(ctrl)
 
-	// currently, each run of an eval spec requires us a new copilot engine.
-	client.EXPECT().Start(gomock.Any()).AnyTimes()
-	client.EXPECT().Stop().AnyTimes()
+	newCopilotClientFn = func(clientOptions *copilot.ClientOptions) execution.CopilotClient {
+		client := newClientMock(ctrl)
+		sess := NewMockCopilotSession(ctrl)
 
-	client.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, config *copilot.SessionConfig) (execution.CopilotSession, error) {
-		require.NotEmpty(t, config.SkillDirectories, "all of our tests expect some skills to be found")
+		// currently, each run of an eval spec requires us a new copilot engine.
+		client.EXPECT().CreateSession(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, config *copilot.SessionConfig) (execution.CopilotSession, error) {
+			require.NotEmpty(t, config.SkillDirectories, "all of our tests expect some skills to be found")
 
-		for _, sp := range config.SkillDirectories {
-			skillsLoaded = append(skillsLoaded, filepath.Base(sp))
-		}
+			for _, sp := range config.SkillDirectories {
+				skillsLoaded = append(skillsLoaded, filepath.Base(sp))
+			}
 
-		return sess, nil
-	}).AnyTimes()
+			return sess, nil
+		}).AnyTimes()
 
-	sessionIDCounter := &atomic.Int32{}
+		sessionIDCounter := &atomic.Int32{}
 
-	sess.EXPECT().SessionID().DoAndReturn(func() string {
-		id := sessionIDCounter.Add(1)
-		return fmt.Sprintf("ID: %d", id)
-	}).AnyTimes()
-	sess.EXPECT().SendAndWait(gomock.Any(), gomock.Any()).AnyTimes()
-	sess.EXPECT().On(gomock.Any()).Return(func() {}).AnyTimes()
-	sess.EXPECT().Disconnect().AnyTimes()
-	client.EXPECT().DeleteSession(gomock.Any(), gomock.Any()).AnyTimes()
+		sess.EXPECT().SessionID().DoAndReturn(func() string {
+			id := sessionIDCounter.Add(1)
+			return fmt.Sprintf("ID: %d", id)
+		}).AnyTimes()
+		sess.EXPECT().SendAndWait(gomock.Any(), gomock.Any()).AnyTimes()
+		sess.EXPECT().On(gomock.Any()).Return(func() {}).AnyTimes()
+		sess.EXPECT().Disconnect().AnyTimes()
+		client.EXPECT().DeleteSession(gomock.Any(), gomock.Any()).AnyTimes()
 
-	newCopilotClientFn = func(clientOptions *copilot.ClientOptions) execution.CopilotClient {
 		return client
 	}
 

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/charmbracelet/huh v1.0.0
 	github.com/github/copilot-sdk/go v0.1.32
 	github.com/go-viper/mapstructure/v2 v2.5.0
+	github.com/klauspost/compress v1.18.3
 	github.com/mattn/go-runewidth v0.0.21
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
 	github.com/spf13/cobra v1.10.2
@@ -111,4 +112,7 @@ require (
 	google.golang.org/protobuf v1.36.11 // indirect
 )
 
-tool go.uber.org/mock/mockgen
+tool (
+	github.com/github/copilot-sdk/go/cmd/bundler
+	go.uber.org/mock/mockgen
+)

go.sum

@@ -159,6 +159,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=

internal/embedded/generate/generate.go

@@ -0,0 +1,93 @@
+// Generates all the proper copilot CLI SDK bundles, so we can use them in waza.
+// The .zst, .license and generated .go files should all be checked in. When waza is built
+// only the relevant copilot CLI package will be added.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/sync/errgroup"
+)
+
+//go:generate go run . windows/arm64 windows/amd64 linux/arm64 linux/amd64 darwin/arm64 darwin/amd64
+
+func main() {
+	g := errgroup.Group{}
+
+	platforms := os.Args[1:]
+
+	fmt.Printf("Generating the following platforms:\n")
+
+	for _, p := range platforms {
+		fmt.Println(p)
+	}
+
+	fmt.Println("Starting...")
+
+	outputDir := ".."
+
+	for _, arg := range platforms {
+		g.Go(func() error {
+			cmd := exec.Command("go", "tool", "bundler", "-platform", arg, "-output", outputDir)
+
+			fmt.Printf("Running %s\n", cmd.String())
+
+			output, err := cmd.CombinedOutput()
+
+			if err != nil {
+				return fmt.Errorf("bundler failed (with output %s): %w", string(output), err)
+			}
+
+			// once it's finished we just need to slap the proper package name on it.
+			platformParts := strings.Split(arg, "/")
+
+			if len(platformParts) != 2 {
+				return fmt.Errorf("bad format for platform %q. Platforms should be <GOOS compatible OS>/<GOARCH compatible arch> (ex: windows/amd64 )", arg)
+			}
+
+			path := filepath.Join(outputDir, fmt.Sprintf("zcopilot_%s_%s.go", platformParts[0], platformParts[1]))
+
+			fmt.Printf("Patching %s's package directive\n", path)
+			return fixCopilotPackageInGoFile(path)
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		fmt.Printf("Failed to generate bundles: %s\n", err)
+		os.Exit(1)
+	} else {
+		fmt.Println("Done, no errors")
+		fmt.Println("You must delete any older .zst or .license files, manually")
+	}
+}
+
+func fixCopilotPackageInGoFile(goFile string) error {
+	contents, err := os.ReadFile(goFile)
+
+	if err != nil {
+		return fmt.Errorf("failed to read %q to fix the package: %w", goFile, err)
+	}
+
+	contents = bytes.Replace(contents, []byte("package main"), []byte("package embedded"), 1)
+
+	err = os.WriteFile(goFile, contents, 0644)
+
+	if err != nil {
+		return fmt.Errorf("failed to rewrite %q to fix the package: %w", goFile, err)
+	}
+
+	cmd := exec.Command("gofmt", "-w", goFile)
+	stdout, err := cmd.CombinedOutput()
+
+	if err != nil {
+		return fmt.Errorf("failed to gofmt %q. output: %s: %w", goFile, stdout, err)
+	}
+
+	return nil
+}