Make agent reload test runtime-compatible (#1201)

* Make agent reload test runtime-compatible

Use a unique reload agent name and assert reload results match the subsequent list result instead of depending on whether the runtime preserves session-configured agents after reload.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Align agent reload tests across SDKs

Apply the same runtime-compatible reload test pattern to Go and Python by using unique reload agent names and asserting reload results match subsequent agent list results.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Add Node agent reload E2E coverage

Add the Node.js counterpart for the runtime-compatible agent reload RPC test, including a unique agent name and reload/list consistency assertion.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: stephentoub

dotnet/test/E2E/RpcAgentE2ETests.cs

@@ -2,6 +2,7 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *--------------------------------------------------------------------------------------------*/
 
+using GitHub.Copilot.SDK.Rpc;
 using GitHub.Copilot.SDK.Test.Harness;
 using Xunit;
 using Xunit.Abstractions;
@@ -104,10 +105,11 @@ public async Task Should_Return_Empty_List_When_No_Custom_Agents_Configured()
     [Fact]
     public async Task Should_Call_Agent_Reload()
     {
-        var session = await CreateSessionAsync(new SessionConfig { CustomAgents = [CreateReloadAgent()] });
+        var reloadAgent = CreateReloadAgent($"reload-test-agent-{Guid.NewGuid():N}");
+        var session = await CreateSessionAsync(new SessionConfig { CustomAgents = [reloadAgent] });
 
         var before = await session.Rpc.Agent.ListAsync();
-        Assert.Single(before.Agents, agent => string.Equals(agent.Name, "reload-test-agent", StringComparison.Ordinal));
+        AssertReloadAgent(before.Agents, reloadAgent);
 
         var result = await session.Rpc.Agent.ReloadAsync();
         var current = await session.Rpc.Agent.ListAsync();
@@ -120,6 +122,13 @@ public async Task Should_Call_Agent_Reload()
             current.Agents.Select(agent => agent.DisplayName).OrderBy(name => name, StringComparer.Ordinal));
     }
 
+    private static void AssertReloadAgent(IEnumerable<AgentInfo> agents, CustomAgentConfig expected)
+    {
+        var agent = Assert.Single(agents, agent => string.Equals(agent.Name, expected.Name, StringComparison.Ordinal));
+        Assert.Equal(expected.DisplayName, agent.DisplayName);
+        Assert.Equal(expected.Description, agent.Description);
+    }
+
     private static List<CustomAgentConfig> CreateCustomAgents() =>
     [
         new()
@@ -138,10 +147,10 @@ private static List<CustomAgentConfig> CreateCustomAgents() =>
         }
     ];
 
-    private static CustomAgentConfig CreateReloadAgent() =>
+    private static CustomAgentConfig CreateReloadAgent(string name) =>
         new()
         {
-            Name = "reload-test-agent",
+            Name = name,
             DisplayName = "Reload Test Agent",
             Description = "Used by the agent reload RPC test.",
             Prompt = "You are a reload test agent.",

go/internal/e2e/agent_and_compact_rpc_e2e_test.go

@@ -1,7 +1,10 @@
 package e2e
 
 import (
+	"fmt"
+	"slices"
 	"testing"
+	"time"
 
 	copilot "github.com/github/copilot-sdk/go"
 	"github.com/github/copilot-sdk/go/internal/e2e/testharness"
@@ -253,6 +256,13 @@ func TestAgentSelectionRpcE2E(t *testing.T) {
 	})
 
 	t.Run("should call agent reload", func(t *testing.T) {
+		reloadAgent := copilot.CustomAgentConfig{
+			Name:        fmt.Sprintf("reload-test-agent-%d", time.Now().UnixNano()),
+			DisplayName: "Reload Test Agent",
+			Description: "Used by the agent reload RPC test.",
+			Prompt:      "You are a reload test agent.",
+		}
+
 		client := copilot.NewClient(&copilot.ClientOptions{
 			CLIPath:  cliPath,
 			UseStdio: copilot.Bool(true),
@@ -266,12 +276,7 @@ func TestAgentSelectionRpcE2E(t *testing.T) {
 		session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
 			OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
 			CustomAgents: []copilot.CustomAgentConfig{
-				{
-					Name:        "reload-test-agent",
-					DisplayName: "Reload Test Agent",
-					Description: "Used by the agent reload RPC test.",
-					Prompt:      "You are a reload test agent.",
-				},
+				reloadAgent,
 			},
 		})
 		if err != nil {
@@ -282,35 +287,58 @@ func TestAgentSelectionRpcE2E(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to list agents: %v", err)
 		}
-		var sawReloadAgent bool
-		for _, agent := range before.Agents {
-			if agent.Name == "reload-test-agent" {
-				sawReloadAgent = true
-				break
-			}
-		}
-		if !sawReloadAgent {
-			t.Fatalf("Expected reload-test-agent in initial Agent.List, got %+v", before.Agents)
-		}
+		assertReloadAgent(t, before.Agents, reloadAgent)
 
-		// Reload should succeed; the runtime currently drops session-configured
-		// CustomAgents on reload, so we only assert the result shape is non-nil.
-		// Once that runtime behavior is fixed, tighten this to assert
-		// reload-test-agent is still present.
 		result, err := session.RPC.Agent.Reload(t.Context())
 		if err != nil {
 			t.Fatalf("Failed to reload agents: %v", err)
 		}
 		if result.Agents == nil {
 			t.Errorf("Expected non-nil Agents after reload")
 		}
+		current, err := session.RPC.Agent.List(t.Context())
+		if err != nil {
+			t.Fatalf("Failed to list agents after reload: %v", err)
+		}
+		if got, want := agentSummaries(result.Agents), agentSummaries(current.Agents); !slices.Equal(got, want) {
+			t.Errorf("Expected reload result agents to match current agents.\nGot:  %v\nWant: %v", got, want)
+		}
 
 		if err := client.Stop(); err != nil {
 			t.Errorf("Expected no errors on stop, got %v", err)
 		}
 	})
 }
 
+func assertReloadAgent(t *testing.T, agents []rpc.AgentInfo, expected copilot.CustomAgentConfig) {
+	t.Helper()
+
+	var matches []rpc.AgentInfo
+	for _, agent := range agents {
+		if agent.Name == expected.Name {
+			matches = append(matches, agent)
+		}
+	}
+	if len(matches) != 1 {
+		t.Fatalf("Expected exactly one %q in Agent.List, got %+v", expected.Name, agents)
+	}
+	if matches[0].DisplayName != expected.DisplayName {
+		t.Errorf("Expected reload agent display name %q, got %q", expected.DisplayName, matches[0].DisplayName)
+	}
+	if matches[0].Description != expected.Description {
+		t.Errorf("Expected reload agent description %q, got %q", expected.Description, matches[0].Description)
+	}
+}
+
+func agentSummaries(agents []rpc.AgentInfo) []string {
+	summaries := make([]string, len(agents))
+	for i, agent := range agents {
+		summaries[i] = fmt.Sprintf("%s\x00%s", agent.Name, agent.DisplayName)
+	}
+	slices.Sort(summaries)
+	return summaries
+}
+
 func TestSessionCompactionRpcE2E(t *testing.T) {
 	ctx := testharness.NewTestContext(t)
 	client := ctx.NewClient()

nodejs/test/e2e/agent_and_compact_rpc.e2e.test.ts

@@ -2,6 +2,7 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *--------------------------------------------------------------------------------------------*/
 
+import { randomUUID } from "node:crypto";
 import { describe, expect, it } from "vitest";
 import { approveAll } from "../../src/index.js";
 import type { CustomAgentConfig } from "../../src/index.js";
@@ -127,6 +128,34 @@ describe("Agent Selection RPC", async () => {
 
         await session.disconnect();
     });
+
+    it("should call agent reload", async () => {
+        const reloadAgent: CustomAgentConfig = {
+            name: `reload-test-agent-${randomUUID().replaceAll("-", "")}`,
+            displayName: "Reload Test Agent",
+            description: "Used by the agent reload RPC test.",
+            prompt: "You are a reload test agent.",
+        };
+
+        const session = await client.createSession({
+            onPermissionRequest: approveAll,
+            customAgents: [reloadAgent],
+        });
+
+        const before = await session.rpc.agent.list();
+        const match = before.agents.find((agent) => agent.name === reloadAgent.name);
+        expect(match).toBeDefined();
+        expect(match!.displayName).toBe(reloadAgent.displayName);
+        expect(match!.description).toBe(reloadAgent.description);
+
+        const result = await session.rpc.agent.reload();
+        expect(result.agents).toBeDefined();
+
+        const current = await session.rpc.agent.list();
+        expect(summarizeAgents(result.agents)).toEqual(summarizeAgents(current.agents));
+
+        await session.disconnect();
+    });
 });
 
 describe("Session Compact RPC", async () => {
@@ -147,3 +176,7 @@ describe("Session Compact RPC", async () => {
         await session.disconnect();
     }, 60000);
 });
+
+function summarizeAgents(agents: { name: string; displayName: string }[]) {
+    return agents.map((agent) => `${agent.name}\x00${agent.displayName}`).sort();
+}

python/e2e/test_agent_and_compact_rpc_e2e.py

@@ -1,5 +1,7 @@
 """E2E tests for Agent Selection and Session Compaction RPC APIs."""
 
+import uuid
+
 import pytest
 
 from copilot import CopilotClient
@@ -174,36 +176,45 @@ async def test_should_return_empty_list_when_no_custom_agents_configured(self):
     async def test_should_call_agent_reload(self):
         """Test reloading agents via RPC."""
         client = CopilotClient(SubprocessConfig(cli_path=CLI_PATH, use_stdio=True))
+        reload_agent = {
+            "name": f"reload-test-agent-{uuid.uuid4().hex}",
+            "display_name": "Reload Agent",
+            "description": "An agent used to validate reload",
+            "prompt": "You are a reload test agent.",
+        }
 
         try:
             await client.start()
             session = await client.create_session(
                 on_permission_request=PermissionHandler.approve_all,
-                custom_agents=[
-                    {
-                        "name": "reload-test-agent",
-                        "display_name": "Reload Agent",
-                        "description": "An agent used to validate reload",
-                        "prompt": "You are a reload test agent.",
-                    }
-                ],
+                custom_agents=[reload_agent],
             )
 
             before = await session.rpc.agent.list()
-            assert any(agent.name == "reload-test-agent" for agent in before.agents)
+            _assert_reload_agent(before.agents, reload_agent)
 
-            # Reload should succeed and return some agent set. The CLI currently
-            # drops session-configured CustomAgents on reload, so we don't
-            # require the reload-test-agent to remain present after reload.
             result = await session.rpc.agent.reload()
             assert result.agents is not None
+            current = await session.rpc.agent.list()
+            assert _agent_summaries(result.agents) == _agent_summaries(current.agents)
 
             await session.disconnect()
             await client.stop()
         finally:
             await client.force_stop()
 
 
+def _assert_reload_agent(agents, expected):
+    matches = [agent for agent in agents if agent.name == expected["name"]]
+    assert len(matches) == 1
+    assert matches[0].display_name == expected["display_name"]
+    assert matches[0].description == expected["description"]
+
+
+def _agent_summaries(agents):
+    return sorted((agent.name, agent.display_name) for agent in agents)
+
+
 class TestSessionCompactionRpc:
     @pytest.mark.asyncio
     async def test_should_compact_session_history_after_messages(self, ctx: E2ETestContext):