fix(telemetry): consistent invocation ID per run and add telemetry test infra

- root.go: set invocation ID only when empty so all API requests in one CLI run
  share the same invocation_id (fixes multiple IDs when gateway PreRun calls
  ValidateAPIKey then connection PreRun overwrote telemetry)
- pkg/cmd: add TestInvocationIDPersistsAcrossMultipleInitTelemetryCalls (fails
  without fix, passes with it)
- test/acceptance: recording proxy (StartRecordingProxy, AssertTelemetryConsistent,
  RunListenWithTimeout, RunWithEnv, NewCLIRunnerWithConfigPath/NoCI), telemetry_test.go
  proxy-based tests, TestTelemetryInvocationIDConsistentWhenValidateAPIKeyInPreRun,
  TestTelemetryListenProxy in listen_test.go, README section

Made-with: Cursor

Author: leggetter

pkg/cmd/root.go

@@ -51,7 +51,9 @@ func initTelemetry(cmd *cobra.Command) {
 	tel.SetEnvironment(hookdeck.DetectEnvironment())
 	tel.SetCommandContext(cmd)
 	tel.SetDeviceName(Config.DeviceName)
-	tel.SetInvocationID(hookdeck.NewInvocationID())
+	if tel.InvocationID == "" {
+		tel.SetInvocationID(hookdeck.NewInvocationID())
+	}
 }
 
 // RootCmd returns the root command for use by tools (e.g. generate-reference).

pkg/cmd/telemetry_test.go

@@ -86,6 +86,39 @@ func TestInitTelemetryResetBetweenCalls(t *testing.T) {
 	require.NotEqual(t, id1, tel2.InvocationID)
 }
 
+// TestInvocationIDPersistsAcrossMultipleInitTelemetryCalls reproduces the v2.0.0 bug: when
+// initTelemetry is called multiple times in one process (e.g. root PreRun, then gateway PreRun,
+// then connection PreRun), the invocation_id must stay the same so all API requests in that run
+// share one ID. Without the fix, each initTelemetry call overwrites with a new ID; this test
+// asserts the ID is unchanged after a second call. On v2.0.0 (repro branch) it fails.
+func TestInvocationIDPersistsAcrossMultipleInitTelemetryCalls(t *testing.T) {
+	hookdeck.ResetTelemetryInstanceForTesting()
+
+	root := &cobra.Command{Use: "hookdeck"}
+	gateway := &cobra.Command{Use: "gateway"}
+	connection := &cobra.Command{Use: "connection"}
+	root.AddCommand(gateway)
+	gateway.AddCommand(connection)
+
+	Config.DeviceName = "test-device"
+	initTelemetry(root)
+	tel := hookdeck.GetTelemetryInstance()
+	firstID := tel.InvocationID
+	require.NotEmpty(t, firstID, "first initTelemetry must set invocation_id")
+
+	// Second call simulates gateway PreRun; must not overwrite invocation_id.
+	initTelemetry(gateway)
+	tel = hookdeck.GetTelemetryInstance()
+	require.Equal(t, firstID, tel.InvocationID,
+		"invocation_id must persist across initTelemetry calls in the same process (fix: set only when empty)")
+
+	// Third call simulates connection PreRun; must still not overwrite.
+	initTelemetry(connection)
+	tel = hookdeck.GetTelemetryInstance()
+	require.Equal(t, firstID, tel.InvocationID,
+		"invocation_id must persist after third initTelemetry call")
+}
+
 // TestInitTelemetryWhenDisabled verifies that initTelemetry always populates the
 // singleton (Source, CommandPath, etc.) even when telemetry is disabled. The
 // call must happen for every command; PerformRequest later skips sending the

test/acceptance/README.md

@@ -18,6 +18,12 @@ These tests require browser-based authentication via `hookdeck login` and must b
 
 **Why Manual?** These tests access endpoints (like `/teams`) that require CLI authentication keys obtained through interactive browser login, which aren't available to CI service accounts.
 
+### Recording proxy (telemetry tests)
+
+Some tests (e.g. `TestTelemetryGatewayConnectionListProxy` in `telemetry_test.go`) use a **recording proxy**: the CLI is run with `--api-base` pointing at a local HTTP server that forwards every request to the real Hookdeck API and records method, path, and the `X-Hookdeck-CLI-Telemetry` header. The same `CLIRunner` and `go run main.go` flow are used as in other acceptance tests; only the API base URL is overridden so traffic goes through the proxy. This verifies that a single CLI run sends consistent telemetry (same `invocation_id` and `command_path`) on all API calls. Helpers: `StartRecordingProxy`, `AssertTelemetryConsistent`.
+
+**Login telemetry test (TestTelemetryLoginProxy)** uses the same proxy approach: it runs `hookdeck login --api-key KEY` with `--api-base` set to the proxy, and asserts exactly one recorded request (GET `/2025-07-01/cli-auth/validate`) with consistent telemetry. It uses **HOOKDECK_CLI_TESTING_CLI_KEY** (not the API/CI key), because the validate endpoint accepts CLI keys from interactive login; if unset, the test is skipped. Other telemetry tests still use the normal API key via `NewCLIRunner`.
+
 ## Setup
 
 ### Local Development

test/acceptance/helpers.go

@@ -5,18 +5,162 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
+	"net/http/httptest"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// defaultAPIUpstream is the real Hookdeck API base URL used by the recording proxy.
+const defaultAPIUpstream = "https://api.hookdeck.com"
+
+// RecordedRequest holds a single HTTP request as captured by the recording proxy.
+type RecordedRequest struct {
+	Method    string
+	Path      string
+	Telemetry string
+}
+
+// RecordingProxy is an HTTP server that forwards requests to the real API and
+// records each request (including X-Hookdeck-CLI-Telemetry). Use it to run the
+// CLI with --api-base pointing at the proxy so all API traffic is captured
+// while still hitting the real backend.
+type RecordingProxy struct {
+	t        *testing.T
+	server   *httptest.Server
+	upstream string
+	mu       sync.Mutex
+	recorded []RecordedRequest
+}
+
+// URL returns the proxy base URL (e.g. http://127.0.0.1:port). Pass this to
+// the CLI as --api-base so requests go through the proxy.
+func (p *RecordingProxy) URL() string {
+	return p.server.URL
+}
+
+// Recorded returns a copy of the slice of recorded requests. Safe to call
+// after the CLI command has finished.
+func (p *RecordingProxy) Recorded() []RecordedRequest {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	out := make([]RecordedRequest, len(p.recorded))
+	copy(out, p.recorded)
+	return out
+}
+
+// Close shuts down the proxy server.
+func (p *RecordingProxy) Close() {
+	p.server.Close()
+}
+
+// StartRecordingProxy starts an httptest.Server that acts as a reverse proxy to
+// upstreamBase (e.g. https://api.hookdeck.com). Every request is recorded
+// (method, path, X-Hookdeck-CLI-Telemetry) and then forwarded to the upstream;
+// the upstream response is returned to the client. Use with CLIRunner.Run("--api-base", proxy.URL(), "gateway", ...).
+func StartRecordingProxy(t *testing.T, upstreamBase string) *RecordingProxy {
+	t.Helper()
+	upstream, err := url.Parse(strings.TrimSuffix(upstreamBase, "/"))
+	require.NoError(t, err, "parse upstream URL")
+
+	p := &RecordingProxy{
+		t:        t,
+		upstream: upstream.String(),
+		recorded: make([]RecordedRequest, 0),
+	}
+
+	p.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Record before forwarding
+		p.mu.Lock()
+		p.recorded = append(p.recorded, RecordedRequest{
+			Method:    r.Method,
+			Path:      r.URL.Path,
+			Telemetry: r.Header.Get("X-Hookdeck-CLI-Telemetry"),
+		})
+		p.mu.Unlock()
+
+		// Build upstream request: same method, path, query, body
+		targetPath := r.URL.Path
+		if r.URL.RawQuery != "" {
+			targetPath += "?" + r.URL.RawQuery
+		}
+		dest, err := url.Parse(upstream.String() + targetPath)
+		require.NoError(p.t, err)
+
+		var bodyReader io.Reader
+		if r.Body != nil {
+			bodyReader = r.Body
+		}
+
+		req, err := http.NewRequest(r.Method, dest.String(), bodyReader)
+		require.NoError(p.t, err)
+
+		// Copy headers that the API cares about
+		for _, k := range []string{
+			"Authorization", "Content-Type", "X-Team-ID", "X-Project-ID",
+			"X-Hookdeck-CLI-Telemetry", "X-Hookdeck-Client-User-Agent", "User-Agent",
+		} {
+			if v := r.Header.Get(k); v != "" {
+				req.Header.Set(k, v)
+			}
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(p.t, err)
+		defer resp.Body.Close()
+
+		// Copy response back
+		for k, v := range resp.Header {
+			for _, vv := range v {
+				w.Header().Add(k, vv)
+			}
+		}
+		w.WriteHeader(resp.StatusCode)
+		_, _ = io.Copy(w, resp.Body)
+	}))
+
+	return p
+}
+
+// telemetryPayload is the structure of the X-Hookdeck-CLI-Telemetry header (JSON).
+type telemetryPayload struct {
+	CommandPath  string `json:"command_path"`
+	InvocationID string `json:"invocation_id"`
+}
+
+// AssertTelemetryConsistent checks that every recorded request that has a
+// telemetry header shares the same invocation_id and command_path, and that
+// command_path equals expectedCommandPath.
+func AssertTelemetryConsistent(t *testing.T, recorded []RecordedRequest, expectedCommandPath string) {
+	t.Helper()
+	var invocationID, commandPath string
+	for i, r := range recorded {
+		if r.Telemetry == "" {
+			continue
+		}
+		var p telemetryPayload
+		require.NoError(t, json.Unmarshal([]byte(r.Telemetry), &p), "request %d: invalid telemetry JSON: %s", i, r.Telemetry)
+		if invocationID == "" {
+			invocationID = p.InvocationID
+			commandPath = p.CommandPath
+		}
+		require.Equal(t, invocationID, p.InvocationID, "request %d (%s %s): invocation_id should be consistent", i, r.Method, r.Path)
+		require.Equal(t, commandPath, p.CommandPath, "request %d (%s %s): command_path should be consistent", i, r.Method, r.Path)
+	}
+	require.Equal(t, expectedCommandPath, commandPath, "command_path should match expected")
+	require.NotEmpty(t, invocationID, "at least one request should have invocation_id")
+}
+
 func init() {
 	// Attempt to load .env file from test/acceptance/.env for local development
 	// In CI, the environment variable will be set directly
@@ -86,6 +230,43 @@ func NewCLIRunner(t *testing.T) *CLIRunner {
 	return runner
 }
 
+// NewCLIRunnerWithConfigPath creates a CLI runner that uses the given config file path.
+// It runs "ci --api-key" so the config is populated (api_key, project_id, project_mode, etc.).
+// Use this when a test needs a known config path (e.g. to write a minimal variant for another run).
+func NewCLIRunnerWithConfigPath(t *testing.T, configPath string) *CLIRunner {
+	t.Helper()
+	apiKey := getAcceptanceAPIKey(t)
+	require.NotEmpty(t, apiKey, "HOOKDECK_CLI_TESTING_API_KEY must be set")
+	projectRoot, err := filepath.Abs("../..")
+	require.NoError(t, err, "Failed to get project root path")
+	runner := &CLIRunner{
+		t:           t,
+		apiKey:      apiKey,
+		projectRoot: projectRoot,
+		configPath:  configPath,
+	}
+	stdout, stderr, err := runner.Run("ci", "--api-key", apiKey)
+	require.NoError(t, err, "Failed to authenticate CLI with config at %s: stdout=%s stderr=%s", configPath, stdout, stderr)
+	return runner
+}
+
+// NewCLIRunnerWithConfigPathNoCI creates a CLI runner that uses the given config file path,
+// without running "ci". Use when the config file is already populated (e.g. a minimal config
+// written by the test). The runner will pass HOOKDECK_CONFIG_FILE to all Run() calls.
+func NewCLIRunnerWithConfigPathNoCI(t *testing.T, configPath string) *CLIRunner {
+	t.Helper()
+	apiKey := getAcceptanceAPIKey(t)
+	require.NotEmpty(t, apiKey, "HOOKDECK_CLI_TESTING_API_KEY must be set")
+	projectRoot, err := filepath.Abs("../..")
+	require.NoError(t, err, "Failed to get project root path")
+	return &CLIRunner{
+		t:           t,
+		apiKey:      apiKey,
+		projectRoot: projectRoot,
+		configPath:  configPath,
+	}
+}
+
 // getAcceptanceAPIKey returns the API key for the current acceptance slice.
 // When ACCEPTANCE_SLICE=1 and HOOKDECK_CLI_TESTING_API_KEY_2 is set, use it; when ACCEPTANCE_SLICE=2 and HOOKDECK_CLI_TESTING_API_KEY_3 is set, use that; else HOOKDECK_CLI_TESTING_API_KEY.
 func getAcceptanceAPIKey(t *testing.T) string {
@@ -198,6 +379,80 @@ func (r *CLIRunner) Run(args ...string) (stdout, stderr string, err error) {
 	return stdoutBuf.String(), stderrBuf.String(), err
 }
 
+// RunWithEnv is like Run but merges extraEnv into the process environment (e.g. for HOOKDECK_CLI_USE_SYSTEM_BINARY).
+// When extraEnv["HOOKDECK_CLI_USE_SYSTEM_BINARY"] == "1", runs the installed "hookdeck" binary on PATH instead of "go run main.go"
+// (e.g. to run tests against 2.0.0 or another installed version).
+func (r *CLIRunner) RunWithEnv(extraEnv map[string]string, args ...string) (stdout, stderr string, err error) {
+	r.t.Helper()
+
+	env := os.Environ()
+	if r.configPath != "" {
+		env = appendEnvOverride(env, "HOOKDECK_CONFIG_FILE", r.configPath)
+	}
+	for k, v := range extraEnv {
+		env = appendEnvOverride(env, k, v)
+	}
+
+	var cmd *exec.Cmd
+	if extraEnv != nil && extraEnv["HOOKDECK_CLI_USE_SYSTEM_BINARY"] == "1" {
+		hookdeckPath, lookErr := exec.LookPath("hookdeck")
+		if lookErr != nil {
+			return "", "", fmt.Errorf("HOOKDECK_CLI_USE_SYSTEM_BINARY=1 but hookdeck not on PATH: %w", lookErr)
+		}
+		cmd = exec.Command(hookdeckPath, args...)
+		cmd.Dir = r.projectRoot
+	} else {
+		mainGoPath := filepath.Join(r.projectRoot, "main.go")
+		cmdArgs := append([]string{"run", mainGoPath}, args...)
+		cmd = exec.Command("go", cmdArgs...)
+		cmd.Dir = r.projectRoot
+	}
+	cmd.Env = env
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+	cmd.Stdout = &stdoutBuf
+	cmd.Stderr = &stderrBuf
+	err = cmd.Run()
+	return stdoutBuf.String(), stderrBuf.String(), err
+}
+
+// RunListenWithTimeout starts the CLI with the given args (e.g. "--api-base", proxyURL,
+// "listen", port, sourceName, "--output", "compact"), lets it run for runDuration, then
+// kills the process. Uses a pre-built binary so we terminate the actual listen process
+// (not a "go run" parent). Returns stdout, stderr, and the error from Wait (often
+// non-nil because the process was killed). Uses the same project root and config env as Run().
+func (r *CLIRunner) RunListenWithTimeout(args []string, runDuration time.Duration) (stdout, stderr string, err error) {
+	r.t.Helper()
+	tmpBinary := filepath.Join(r.projectRoot, "hookdeck-listen-test-"+generateTimestamp())
+	defer os.Remove(tmpBinary)
+
+	buildCmd := exec.Command("go", "build", "-o", tmpBinary, ".")
+	buildCmd.Dir = r.projectRoot
+	if err := buildCmd.Run(); err != nil {
+		return "", "", fmt.Errorf("build CLI for listen test: %w", err)
+	}
+
+	cmd := exec.Command(tmpBinary, args...)
+	cmd.Dir = r.projectRoot
+	if r.configPath != "" {
+		cmd.Env = appendEnvOverride(os.Environ(), "HOOKDECK_CONFIG_FILE", r.configPath)
+	}
+	var stdoutBuf, stderrBuf bytes.Buffer
+	cmd.Stdout = &stdoutBuf
+	cmd.Stderr = &stderrBuf
+	if err := cmd.Start(); err != nil {
+		return "", "", err
+	}
+	timer := time.AfterFunc(runDuration, func() {
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+		}
+	})
+	defer timer.Stop()
+	waitErr := cmd.Wait()
+	return stdoutBuf.String(), stderrBuf.String(), waitErr
+}
+
 // RunFromCwd executes the CLI from the current working directory.
 // This is useful for tests that need to test --local flag behavior,
 // which creates config in the current directory.

test/acceptance/listen_test.go

@@ -163,3 +163,36 @@ func TestListenCommandWithContext(t *testing.T) {
 
 	t.Logf("Listen command terminated via context cancellation")
 }
+
+// TestTelemetryListenProxy runs "listen" with the CLI pointed at a recording proxy
+// that forwards to the real API. It lets listen run for a few seconds (so it can
+// perform initial API calls: get sources, get connections, create session), then
+// stops it. Asserts that every API request in that run has the same invocation_id
+// and command_path "hookdeck listen".
+func TestTelemetryListenProxy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping acceptance test in short mode")
+	}
+
+	cli := NewCLIRunner(t)
+	proxy := StartRecordingProxy(t, defaultAPIUpstream)
+	defer proxy.Close()
+
+	timestamp := generateTimestamp()
+	sourceName := "test-telemetry-" + timestamp
+
+	_, _, _ = cli.RunListenWithTimeout([]string{
+		"--api-base", proxy.URL(),
+		"listen", "9999", sourceName,
+		"--output", "compact",
+	}, 6*time.Second)
+	// Process is killed after 6s; we don't assert on exit error
+
+	recorded := proxy.Recorded()
+	require.GreaterOrEqual(t, len(recorded), 1,
+		"expected at least one API request from listen (sources, connections, or cli-sessions)")
+	for i, req := range recorded {
+		t.Logf("listen API request %d: %s %s (telemetry: %s)", i+1, req.Method, req.Path, req.Telemetry)
+	}
+	AssertTelemetryConsistent(t, recorded, "hookdeck listen")
+}