From b944097a8bec38fd2fbc87d1d0819b2ee5efd129 Mon Sep 17 00:00:00 2001 From: stringintech Date: Mon, 1 Dec 2025 14:35:42 +0330 Subject: [PATCH 1/6] Refactor handler lifecycle management with timeouts and auto-recovery Extract handler process management into a separate Handler type with: - Read timeout to prevent hanging on unresponsive handlers - Shutdown with fallback to force-kill on timeout - Automatic handler respawn when broken, allowing remaining tests to continue - Single handler instance reused across test suites --- cmd/runner/main.go | 17 +++--- runner/handler.go | 137 ++++++++++++++++++++++++++++++++++++++++++ runner/runner.go | 147 +++++++++++++++++++++------------------------ 3 files changed, 214 insertions(+), 87 deletions(-) create mode 100644 runner/handler.go diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 77a5729..ba25039 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -33,6 +33,14 @@ func main() { os.Exit(1) } + // Create test runner + testRunner, err := runner.NewTestRunner(*handlerPath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating test runner: %v\n", err) + os.Exit(1) + } + defer testRunner.CloseHandler() + // Run tests totalPassed := 0 totalFailed := 0 @@ -48,17 +56,8 @@ func main() { continue } - // Create test runner - testRunner, err := runner.NewTestRunner(*handlerPath) - if err != nil { - fmt.Fprintf(os.Stderr, "Error creating test runner: %v\n", err) - continue - } - // Run suite result := testRunner.RunTestSuite(*suite) - testRunner.Close() - printResults(suite, result) totalPassed += result.PassedTests diff --git a/runner/handler.go b/runner/handler.go new file mode 100644 index 0000000..daf3260 --- /dev/null +++ b/runner/handler.go @@ -0,0 +1,137 @@ +package runner + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "log/slog" + "os/exec" + "time" +) + +var ( + // ErrHandlerTimeout indicates the handler did not respond within the timeout + ErrHandlerTimeout = errors.New("handler timeout") + // ErrHandlerClosed indicates the handler closed stdout unexpectedly + ErrHandlerClosed = errors.New("handler closed unexpectedly") +) + +// Handler manages a conformance handler process communicating via stdin/stdout +type Handler struct { + cmd *exec.Cmd + stdin io.WriteCloser + stdout *bufio.Scanner + stderr io.ReadCloser +} + +// NewHandler spawns a new handler process at the given path +func NewHandler(path string) (*Handler, error) { + cmd := exec.Command(path) + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, fmt.Errorf("failed to create stdin pipe: %w", err) + } + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to create stdout pipe: %w", err) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to create stderr pipe: %w", err) + } + + // Start() automatically closes all pipes on failure, no manual cleanup needed + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start handler: %w", err) + } + + return &Handler{ + cmd: cmd, + stdin: stdin, + stdout: bufio.NewScanner(stdout), + stderr: stderr, + }, nil +} + +// SendLine writes a line to the handler's stdin +func (h *Handler) SendLine(line []byte) error { + _, err := h.stdin.Write(append(line, '\n')) + return err +} + +// ReadLine reads a line from the handler's stdout with a 10-second timeout +func (h *Handler) ReadLine() ([]byte, error) { + // Use a timeout for Scan() in case the handler hangs + scanDone := make(chan bool, 1) + go func() { + scanDone <- h.stdout.Scan() + }() + + var baseErr error + select { + case ok := <-scanDone: + if ok { + return h.stdout.Bytes(), nil + } + if err := h.stdout.Err(); err != nil { + return nil, err + } + // EOF - handler closed stdout prematurely, fall through to kill and capture stderr + baseErr = ErrHandlerClosed + case <-time.After(10 * time.Second): + // Timeout - handler didn't respond, fall through to kill and capture stderr + baseErr = ErrHandlerTimeout + } + + // Kill the process immediately to force stderr to close. + // Without this, there's a rare scenario where stdout closes but stderr remains open, + // causing io.ReadAll(h.stderr) below to block indefinitely waiting for stderr EOF. + if h.cmd.Process != nil { + h.cmd.Process.Kill() + } + + // Capture stderr to provide diagnostic information when the handler fails. + if stderrOut, err := io.ReadAll(h.stderr); err == nil && len(stderrOut) > 0 { + return nil, fmt.Errorf("%w: %s", baseErr, bytes.TrimSpace(stderrOut)) + } + return nil, baseErr +} + +// Close closes stdin and waits for the handler to exit with a 5-second timeout. +// If the handler doesn't exit within the timeout, it is killed. +func (h *Handler) Close() { + if h.stdin != nil { + // Close stdin to signal the handler that we're done sending requests. + // Per the handler specification, the handler should exit cleanly when stdin closes. + h.stdin.Close() + } + if h.cmd != nil { + // Wait for the handler to exit cleanly in response to stdin closing. + // Wait() automatically closes all remaining pipes after the process exits. + // Use a timeout in case the handler doesn't respect the protocol. + done := make(chan error, 1) + go func() { + done <- h.cmd.Wait() + }() + + select { + case err := <-done: + if err != nil { + slog.Warn("Handler exit with error", "error", err) + } + case <-time.After(5 * time.Second): + slog.Warn("Handler did not exit within a 5-second timeout, killing process") + if h.cmd.Process != nil { + h.cmd.Process.Kill() + // Call Wait() again to let the process finish cleanup (closing pipes, etc.) + // No timeout needed since Kill() should guarantee the process will exit + h.cmd.Wait() + } + } + } +} diff --git a/runner/runner.go b/runner/runner.go index 11fed46..f7d3490 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1,25 +1,19 @@ package runner import ( - "bufio" - "bytes" "embed" "encoding/json" "fmt" - "io" "io/fs" + "log/slog" "os" - "os/exec" "path/filepath" ) // TestRunner executes test suites against a handler binary type TestRunner struct { handlerPath string - cmd *exec.Cmd - stdin io.WriteCloser - stdout *bufio.Scanner - stderr io.ReadCloser + handler *Handler } // NewTestRunner creates a new test runner @@ -28,82 +22,64 @@ func NewTestRunner(handlerPath string) (*TestRunner, error) { return nil, fmt.Errorf("handler binary not found: %s", handlerPath) } - cmd := exec.Command(handlerPath) - - stdin, err := cmd.StdinPipe() + handler, err := NewHandler(handlerPath) if err != nil { - return nil, fmt.Errorf("failed to create stdin pipe: %w", err) - } - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("failed to create stdout pipe: %w", err) - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, fmt.Errorf("failed to create stderr pipe: %w", err) - } - - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("failed to start handler: %w", err) + return nil, err } return &TestRunner{ handlerPath: handlerPath, - cmd: cmd, - stdin: stdin, - stdout: bufio.NewScanner(stdout), - stderr: stderr, + handler: handler, }, nil } -// Close terminates the handler process -func (tr *TestRunner) Close() error { - if tr.stdin != nil { - tr.stdin.Close() - } - if tr.cmd != nil { - return tr.cmd.Wait() +// SendRequest sends a request to the handler, spawning a new handler if needed +func (tr *TestRunner) SendRequest(req Request) error { + if tr.handler == nil { + handler, err := NewHandler(tr.handlerPath) + if err != nil { + return fmt.Errorf("failed to spawn new handler: %w", err) + } + tr.handler = handler } - return nil -} -// SendRequest sends a request and reads the response -func (tr *TestRunner) SendRequest(req Request) (*Response, error) { reqData, err := json.Marshal(req) if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) + return fmt.Errorf("failed to marshal request: %w", err) } - if _, err := tr.stdin.Write(append(reqData, '\n')); err != nil { - return nil, fmt.Errorf("failed to write request: %w", err) + if err := tr.handler.SendLine(reqData); err != nil { + slog.Warn("Failed to write request, cleaning up handler (will spawn new one for remaining tests)", "error", err) + tr.CloseHandler() + return fmt.Errorf("failed to write request: %w", err) } + return nil +} - // Read response - if !tr.stdout.Scan() { - if err := tr.stdout.Err(); err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - // Handler closed stdout prematurely. - // Kill the process immediately to force stderr to close. - // Without this, there's a rare scenario where stdout closes but stderr remains open, - // causing io.ReadAll(tr.stderr) below to block indefinitely waiting for stderr EOF. - if tr.cmd.Process != nil { - tr.cmd.Process.Kill() - } - if stderrOut, err := io.ReadAll(tr.stderr); err == nil && len(stderrOut) > 0 { - return nil, fmt.Errorf("handler closed unexpectedly: %s", bytes.TrimSpace(stderrOut)) - } - return nil, fmt.Errorf("handler closed unexpectedly") +// ReadResponse reads and unmarshals a response from the handler +func (tr *TestRunner) ReadResponse() (*Response, error) { + line, err := tr.handler.ReadLine() + if err != nil { + slog.Warn("Failed to read response, cleaning up handler (will spawn new one for remaining tests)", "error", err) + tr.CloseHandler() + return nil, err } - respLine := tr.stdout.Text() + var resp Response - if err := json.Unmarshal([]byte(respLine), &resp); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) + if err := json.Unmarshal(line, &resp); err != nil { + return nil, err } return &resp, nil } +// CloseHandler closes the handler and sets it to nil +func (tr *TestRunner) CloseHandler() { + if tr.handler == nil { + return + } + tr.handler.Close() + tr.handler = nil +} + // RunTestSuite executes a test suite func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult { result := TestResult{ @@ -112,15 +88,7 @@ func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult { } for _, test := range suite.Tests { - err := tr.runTest(test) - testResult := SingleTestResult{ - TestID: test.ID, - Passed: err == nil, - } - if err != nil { - testResult.Message = err.Error() - } - + testResult := tr.runTest(test) result.TestResults = append(result.TestResults, testResult) if testResult.Passed { result.PassedTests++ @@ -132,21 +100,44 @@ func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult { return result } -// runTest executes a single test case and validates the response. -// Returns an error if communication with the handler fails or validation fails. -func (tr *TestRunner) runTest(test TestCase) error { +// runTest executes a single test case by sending a request, reading the response, +// and validating the result matches expected output +func (tr *TestRunner) runTest(test TestCase) SingleTestResult { req := Request{ ID: test.ID, Method: test.Method, Params: test.Params, } - resp, err := tr.SendRequest(req) + err := tr.SendRequest(req) if err != nil { - return err + return SingleTestResult{ + TestID: test.ID, + Passed: false, + Message: fmt.Sprintf("Failed to send request: %v", err), + } } - return validateResponse(test, resp) + resp, err := tr.ReadResponse() + if err != nil { + return SingleTestResult{ + TestID: test.ID, + Passed: false, + Message: fmt.Sprintf("Failed to read response: %v", err), + } + } + + if err := validateResponse(test, resp); err != nil { + return SingleTestResult{ + TestID: test.ID, + Passed: false, + Message: fmt.Sprintf("Invalid response: %s", err.Error()), + } + } + return SingleTestResult{ + TestID: test.ID, + Passed: true, + } } // validateResponse validates that a response matches the expected test outcome. From 7d8a41a91c6267d3e5327911d6339e9db34e2db5 Mon Sep 17 00:00:00 2001 From: stringintech Date: Mon, 1 Dec 2025 16:34:11 +0330 Subject: [PATCH 2/6] Add test re-exec pattern for simulating handler behaviors Introduce HandlerConfig struct and refactor NewHandler to support custom args/env. Add TestMain-based re-exec pattern allowing the test binary to serve as both test runner and mock handler subprocess. This enables unit testing of different handler behaviors (normal, crash, hang, unresponsive) without external fixtures, verifying the runner correctly handles each scenario. --- Makefile | 2 +- runner/handler.go | 16 +++++-- runner/handler_test.go | 98 ++++++++++++++++++++++++++++++++++++++++++ runner/runner.go | 4 +- 4 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 runner/handler_test.go diff --git a/Makefile b/Makefile index 60449e2..90433f8 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ mock-handler: test: @echo "Running runner unit tests..." - go test ./runner/... + go test -v ./runner/... @echo "Running conformance tests with mock handler..." $(RUNNER_BIN) -handler $(MOCK_HANDLER_BIN) diff --git a/runner/handler.go b/runner/handler.go index daf3260..b9824a0 100644 --- a/runner/handler.go +++ b/runner/handler.go @@ -18,6 +18,13 @@ var ( ErrHandlerClosed = errors.New("handler closed unexpectedly") ) +// HandlerConfig configures a handler process +type HandlerConfig struct { + Path string + Args []string + Env []string +} + // Handler manages a conformance handler process communicating via stdin/stdout type Handler struct { cmd *exec.Cmd @@ -26,9 +33,12 @@ type Handler struct { stderr io.ReadCloser } -// NewHandler spawns a new handler process at the given path -func NewHandler(path string) (*Handler, error) { - cmd := exec.Command(path) +// NewHandler spawns a new handler process with the given configuration +func NewHandler(cfg HandlerConfig) (*Handler, error) { + cmd := exec.Command(cfg.Path, cfg.Args...) + if cfg.Env != nil { + cmd.Env = append(cmd.Environ(), cfg.Env...) + } stdin, err := cmd.StdinPipe() if err != nil { diff --git a/runner/handler_test.go b/runner/handler_test.go new file mode 100644 index 0000000..17b86c5 --- /dev/null +++ b/runner/handler_test.go @@ -0,0 +1,98 @@ +package runner + +import ( + "bufio" + "fmt" + "os" + "testing" +) + +const ( + // envTestAsSubprocess signals the binary to run as a subprocess helper. + envTestAsSubprocess = "TEST_AS_SUBPROCESS" + + // envTestHelperName specifies which helper function to execute in subprocess mode. + envTestHelperName = "TEST_HELPER_NAME" + + helperNameNormal = "normal" +) + +// testHelpers maps helper names to functions that simulate different handler behaviors. +var testHelpers = map[string]func(){ + helperNameNormal: helperNormal, +} + +// TestMain allows the test binary to serve two purposes: +// 1. Normal mode: runs tests when TEST_AS_SUBPROCESS != "1" +// 2. Subprocess mode: executes a helper function when TEST_AS_SUBPROCESS == "1" +// +// This enables tests to spawn the binary itself as a mock handler subprocess, +// avoiding the need for separate test fixture binaries. +func TestMain(m *testing.M) { + if os.Getenv(envTestAsSubprocess) != "1" { + // Run tests normally + os.Exit(m.Run()) + } + + // Run as subprocess helper based on which helper is requested + helperName := os.Getenv(envTestHelperName) + if helper, ok := testHelpers[helperName]; ok { + helper() + } else { + fmt.Fprintf(os.Stderr, "Unknown test helper: %s\n", helperName) + os.Exit(1) + } +} + +// TestHandler_NormalOperation tests that a well-behaved handler works correctly +func TestHandler_NormalOperation(t *testing.T) { + h, err := newHandlerForTest(t, helperNameNormal) + if err != nil { + t.Fatalf("Failed to create handler: %v", err) + } + defer h.Close() + + // Send a request to the handler + request := `{"id":1,"method":"test"}` + if err := h.SendLine([]byte(request)); err != nil { + t.Fatalf("Failed to send request: %v", err) + } + + // Read the response + line, err := h.ReadLine() + if err != nil { + t.Fatalf("Failed to read line: %v", err) + } + + expected := `{"id":1,"result":true}` + if string(line) != expected { + t.Errorf("Expected %q, got %q", expected, string(line)) + } +} + +// helperNormal simulates a normal well-behaved handler that reads a request, +// validates it, and sends a response. +func helperNormal() { + // Read requests from stdin and respond with expected results + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + request := scanner.Text() + expected := `{"id":1,"method":"test"}` + if request != expected { + fmt.Fprintf(os.Stderr, "Expected request %q, got %q\n", expected, request) + os.Exit(1) + } + fmt.Println(`{"id":1,"result":true}`) + } +} + +// newHandlerForTest creates a Handler that runs a test helper as a subprocess. +// The helperName identifies which helper to run (e.g., "normal", "crash", "hang"). +func newHandlerForTest(t *testing.T, helperName string) (*Handler, error) { + t.Helper() + + return NewHandler(HandlerConfig{ + Path: os.Args[0], + Env: []string{"TEST_AS_SUBPROCESS=1", "TEST_HELPER_NAME=" + helperName}, + }) +} diff --git a/runner/runner.go b/runner/runner.go index f7d3490..c08a1f0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -22,7 +22,7 @@ func NewTestRunner(handlerPath string) (*TestRunner, error) { return nil, fmt.Errorf("handler binary not found: %s", handlerPath) } - handler, err := NewHandler(handlerPath) + handler, err := NewHandler(HandlerConfig{Path: handlerPath}) if err != nil { return nil, err } @@ -36,7 +36,7 @@ func NewTestRunner(handlerPath string) (*TestRunner, error) { // SendRequest sends a request to the handler, spawning a new handler if needed func (tr *TestRunner) SendRequest(req Request) error { if tr.handler == nil { - handler, err := NewHandler(tr.handlerPath) + handler, err := NewHandler(HandlerConfig{Path: tr.handlerPath}) if err != nil { return fmt.Errorf("failed to spawn new handler: %w", err) } From 7bb0694a8636ac9d5a9aa207902088255c51b112 Mon Sep 17 00:00:00 2001 From: stringintech Date: Mon, 1 Dec 2025 17:28:31 +0330 Subject: [PATCH 3/6] Add configurable timeout for handler response to each test case Add --handler-timeout flag to runner binary allowing users to configure max response wait time per test case (default: 10s). Add TestHandler_Unresponsive to verify timeout behavior when handler becomes unresponsive. --- cmd/runner/main.go | 4 ++- runner/handler.go | 33 ++++++++++++++------- runner/handler_test.go | 65 +++++++++++++++++++++++++++++++++++++----- runner/runner.go | 25 ++++++++++------ 4 files changed, 100 insertions(+), 27 deletions(-) diff --git a/cmd/runner/main.go b/cmd/runner/main.go index ba25039..ee561a2 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "strings" + "time" "github.com/stringintech/kernel-bindings-tests/runner" "github.com/stringintech/kernel-bindings-tests/testdata" @@ -13,6 +14,7 @@ import ( func main() { handlerPath := flag.String("handler", "", "Path to handler binary") + handlerTimeout := flag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)") flag.Parse() if *handlerPath == "" { @@ -34,7 +36,7 @@ func main() { } // Create test runner - testRunner, err := runner.NewTestRunner(*handlerPath) + testRunner, err := runner.NewTestRunner(*handlerPath, *handlerTimeout) if err != nil { fmt.Fprintf(os.Stderr, "Error creating test runner: %v\n", err) os.Exit(1) diff --git a/runner/handler.go b/runner/handler.go index b9824a0..2239e76 100644 --- a/runner/handler.go +++ b/runner/handler.go @@ -23,18 +23,23 @@ type HandlerConfig struct { Path string Args []string Env []string + // Timeout specifies the maximum duration to wait when reading from the handler's + // stdout. If zero, defaults to 10 seconds. The handler is killed if it fails to + // write output within this timeout. + Timeout time.Duration } // Handler manages a conformance handler process communicating via stdin/stdout type Handler struct { - cmd *exec.Cmd - stdin io.WriteCloser - stdout *bufio.Scanner - stderr io.ReadCloser + cmd *exec.Cmd + stdin io.WriteCloser + stdout *bufio.Scanner + stderr io.ReadCloser + timeout time.Duration } // NewHandler spawns a new handler process with the given configuration -func NewHandler(cfg HandlerConfig) (*Handler, error) { +func NewHandler(cfg *HandlerConfig) (*Handler, error) { cmd := exec.Command(cfg.Path, cfg.Args...) if cfg.Env != nil { cmd.Env = append(cmd.Environ(), cfg.Env...) @@ -60,11 +65,17 @@ func NewHandler(cfg HandlerConfig) (*Handler, error) { return nil, fmt.Errorf("failed to start handler: %w", err) } + timeout := cfg.Timeout + if timeout == 0 { + timeout = 10 * time.Second + } + return &Handler{ - cmd: cmd, - stdin: stdin, - stdout: bufio.NewScanner(stdout), - stderr: stderr, + cmd: cmd, + stdin: stdin, + stdout: bufio.NewScanner(stdout), + stderr: stderr, + timeout: timeout, }, nil } @@ -74,7 +85,7 @@ func (h *Handler) SendLine(line []byte) error { return err } -// ReadLine reads a line from the handler's stdout with a 10-second timeout +// ReadLine reads a line from the handler's stdout with a configurable timeout func (h *Handler) ReadLine() ([]byte, error) { // Use a timeout for Scan() in case the handler hangs scanDone := make(chan bool, 1) @@ -93,7 +104,7 @@ func (h *Handler) ReadLine() ([]byte, error) { } // EOF - handler closed stdout prematurely, fall through to kill and capture stderr baseErr = ErrHandlerClosed - case <-time.After(10 * time.Second): + case <-time.After(h.timeout): // Timeout - handler didn't respond, fall through to kill and capture stderr baseErr = ErrHandlerTimeout } diff --git a/runner/handler_test.go b/runner/handler_test.go index 17b86c5..7155038 100644 --- a/runner/handler_test.go +++ b/runner/handler_test.go @@ -2,9 +2,11 @@ package runner import ( "bufio" + "errors" "fmt" "os" "testing" + "time" ) const ( @@ -14,12 +16,14 @@ const ( // envTestHelperName specifies which helper function to execute in subprocess mode. envTestHelperName = "TEST_HELPER_NAME" - helperNameNormal = "normal" + helperNameNormal = "normal" + helperNameUnresponsive = "unresponsive" ) // testHelpers maps helper names to functions that simulate different handler behaviors. var testHelpers = map[string]func(){ - helperNameNormal: helperNormal, + helperNameNormal: helperNormal, + helperNameUnresponsive: helperUnresponsive, } // TestMain allows the test binary to serve two purposes: @@ -46,7 +50,7 @@ func TestMain(m *testing.M) { // TestHandler_NormalOperation tests that a well-behaved handler works correctly func TestHandler_NormalOperation(t *testing.T) { - h, err := newHandlerForTest(t, helperNameNormal) + h, err := newHandlerForTest(t, helperNameNormal, 0) if err != nil { t.Fatalf("Failed to create handler: %v", err) } @@ -86,13 +90,60 @@ func helperNormal() { } } +// TestHandler_Unresponsive tests that the runner correctly handles an unresponsive handler +func TestHandler_Unresponsive(t *testing.T) { + h, err := newHandlerForTest(t, helperNameUnresponsive, 100*time.Millisecond) + if err != nil { + t.Fatalf("Failed to create handler: %v", err) + } + defer h.Close() + + // Send a request to the handler + request := `{"id":1,"method":"test"}` + if err := h.SendLine([]byte(request)); err != nil { + t.Fatalf("Failed to send request: %v", err) + } + + // Try to read the response - should Timeout + start := time.Now() + _, err = h.ReadLine() + elapsed := time.Since(start) + + if err == nil { + t.Fatal("Expected error from unresponsive handler, got nil") + } + + // Verify it's the Timeout error we expect + if !errors.Is(err, ErrHandlerTimeout) { + t.Errorf("Expected ErrHandlerTimeout, got: %v", err) + } + + // Verify Timeout happened quickly (within reasonable margin) + if elapsed > 200*time.Millisecond { + t.Errorf("Timeout took too long: %v (expected ~100ms)", elapsed) + } +} + +// helperUnresponsive simulates a handler that receives requests but never responds, +// triggering the Timeout mechanism in the runner. +func helperUnresponsive() { + // Read from stdin to prevent broken pipe, but never write responses + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + // Sleep indefinitely to simulate unresponsiveness + time.Sleep(1 * time.Hour) + } +} + // newHandlerForTest creates a Handler that runs a test helper as a subprocess. // The helperName identifies which helper to run (e.g., "normal", "crash", "hang"). -func newHandlerForTest(t *testing.T, helperName string) (*Handler, error) { +// The timeout parameter sets the per-request timeout (0 uses default). +func newHandlerForTest(t *testing.T, helperName string, timeout time.Duration) (*Handler, error) { t.Helper() - return NewHandler(HandlerConfig{ - Path: os.Args[0], - Env: []string{"TEST_AS_SUBPROCESS=1", "TEST_HELPER_NAME=" + helperName}, + return NewHandler(&HandlerConfig{ + Path: os.Args[0], + Env: []string{"TEST_AS_SUBPROCESS=1", "TEST_HELPER_NAME=" + helperName}, + Timeout: timeout, }) } diff --git a/runner/runner.go b/runner/runner.go index c08a1f0..0c6abe3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,35 +8,44 @@ import ( "log/slog" "os" "path/filepath" + "time" ) // TestRunner executes test suites against a handler binary type TestRunner struct { - handlerPath string - handler *Handler + handler *Handler + handlerConfig *HandlerConfig } -// NewTestRunner creates a new test runner -func NewTestRunner(handlerPath string) (*TestRunner, error) { +// NewTestRunner creates a new test runner for executing test suites against a handler binary. +// The handlerTimeout parameter specifies the maximum duration to wait for the handler to +// respond to each test case. If zero, defaults to 10 seconds. +func NewTestRunner(handlerPath string, handlerTimeout time.Duration) (*TestRunner, error) { if _, err := os.Stat(handlerPath); os.IsNotExist(err) { return nil, fmt.Errorf("handler binary not found: %s", handlerPath) } - handler, err := NewHandler(HandlerConfig{Path: handlerPath}) + handler, err := NewHandler(&HandlerConfig{ + Path: handlerPath, + Timeout: handlerTimeout, + }) if err != nil { return nil, err } return &TestRunner{ - handlerPath: handlerPath, - handler: handler, + handler: handler, + handlerConfig: &HandlerConfig{ + Path: handlerPath, + Timeout: handlerTimeout, + }, }, nil } // SendRequest sends a request to the handler, spawning a new handler if needed func (tr *TestRunner) SendRequest(req Request) error { if tr.handler == nil { - handler, err := NewHandler(HandlerConfig{Path: tr.handlerPath}) + handler, err := NewHandler(tr.handlerConfig) if err != nil { return fmt.Errorf("failed to spawn new handler: %w", err) } From 655dcfcac727d741fa0f0238c5854252436ee9a6 Mon Sep 17 00:00:00 2001 From: stringintech Date: Mon, 1 Dec 2025 17:53:32 +0330 Subject: [PATCH 4/6] Add test for handler crash with stderr capture Adds TestHandler_Crash to verify the runner correctly handles handlers that crash while processing requests. The test confirms ErrHandlerClosed is returned and that panic messages from stderr are captured in the error. --- runner/handler_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/runner/handler_test.go b/runner/handler_test.go index 7155038..228ebff 100644 --- a/runner/handler_test.go +++ b/runner/handler_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "strings" "testing" "time" ) @@ -16,14 +17,17 @@ const ( // envTestHelperName specifies which helper function to execute in subprocess mode. envTestHelperName = "TEST_HELPER_NAME" + // Handler simulation test identifiers. helperNameNormal = "normal" helperNameUnresponsive = "unresponsive" + helperNameCrash = "crash" ) // testHelpers maps helper names to functions that simulate different handler behaviors. var testHelpers = map[string]func(){ helperNameNormal: helperNormal, helperNameUnresponsive: helperUnresponsive, + helperNameCrash: helperCrash, } // TestMain allows the test binary to serve two purposes: @@ -135,6 +139,49 @@ func helperUnresponsive() { } } +// TestHandler_Crash tests that the runner correctly handles a handler that crashes +// while processing a request +func TestHandler_Crash(t *testing.T) { + h, err := newHandlerForTest(t, helperNameCrash, 0) + if err != nil { + t.Fatalf("Failed to create handler: %v", err) + } + defer h.Close() + + // Send a request to the handler + request := `{"id":1,"method":"test"}` + if err := h.SendLine([]byte(request)); err != nil { + t.Fatalf("Failed to send request: %v", err) + } + + // Try to read the response - should get ErrHandlerClosed + _, err = h.ReadLine() + + if err == nil { + t.Fatal("Expected error from crashed handler, got nil") + } + + // Verify it's the handler closed error we expect + if !errors.Is(err, ErrHandlerClosed) { + t.Errorf("Expected ErrHandlerClosed, got: %v", err) + } + + // Verify the error message contains the panic string from stderr + if !strings.Contains(err.Error(), "simulated handler crash") { + t.Errorf("Expected error to contain 'simulated handler crash', got: %v", err) + } +} + +// helperCrash simulates a handler that crashes while processing a request, +// triggering the ErrHandlerClosed error in the runner. +func helperCrash() { + // Read requests from stdin but panic instead of responding + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + panic("simulated handler crash") + } +} + // newHandlerForTest creates a Handler that runs a test helper as a subprocess. // The helperName identifies which helper to run (e.g., "normal", "crash", "hang"). // The timeout parameter sets the per-request timeout (0 uses default). From ba7369408c1ab62c9dd202ed180933fc99919955 Mon Sep 17 00:00:00 2001 From: stringintech Date: Mon, 1 Dec 2025 18:13:15 +0330 Subject: [PATCH 5/6] Add total execution timeout flag Add --timeout flag (default 30s) to limit total test execution time across all test suites, complementing per-test-case --handler-timeout. --- cmd/runner/main.go | 10 ++++++++-- runner/runner.go | 31 ++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/cmd/runner/main.go b/cmd/runner/main.go index ee561a2..fa756d7 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "flag" "fmt" "io/fs" @@ -15,6 +16,7 @@ import ( func main() { handlerPath := flag.String("handler", "", "Path to handler binary") handlerTimeout := flag.Duration("handler-timeout", 10*time.Second, "Max time to wait for handler to respond to each test case (e.g., 10s, 500ms)") + timeout := flag.Duration("timeout", 30*time.Second, "Total timeout for executing all test suites (e.g., 30s, 1m)") flag.Parse() if *handlerPath == "" { @@ -36,13 +38,17 @@ func main() { } // Create test runner - testRunner, err := runner.NewTestRunner(*handlerPath, *handlerTimeout) + testRunner, err := runner.NewTestRunner(*handlerPath, *handlerTimeout, *timeout) if err != nil { fmt.Fprintf(os.Stderr, "Error creating test runner: %v\n", err) os.Exit(1) } defer testRunner.CloseHandler() + // Create context with total execution timeout + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + // Run tests totalPassed := 0 totalFailed := 0 @@ -59,7 +65,7 @@ func main() { } // Run suite - result := testRunner.RunTestSuite(*suite) + result := testRunner.RunTestSuite(ctx, *suite) printResults(suite, result) totalPassed += result.PassedTests diff --git a/runner/runner.go b/runner/runner.go index 0c6abe3..3c2c441 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1,6 +1,7 @@ package runner import ( + "context" "embed" "encoding/json" "fmt" @@ -15,12 +16,15 @@ import ( type TestRunner struct { handler *Handler handlerConfig *HandlerConfig + timeout time.Duration } // NewTestRunner creates a new test runner for executing test suites against a handler binary. // The handlerTimeout parameter specifies the maximum duration to wait for the handler to // respond to each test case. If zero, defaults to 10 seconds. -func NewTestRunner(handlerPath string, handlerTimeout time.Duration) (*TestRunner, error) { +// The timeout parameter specifies the total duration allowed for running all tests +// across all test suites. If zero, defaults to 30 seconds. +func NewTestRunner(handlerPath string, handlerTimeout time.Duration, timeout time.Duration) (*TestRunner, error) { if _, err := os.Stat(handlerPath); os.IsNotExist(err) { return nil, fmt.Errorf("handler binary not found: %s", handlerPath) } @@ -33,12 +37,17 @@ func NewTestRunner(handlerPath string, handlerTimeout time.Duration) (*TestRunne return nil, err } + if timeout == 0 { + timeout = 30 * time.Second + } + return &TestRunner{ handler: handler, handlerConfig: &HandlerConfig{ Path: handlerPath, Timeout: handlerTimeout, }, + timeout: timeout, }, nil } @@ -89,15 +98,16 @@ func (tr *TestRunner) CloseHandler() { tr.handler = nil } -// RunTestSuite executes a test suite -func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult { +// RunTestSuite executes a test suite. The context can be used to enforce a total +// execution timeout across all test suites. +func (tr *TestRunner) RunTestSuite(ctx context.Context, suite TestSuite) TestResult { result := TestResult{ SuiteName: suite.Name, TotalTests: len(suite.Tests), } for _, test := range suite.Tests { - testResult := tr.runTest(test) + testResult := tr.runTest(ctx, test) result.TestResults = append(result.TestResults, testResult) if testResult.Passed { result.PassedTests++ @@ -111,7 +121,18 @@ func (tr *TestRunner) RunTestSuite(suite TestSuite) TestResult { // runTest executes a single test case by sending a request, reading the response, // and validating the result matches expected output -func (tr *TestRunner) runTest(test TestCase) SingleTestResult { +func (tr *TestRunner) runTest(ctx context.Context, test TestCase) SingleTestResult { + // Check if context is already cancelled + select { + case <-ctx.Done(): + return SingleTestResult{ + TestID: test.ID, + Passed: false, + Message: fmt.Sprintf("Total execution timeout exceeded (%v)", tr.timeout), + } + default: + } + req := Request{ ID: test.ID, Method: test.Method, From d6bf3b06db316a04119722386ab3f143484305bf Mon Sep 17 00:00:00 2001 From: stringintech Date: Mon, 1 Dec 2025 18:25:51 +0330 Subject: [PATCH 6/6] Document timeout flags and handler recovery in README.md --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 9f075c8..62f81f3 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,20 @@ make runner # Run the test runner against your handler binary ./build/runner --handler + +# Configure timeouts (optional) +./build/runner --handler \ + --handler-timeout 30s \ # Max wait per test case (default: 10s) + --timeout 2m # Total execution limit (default: 30s) ``` +#### Timeout Flags + +- **`--handler-timeout`** (default: 10s): Maximum time to wait for handler response to each test case. Prevents hangs on unresponsive handlers. +- **`--timeout`** (default: 30s): Total execution time limit across all test suites. Ensures bounded test runs. + +The runner automatically detects and recovers from crashed/unresponsive handlers, allowing remaining tests to continue. + ### Testing the Runner Build and test the runner: