diff --git a/.changeset/brave-functions-run.md b/.changeset/brave-functions-run.md new file mode 100644 index 0000000..bdb6ef0 --- /dev/null +++ b/.changeset/brave-functions-run.md @@ -0,0 +1,5 @@ +--- +"@shopify/shopify-function-test-helpers": minor +--- + +Add function run metadata to `runFunction`, including instruction count, memory usage, and module size. diff --git a/README.md b/README.md index 0f5a69d..00df274 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,12 @@ describe("Default Integration Test", () => { ); expect(runResult.error).toBeNull(); - expect(runResult.result.output).toEqual(fixture.expectedOutput); + expect(runResult.result?.output).toEqual(fixture.expectedOutput); + expect(runResult.metadata).toEqual({ + instructionCount: expect.any(Number), + memoryUsageKiB: expect.any(Number), + moduleSizeKiB: expect.any(Number) + }); }, 10000); }); }); diff --git a/src/methods/run-function.ts b/src/methods/run-function.ts index 95c4a9e..4222c68 100644 --- a/src/methods/run-function.ts +++ b/src/methods/run-function.ts @@ -6,12 +6,74 @@ import { spawn } from "child_process"; import { FixtureData } from "./load-fixture.js"; +/** + * Metadata reported by function-runner for a function execution. + */ +export interface FunctionRunMetadata { + instructionCount: number; + memoryUsageKiB: number; + moduleSizeKiB: number; +} + /** * Interface for the run function result */ export interface RunFunctionResult { result: { output: any } | null; + metadata: FunctionRunMetadata | null; + error: string | null; +} + +function parseFunctionRunnerResult(stdout: string): { + result: RunFunctionResult["result"]; + metadata: FunctionRunMetadata | null; error: string | null; +} { + let result: unknown; + + try { + result = JSON.parse(stdout); + } catch (parseError) { + // function-runner is not guaranteed to return JSON when it fails. + return { + result: null, + metadata: null, + error: `Failed to parse function-runner output: ${parseError instanceof Error ? parseError.message : "Unknown error"}`, + }; + } + + const invalidShapeResult = { + result: null, + metadata: null, + error: `function-runner returned unexpected format. Received: ${JSON.stringify(result)}`, + }; + + if (typeof result !== "object" || result === null) { + return invalidShapeResult; + } + + const resultObject = result as Record; + const { output, instructions, size: moduleSize } = resultObject; + const memoryUsage = resultObject.memory_usage; + + if ( + output === undefined || + typeof instructions !== "number" || + typeof memoryUsage !== "number" || + typeof moduleSize !== "number" + ) { + return invalidShapeResult; + } + + return { + result: { output }, + metadata: { + instructionCount: instructions, + memoryUsageKiB: memoryUsage, + moduleSizeKiB: moduleSize, + }, + error: null, + }; } /** @@ -68,41 +130,24 @@ export async function runFunction( }); runnerProcess.on("close", (code) => { + const functionRunnerResult = parseFunctionRunnerResult(stdout); + if (code !== 0) { resolve({ result: null, + metadata: functionRunnerResult.metadata, error: `function-runner failed with exit code ${code}: ${stderr}`, }); return; } - try { - const result = JSON.parse(stdout); - - // function-runner output format: { output: {...} } - if (!result.output) { - resolve({ - result: null, - error: `function-runner returned unexpected format - missing 'output' field. Received: ${JSON.stringify(result)}`, - }); - return; - } - - resolve({ - result: { output: result.output }, - error: null, - }); - } catch (parseError) { - resolve({ - result: null, - error: `Failed to parse function-runner output: ${parseError instanceof Error ? parseError.message : "Unknown error"}`, - }); - } + resolve(functionRunnerResult); }); runnerProcess.on("error", (error) => { resolve({ result: null, + metadata: null, error: `Failed to start function-runner: ${error.message}`, }); }); @@ -114,11 +159,13 @@ export async function runFunction( if (error instanceof Error) { return { result: null, + metadata: null, error: error.message, }; } else { return { result: null, + metadata: null, error: "Unknown error occurred", }; } diff --git a/src/wasm-testing-helpers.ts b/src/wasm-testing-helpers.ts index 2f02140..bb0f777 100644 --- a/src/wasm-testing-helpers.ts +++ b/src/wasm-testing-helpers.ts @@ -20,7 +20,10 @@ export { validateFixtureInput } from "./methods/validate-fixture-input.js"; // Export types for consumers export type { FixtureData } from "./methods/load-fixture.js"; export type { BuildFunctionResult } from "./methods/build-function.js"; -export type { RunFunctionResult } from "./methods/run-function.js"; +export type { + FunctionRunMetadata, + RunFunctionResult, +} from "./methods/run-function.js"; export type { FunctionInfo } from "./methods/get-function-info.js"; export type { ValidateTestAssetsOptions, diff --git a/test/methods/run-function.test.ts b/test/methods/run-function.test.ts index 3983d7e..8c0a60f 100644 --- a/test/methods/run-function.test.ts +++ b/test/methods/run-function.test.ts @@ -46,6 +46,46 @@ describe("runFunction", () => { vi.clearAllMocks(); }); + function runnerOutputJson( + change?: (result: Record) => void, + ): string { + const result: Record = { + output: { operations: [] }, + instructions: 4423, + size: 49, + // eslint-disable-next-line @typescript-eslint/naming-convention + memory_usage: 1088, + }; + + change?.(result); + + return JSON.stringify(result); + } + + async function runFunctionWithStdout(stdout: string) { + const fixture: FixtureData = { + export: "cart-validations-generate-run", + input: { cart: { lines: [] } }, + expectedOutput: {}, + target: "cart.validations.generate.run", + }; + + const resultPromise = runFunction( + fixture, + "/path/to/function-runner", + "/path/to/function.wasm", + "/path/to/query.graphql", + "/path/to/schema.graphql", + ); + + setImmediate(() => { + mockStdout.emit("data", Buffer.from(stdout)); + mockProcess.emit("close", 0); + }); + + return resultPromise; + } + it("should run a function successfully and return result", async () => { const fixture: FixtureData = { export: "cart-validations-generate-run", @@ -75,9 +115,13 @@ describe("runFunction", () => { // Simulate successful function execution const expectedOutput = { + size: 49, + instructions: 4423, output: { operations: [], }, + // eslint-disable-next-line @typescript-eslint/naming-convention + memory_usage: 1088, }; setImmediate(() => { mockStdout.emit("data", Buffer.from(JSON.stringify(expectedOutput))); @@ -88,7 +132,12 @@ describe("runFunction", () => { expect(result).toBeDefined(); expect(result.error).toBeNull(); - expect(result.result).toEqual(expectedOutput); + expect(result.result).toEqual({ output: expectedOutput.output }); + expect(result.metadata).toEqual({ + instructionCount: 4423, + memoryUsageKiB: 1088, + moduleSizeKiB: 49, + }); // Verify spawn was called with correct arguments expect(mockSpawn).toHaveBeenCalledWith( @@ -145,6 +194,7 @@ describe("runFunction", () => { expect(result.error).toContain("function-runner failed with exit code 1"); expect(result.error).toContain("Error: Export not found"); expect(result.result).toBeNull(); + expect(result.metadata).toBeNull(); }); it("should handle process spawn errors", async () => { @@ -180,6 +230,7 @@ describe("runFunction", () => { expect(result.error).toContain("Failed to start function-runner"); expect(result.error).toContain("ENOENT"); expect(result.result).toBeNull(); + expect(result.metadata).toBeNull(); }); it("should handle invalid JSON output from function-runner", async () => { @@ -214,6 +265,7 @@ describe("runFunction", () => { expect(result).toBeDefined(); expect(result.error).toContain("Failed to parse function-runner output"); expect(result.result).toBeNull(); + expect(result.metadata).toBeNull(); }); it("should handle multiple stdout/stderr chunks", async () => { @@ -238,9 +290,10 @@ describe("runFunction", () => { ); // Simulate output in multiple chunks - const outputPart1 = '{"output":'; + const outputPart1 = + '{"name":"function.wasm","size":49,"instructions":4423,"logs":"","input":{},"output":'; const outputPart2 = '{"operations":[]}'; - const outputPart3 = "}"; + const outputPart3 = ',"success":true,"memory_usage":1088}'; setImmediate(() => { mockStdout.emit("data", Buffer.from(outputPart1)); @@ -258,9 +311,44 @@ describe("runFunction", () => { operations: [], }, }); + expect(result.metadata).toEqual({ + instructionCount: 4423, + memoryUsageKiB: 1088, + moduleSizeKiB: 49, + }); }); - it("should reject output without explicit output wrapper", async () => { + it("should return function-runner error without metadata for invalid JSON shape on non-zero exit code", async () => { + const fixture: FixtureData = { + export: "cart-validations-generate-run", + input: { cart: { lines: [] } }, + expectedOutput: {}, + target: "cart.validations.generate.run", + }; + + const resultPromise = runFunction( + fixture, + "/path/to/function-runner", + "/path/to/function.wasm", + "/path/to/query.graphql", + "/path/to/schema.graphql", + ); + + setImmediate(() => { + mockStdout.emit("data", Buffer.from(JSON.stringify({ error: "boom" }))); + mockStderr.emit("data", Buffer.from("Function failed")); + mockProcess.emit("close", 1); + }); + + const result = await resultPromise; + + expect(result.error).toContain("function-runner failed with exit code 1"); + expect(result.error).toContain("Function failed"); + expect(result.result).toBeNull(); + expect(result.metadata).toBeNull(); + }); + + it("should return metadata from parseable output on non-zero exit code", async () => { const fixture: FixtureData = { export: "cart-validations-generate-run", input: { cart: { lines: [] } }, @@ -281,26 +369,74 @@ describe("runFunction", () => { schemaPath, ); - // Simulate output without "output" wrapper setImmediate(() => { mockStdout.emit( "data", Buffer.from( JSON.stringify({ - operations: [], + size: 42, + instructions: 999, + output: {}, + // eslint-disable-next-line @typescript-eslint/naming-convention + memory_usage: 1000, }), ), ); - mockProcess.emit("close", 0); + mockStderr.emit("data", Buffer.from("Function failed")); + mockProcess.emit("close", 1); }); const result = await resultPromise; - expect(result).toBeDefined(); - expect(result.error).toContain( - "function-runner returned unexpected format", - ); - expect(result.error).toContain("missing 'output' field"); + expect(result.error).toContain("function-runner failed with exit code 1"); + expect(result.error).toContain("Function failed"); expect(result.result).toBeNull(); + expect(result.metadata).toEqual({ + instructionCount: 999, + memoryUsageKiB: 1000, + moduleSizeKiB: 42, + }); }); + + it.each([ + ["null", () => JSON.stringify(null)], + ["string", () => JSON.stringify("not an object")], + [ + "missing output", + () => runnerOutputJson((result) => delete result.output), + ], + [ + "missing instructions", + () => runnerOutputJson((result) => delete result.instructions), + ], + [ + "non-number instructions", + () => runnerOutputJson((result) => (result.instructions = "4423")), + ], + [ + "missing memory usage", + () => runnerOutputJson((result) => delete result.memory_usage), + ], + [ + "non-number memory usage", + () => runnerOutputJson((result) => (result.memory_usage = "1088")), + ], + ["missing size", () => runnerOutputJson((result) => delete result.size)], + [ + "non-number size", + () => runnerOutputJson((result) => (result.size = "49")), + ], + ])( + "should reject invalid function-runner JSON shape: %s", + async (_name, getStdout) => { + const result = await runFunctionWithStdout(getStdout()); + + expect(result).toBeDefined(); + expect(result.error).toContain( + "function-runner returned unexpected format", + ); + expect(result.result).toBeNull(); + expect(result.metadata).toBeNull(); + }, + ); });