From f016836f341e14219499589d962eafc51927afe0 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:26:55 +0000 Subject: [PATCH 01/48] feat(swift-core): gate local functionCall actions on user activation --- .../A2UICore/Errors/FunctionError.swift | 6 +- swift/core/Sources/A2UICore/Models/Node.swift | 7 + .../A2UICore/Models/ResolvedAction.swift | 45 +++- .../A2UICore/Resolution/NodeResolver.swift | 17 +- .../Sources/A2UICore/State/DataContext.swift | 44 +++- .../UserActivationActionTests.swift | 249 ++++++++++++++++++ 6 files changed, 354 insertions(+), 14 deletions(-) create mode 100644 swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift diff --git a/swift/core/Sources/A2UICore/Errors/FunctionError.swift b/swift/core/Sources/A2UICore/Errors/FunctionError.swift index 1eae6421fb..dca861809e 100644 --- a/swift/core/Sources/A2UICore/Errors/FunctionError.swift +++ b/swift/core/Sources/A2UICore/Errors/FunctionError.swift @@ -27,6 +27,7 @@ public enum FunctionError: Error, Sendable, Equatable, CustomStringConvertible, case duplicate(callID: String) case noListener(name: String) case callerDisallowed(name: String, caller: String) + case userActivationRequired(name: String) case remoteError(code: String, message: String) /// Standardized RPC error code corresponding to this error. @@ -36,7 +37,8 @@ public enum FunctionError: Error, Sendable, Equatable, CustomStringConvertible, .missingArgument, .invalidArgumentType, .securityConstraintViolation, - .callerDisallowed: + .callerDisallowed, + .userActivationRequired: return "INVALID_FUNCTION_CALL" case .executionFailed: return "EXECUTION_ERROR" @@ -79,6 +81,8 @@ public enum FunctionError: Error, Sendable, Equatable, CustomStringConvertible, return "No outbound listener registered to invoke function '\(name)'" case .callerDisallowed(let name, let caller): return "Function '\(name)' cannot be called by \(caller)" + case .userActivationRequired(let name): + return "Function '\(name)' requires user activation context to execute." case .remoteError(let code, let message): return "Remote function error (\(code)): \(message)" } diff --git a/swift/core/Sources/A2UICore/Models/Node.swift b/swift/core/Sources/A2UICore/Models/Node.swift index c0d05883d8..291c343092 100644 --- a/swift/core/Sources/A2UICore/Models/Node.swift +++ b/swift/core/Sources/A2UICore/Models/Node.swift @@ -305,10 +305,17 @@ extension Node { } /// Returns the resolved action for the given property key. + @available(*, deprecated, renamed: "action(for:userActivation:)") public func action(for key: String) -> ResolvedAction? { properties[key] as? ResolvedAction } + /// Returns the resolved action for the given property key, gated on `userActivation` + /// when the action invokes a local function that requires user activation. + public func action(for key: String, userActivation: Bool) -> ResolvedAction? { + (properties[key] as? ResolvedAction)?.withUserActivation(userActivation) + } + /// Returns the single child `Node` for the given property key. public func child(for key: String) -> Node? { properties[key] as? Node diff --git a/swift/core/Sources/A2UICore/Models/ResolvedAction.swift b/swift/core/Sources/A2UICore/Models/ResolvedAction.swift index 26f45e0139..1322b2f41d 100644 --- a/swift/core/Sources/A2UICore/Models/ResolvedAction.swift +++ b/swift/core/Sources/A2UICore/Models/ResolvedAction.swift @@ -40,7 +40,8 @@ public struct ResolvedAction: Sendable { package let rawUserMessage: JSONValue? - private let triggerClosure: @Sendable @MainActor ([String: JSONValue]?) throws -> Void + private let userActivation: Bool + private let triggerClosure: @Sendable @MainActor (Bool, [String: JSONValue]?) throws -> Void /// Creates a new resolved action with the specified identity and /// trigger closure. @@ -57,7 +58,8 @@ public struct ResolvedAction: Sendable { self.dataContextPath = dataContextPath self.rawUserMessage = rawUserMessage self.metadata = metadata - self.triggerClosure = { _ in trigger() } + self.userActivation = true + self.triggerClosure = { _, _ in trigger() } } /// Creates a new resolved action with a trigger closure that accepts optional call-time metadata. @@ -74,19 +76,52 @@ public struct ResolvedAction: Sendable { self.dataContextPath = dataContextPath self.rawUserMessage = rawUserMessage self.metadata = metadata - self.triggerClosure = triggerWithMetadata + self.userActivation = true + self.triggerClosure = { _, callMetadata in try triggerWithMetadata(callMetadata) } + } + + package init( + identity: Identity, + sourceComponentID: String? = nil, + dataContextPath: String? = nil, + rawUserMessage: JSONValue? = nil, + metadata: [String: JSONValue]? = nil, + userActivation: Bool = true, + triggerWithUserActivation: + @escaping @Sendable @MainActor (Bool, [String: JSONValue]?) throws -> Void + ) { + self.identity = identity + self.sourceComponentID = sourceComponentID + self.dataContextPath = dataContextPath + self.rawUserMessage = rawUserMessage + self.metadata = metadata + self.userActivation = userActivation + self.triggerClosure = triggerWithUserActivation + } + + /// Returns a copy of this resolved action configured with the given user activation state. + package func withUserActivation(_ userActivation: Bool) -> ResolvedAction { + ResolvedAction( + identity: identity, + sourceComponentID: sourceComponentID, + dataContextPath: dataContextPath, + rawUserMessage: rawUserMessage, + metadata: metadata, + userActivation: userActivation, + triggerWithUserActivation: triggerClosure + ) } /// Triggers the action using parameter-free function-call syntax. @MainActor public func callAsFunction() { - try? triggerClosure(nil) + try? triggerClosure(userActivation, nil) } /// Triggers the action using function-call syntax, validating and merging call-site metadata. @MainActor public func callAsFunction(metadata: [String: JSONValue]?) throws { - try triggerClosure(metadata) + try triggerClosure(userActivation, metadata) } } diff --git a/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift b/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift index 7e46a0bb2b..7a50e4bfea 100644 --- a/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift +++ b/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift @@ -1193,7 +1193,8 @@ public final class NodeResolver: Sendable { private func evaluateDynamicValue( _ value: JSONValue, basePath: String?, - index: Int? = nil + index: Int? = nil, + userActivation: Bool = true ) -> JSONValue { let currentData = dataModel.data let previousReadHook = dataModel.onReadPath @@ -1210,9 +1211,10 @@ public final class NodeResolver: Sendable { path: basePath ?? "", functionHandler: self, protocolVersion: effectiveProtocolVersion, - index: index + index: index, + userActivation: userActivation ) - return context.resolveDynamicValue(value) + return context.resolveDynamicValue(value, userActivation: userActivation) } private func coerceToString(_ value: JSONValue?) -> String? { @@ -1658,12 +1660,17 @@ public final class NodeResolver: Sendable { identity: unresolvedIdentity, sourceComponentID: componentID, dataContextPath: basePath, - trigger: { [weak self] in + triggerWithUserActivation: { [weak self] userActivation, _ in guard let self else { return } guard self.reportFailedActionChecksIfNeeded(checks, componentID: componentID) else { return } - _ = self.evaluateDynamicValue(functionCallValue, basePath: basePath, index: index) + _ = self.evaluateDynamicValue( + functionCallValue, + basePath: basePath, + index: index, + userActivation: userActivation + ) } ) } diff --git a/swift/core/Sources/A2UICore/State/DataContext.swift b/swift/core/Sources/A2UICore/State/DataContext.swift index 97471bbdde..01fe45937e 100644 --- a/swift/core/Sources/A2UICore/State/DataContext.swift +++ b/swift/core/Sources/A2UICore/State/DataContext.swift @@ -25,6 +25,7 @@ public final class DataContext { public let dataModel: DataModel public let protocolVersion: A2UIProtocolVersion? public let index: Int? + public let userActivation: Bool /// A reference to the function handler to evaluate dynamic function calls. public weak var functionHandler: FunctionHandler? @@ -34,13 +35,31 @@ public final class DataContext { path: String, functionHandler: FunctionHandler, protocolVersion: A2UIProtocolVersion? = nil, - index: Int? = nil + index: Int? = nil, + userActivation: Bool = true ) { self.dataModel = dataModel self.path = path self.functionHandler = functionHandler self.protocolVersion = protocolVersion self.index = index + self.userActivation = userActivation + } + + private init( + dataModel: DataModel, + path: String, + optionalFunctionHandler: FunctionHandler?, + protocolVersion: A2UIProtocolVersion?, + index: Int?, + userActivation: Bool + ) { + self.dataModel = dataModel + self.path = path + self.functionHandler = optionalFunctionHandler + self.protocolVersion = protocolVersion + self.index = index + self.userActivation = userActivation } public var atLeastV10: Bool { @@ -81,7 +100,8 @@ public final class DataContext { path: absPath, functionHandler: handler, protocolVersion: protocolVersion, - index: index ?? self.index + index: index ?? self.index, + userActivation: userActivation ) } @@ -93,7 +113,17 @@ public final class DataContext { /// /// Recursively evaluates data bindings (`path` / `@path`), function calls /// (`call` / `@call`), arrays, and nested plain objects. - public func resolveDynamicValue(_ value: JSONValue) -> JSONValue { + public func resolveDynamicValue(_ value: JSONValue, userActivation: Bool? = nil) -> JSONValue { + if let userActivation, userActivation != self.userActivation { + return DataContext( + dataModel: dataModel, + path: path, + optionalFunctionHandler: functionHandler, + protocolVersion: protocolVersion, + index: index, + userActivation: userActivation + ).resolveDynamicValue(value) + } switch value { case .array(let arr): return .array(arr.map { resolveDynamicValue($0) }) @@ -161,6 +191,14 @@ public final class DataContext { return .null } + if function.api.requiresUserActivation && !userActivation { + functionHandler?.handleFunctionError( + FunctionError.userActivationRequired(name: callName), + functionName: callName + ) + return .null + } + var resolvedArgs: [String: JSONValue] = [:] let argPropsSchema: OrderedDictionary? = atLeastV10 ? function.api.schema.jsonValue["properties"]?.objectValue : nil diff --git a/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift b/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift new file mode 100644 index 0000000000..0c7d981b86 --- /dev/null +++ b/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift @@ -0,0 +1,249 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import A2UICore +import A2UIJSON +import Foundation +import JSONSchema +import OrderedJSON +import Testing + +@MainActor +private final class SpyUserActivationFunction: FunctionImplementation, @unchecked Sendable { + let api: FunctionAPI + var callCount = 0 + + init(name: String, requiresUserActivation: Bool) throws { + let emptySchema = try Schema(instance: #"{"type": "object"}"#) + self.api = FunctionAPI( + name: name, + returnType: .string, + schema: emptySchema, + allowedCallers: .rendererOnly, + requiresUserActivation: requiresUserActivation + ) + } + + func evaluate(arguments: [String: JSONValue], context: DataContext) throws -> JSONValue { + callCount += 1 + return .string("ran-\(api.name)") + } +} + +@MainActor +private final class RecordingFunctionHandler: FunctionHandler { + private let functions: [String: any FunctionImplementation] + var recordedErrors: [(error: any Error, functionName: String)] = [] + + init(functions: [any FunctionImplementation]) { + self.functions = Dictionary(uniqueKeysWithValues: functions.map { ($0.api.name, $0) }) + } + + func function(named name: String, catalogID: String?) -> (any FunctionImplementation)? { + functions[name] + } + + func handleFunctionError(_ error: any Error, functionName: String) { + recordedErrors.append((error, functionName)) + } +} + +@MainActor +struct UserActivationActionTests { + private static func makeButtonCatalog( + functions: [any FunctionImplementation] + ) throws -> AnyCatalog { + let common = "https://a2ui.org/schemas/v0_9_1/common.json#/$defs" + let buttonSchema = try Schema( + instance: """ + { + "type": "object", + "properties": { + "action": { "$ref": "\(common)/Action" } + } + } + """, + remoteSchemas: A2UICommonSchema.allSchemas + ) + return Catalog( + id: "test-cat", + protocolVersion: .v10, + components: [AnyComponentAPI(name: "Button", schema: buttonSchema)], + functions: functions + ).eraseToAnyCatalog() + } + + private static func makeButtonResolver( + functionName: String, + catalog: AnyCatalog, + actionHandler: TestProcessorActionHandler + ) -> NodeResolver { + let resolver = NodeResolver( + surfaceID: "main", + catalogs: ["test-cat": catalog], + defaultCatalogID: "test-cat", + actionHandler: actionHandler, + protocolVersion: .v10 + ) + resolver.componentsModel.addComponent( + ComponentModel( + id: "root", + type: "Button", + properties: [ + "action": .object([ + "functionCall": .object([ + "@call": .string(functionName), + "args": .object([:]), + ]) + ]) + ] + ) + ) + return resolver + } + + @Test func gatedActionRefusedWithoutUserActivation() throws { + let gatedFn = try SpyUserActivationFunction( + name: "openExternal", + requiresUserActivation: true + ) + let catalog = try Self.makeButtonCatalog(functions: [gatedFn]) + let actionHandler = TestProcessorActionHandler() + let resolver = Self.makeButtonResolver( + functionName: "openExternal", + catalog: catalog, + actionHandler: actionHandler + ) + + let root = try #require(resolver.resolveTree()) + let programmaticAction = try #require(root.action(for: "action", userActivation: false)) + programmaticAction() + + #expect(gatedFn.callCount == 0) + #expect(actionHandler.capturedActions.isEmpty) + #expect(actionHandler.capturedErrors.count == 1) + if case .generic(let err) = actionHandler.capturedErrors.first { + #expect(err.code == "EXPRESSION_ERROR") + #expect(err.surfaceID == "main") + #expect(err.expression == "openExternal") + #expect( + err.message == "Function 'openExternal' requires user activation context to execute." + ) + } else { + Issue.record("Expected EXPRESSION_ERROR generic error, got \(actionHandler.capturedErrors)") + } + } + + @Test func gatedActionRunsWithUserActivation() throws { + let gatedFn = try SpyUserActivationFunction( + name: "openExternal", + requiresUserActivation: true + ) + let catalog = try Self.makeButtonCatalog(functions: [gatedFn]) + let actionHandler = TestProcessorActionHandler() + let resolver = Self.makeButtonResolver( + functionName: "openExternal", + catalog: catalog, + actionHandler: actionHandler + ) + + let root = try #require(resolver.resolveTree()) + let gestureAction = try #require(root.action(for: "action", userActivation: true)) + gestureAction() + + #expect(gatedFn.callCount == 1) + #expect(actionHandler.capturedActions.isEmpty) + #expect(actionHandler.capturedErrors.isEmpty) + + let legacyAction = try #require(root.action(for: "action")) + legacyAction() + + #expect(gatedFn.callCount == 2) + #expect(actionHandler.capturedErrors.isEmpty) + } + + @Test func ungatedActionRunsWithOrWithoutUserActivation() throws { + let ungatedFn = try SpyUserActivationFunction( + name: "increment", + requiresUserActivation: false + ) + let catalog = try Self.makeButtonCatalog(functions: [ungatedFn]) + let actionHandler = TestProcessorActionHandler() + let resolver = Self.makeButtonResolver( + functionName: "increment", + catalog: catalog, + actionHandler: actionHandler + ) + + let root = try #require(resolver.resolveTree()) + let withoutActivation = try #require(root.action(for: "action", userActivation: false)) + withoutActivation() + #expect(ungatedFn.callCount == 1) + #expect(actionHandler.capturedErrors.isEmpty) + + let withActivation = try #require(root.action(for: "action", userActivation: true)) + withActivation() + #expect(ungatedFn.callCount == 2) + #expect(actionHandler.capturedErrors.isEmpty) + } + + @Test func dataContextRefusesGatedFunctionWithoutUserActivation() throws { + let gatedFn = try SpyUserActivationFunction( + name: "openExternal", + requiresUserActivation: true + ) + let handler = RecordingFunctionHandler(functions: [gatedFn]) + let dataModel = DataModel() + let callExpr: JSONValue = .object([ + "@call": .string("openExternal"), + "args": .object([:]), + ]) + + let inactiveContext = DataContext( + dataModel: dataModel, + path: "/", + functionHandler: handler, + protocolVersion: .v10, + userActivation: false + ) + let deniedResult = inactiveContext.resolveDynamicValue(callExpr) + #expect(deniedResult == .null) + #expect(gatedFn.callCount == 0) + #expect(handler.recordedErrors.count == 1) + + let recordedError = try #require(handler.recordedErrors.first?.error as? FunctionError) + #expect(recordedError == .userActivationRequired(name: "openExternal")) + #expect(recordedError.rpcCode == "INVALID_FUNCTION_CALL") + #expect( + recordedError.description + == "Function 'openExternal' requires user activation context to execute." + ) + + let activeContext = DataContext( + dataModel: dataModel, + path: "/", + functionHandler: handler, + protocolVersion: .v10, + userActivation: true + ) + let overrideDenied = activeContext.resolveDynamicValue(callExpr, userActivation: false) + #expect(overrideDenied == .null) + #expect(gatedFn.callCount == 0) + #expect(handler.recordedErrors.count == 2) + + let allowedResult = activeContext.resolveDynamicValue(callExpr) + #expect(allowedResult == .string("ran-openExternal")) + #expect(gatedFn.callCount == 1) + } +} From 0e98b9a36cedba2803a9f83f8dd31f1606d7f300 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:55:43 +0000 Subject: [PATCH 02/48] fix(swift-core): gate passive bindings, checks, and event actions on user activation --- .../A2UICore/Resolution/NodeResolver.swift | 21 ++++-- .../UserActivationActionTests.swift | 68 +++++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift b/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift index 7a50e4bfea..ed3bdf6362 100644 --- a/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift +++ b/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift @@ -1194,7 +1194,7 @@ public final class NodeResolver: Sendable { _ value: JSONValue, basePath: String?, index: Int? = nil, - userActivation: Bool = true + userActivation: Bool = false ) -> JSONValue { let currentData = dataModel.data let previousReadHook = dataModel.onReadPath @@ -1214,7 +1214,7 @@ public final class NodeResolver: Sendable { index: index, userActivation: userActivation ) - return context.resolveDynamicValue(value, userActivation: userActivation) + return context.resolveDynamicValue(value) } private func coerceToString(_ value: JSONValue?) -> String? { @@ -1609,7 +1609,7 @@ public final class NodeResolver: Sendable { dataContextPath: basePath, rawUserMessage: userMessageValue, metadata: actionMetadata, - triggerWithMetadata: { [weak self] callMetadata in + triggerWithUserActivation: { [weak self] userActivation, callMetadata in guard let self else { return } guard self.reportFailedActionChecksIfNeeded(checks, componentID: componentID) else { return @@ -1617,10 +1617,16 @@ public final class NodeResolver: Sendable { let resolvedContext = self.evaluateActionContext( contextDictionary, basePath: basePath, - index: index + index: index, + userActivation: userActivation ) let resolvedUserMessage = userMessageValue.flatMap { - self.evaluateDynamicValue($0, basePath: basePath, index: index).stringValue + self.evaluateDynamicValue( + $0, + basePath: basePath, + index: index, + userActivation: userActivation + ).stringValue } let validatedBase = try MetadataValidator.validate( rawMetadata, @@ -1700,11 +1706,12 @@ public final class NodeResolver: Sendable { private func evaluateActionContext( _ contextDictionary: [String: JSONValue]?, basePath: String?, - index: Int? + index: Int?, + userActivation: Bool = false ) -> [String: JSONValue] { guard let contextDictionary else { return [:] } return contextDictionary.mapValues { - evaluateDynamicValue($0, basePath: basePath, index: index) + evaluateDynamicValue($0, basePath: basePath, index: index, userActivation: userActivation) } } diff --git a/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift b/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift index 0c7d981b86..98bfad759a 100644 --- a/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift +++ b/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift @@ -70,6 +70,8 @@ struct UserActivationActionTests { { "type": "object", "properties": { + "label": { "$ref": "\(common)/DynamicString" }, + "checks": { "type": "array", "items": { "$ref": "\(common)/CheckRule" } }, "action": { "$ref": "\(common)/Action" } } } @@ -246,4 +248,70 @@ struct UserActivationActionTests { #expect(allowedResult == .string("ran-openExternal")) #expect(gatedFn.callCount == 1) } + + @Test func passiveBindingsAndChecksRefuseGatedFunctions() throws { + let gatedFn = try SpyUserActivationFunction( + name: "openExternal", + requiresUserActivation: true + ) + let catalog = try Self.makeButtonCatalog(functions: [gatedFn]) + let actionHandler = TestProcessorActionHandler() + let resolver = NodeResolver( + surfaceID: "main", + catalogs: ["test-cat": catalog], + defaultCatalogID: "test-cat", + actionHandler: actionHandler, + protocolVersion: .v10 + ) + let callExpr: JSONValue = .object([ + "@call": .string("openExternal"), + "args": .object([:]), + ]) + resolver.componentsModel.addComponent( + ComponentModel( + id: "root", + type: "Button", + properties: [ + "label": callExpr, + "checks": .array([ + .object([ + "condition": callExpr, + "message": .string("Blocked check"), + ]) + ]), + ] + ) + ) + + let root = try #require(resolver.resolveTree()) + #expect(root.string(for: "label") == nil) + #expect(root.isValid == false) + #expect(gatedFn.callCount == 0) + #expect(actionHandler.capturedErrors.count == 2) + + resolver.componentsModel.addComponent( + ComponentModel( + id: "root", + type: "Button", + properties: [ + "action": .object([ + "event": .object([ + "name": .string("submit"), + "context": .object(["urlResult": callExpr]), + ]) + ]) + ] + ) + ) + + let eventRoot = try #require(resolver.resolveTree()) + let programmaticEvent = try #require(eventRoot.action(for: "action", userActivation: false)) + programmaticEvent() + #expect(gatedFn.callCount == 0) + #expect(actionHandler.capturedErrors.count == 3) + + let gestureEvent = try #require(eventRoot.action(for: "action", userActivation: true)) + gestureEvent() + #expect(gatedFn.callCount == 1) + } } From a7e8a8a84ad19958c15e4a6dc737b99afe4b5a6d Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 01:57:39 +0000 Subject: [PATCH 03/48] test(conformance): add executed-case manifest schema and comparer --- conformance/README.md | 2 + ...onformance_harness_implementation_guide.md | 38 +- .../executed_case_manifest.schema.json | 49 ++ conformance/manifest_corpora.json | 8 + conformance/scripts/check_executed_cases.py | 481 +++++++++++++++ .../tests/test_check_executed_cases.py | 548 ++++++++++++++++++ 6 files changed, 1125 insertions(+), 1 deletion(-) create mode 100644 conformance/executed_case_manifest.schema.json create mode 100644 conformance/manifest_corpora.json create mode 100644 conformance/scripts/check_executed_cases.py create mode 100644 conformance/tests/test_check_executed_cases.py diff --git a/conformance/README.md b/conformance/README.md index 6eaf8266fd..07d3ff09ba 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -103,6 +103,8 @@ Every conformance test runner must declare two top-level configuration variables 1. **`SUPPORTED_PROTOCOL_VERSIONS`**: A set/list of A2UI protocol versions supported by the SDK or harness (e.g. `{'0.8', '0.9', '1.0'}`). Test cases specifying a version outside this set are skipped cleanly. 2. **`SKIP_TEST_NAMES`**: A transition skip list containing test case names to temporarily skip during active feature transitions. At the start of a feature migration, test names for unimplemented features are added to this set and progressively removed as feature implementations complete. Skipped tests are logged as `[SKIPPED]` without marking test runs as failures. +When `A2UI_CONFORMANCE_MANIFEST_DIR` is set in CI, each harness writes one JSON Lines record per executed or skipped case to that directory, validated against [`executed_case_manifest.schema.json`](executed_case_manifest.schema.json), and [`scripts/check_executed_cases.py`](scripts/check_executed_cases.py) compares the recorded cases against the suite globs in [`manifest_corpora.json`](manifest_corpora.json). See [Executed-case manifest](conformance_harness_implementation_guide.md#5-executed-case-manifest) in the harness implementation guide for the record format, strict dispatch rules, and comparer flags. + Client-side implementations run these suites too: > **Note**: Conformance tests provide the primary verification of protocol and inference format specifications across languages. Unit tests in SDK packages are reserved for language-specific implementation residue (e.g., exception types, thread-safety, stream lifecycle). To test everything, both unit tests and conformance tests must be run. diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index 2653adfb82..6c4eda1120 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -79,7 +79,11 @@ Configure two top-level constants in your runner as described in [README.md](REA Follow three safeguards so your test suite never hides regressions behind green results: 1. Assert in every suite file that the loaded case list is not empty. If a file path breaks during refactoring, the suite fails immediately instead of passing zero tests. -2. Throw an unhandled-action error in the `default` branch of every `action` dispatcher. Leaving out a `default` branch lets newly added actions pass without running any assertions. The Python Core harness goes one step further and fails when a suite file has no handler at all unless the file is listed in `UNRUNNABLE_SUITES` with a reason. +2. Enforce the strict dispatch rule across every dispatcher and handler: + 1. An unknown `action`, step `op`, `args` key, scenario key, step key, or `expect*` key fails the case with `Unknown '' in #`. The case-level metadata keys `name`, `description`, and `features` are always accepted and ignored. + 2. The only escape is a per-harness `UNSUPPORTED` map from `""` or `"."` to a reason. A matching case is recorded as `skipped` with that reason, never as `passed`. An `UNSUPPORTED` or expected-failure entry that matches no case fails the run. + 3. Each handler declares its keys in one table next to the handler (Python `_KNOWN_KEYS: dict[str, frozenset[str]]`, TypeScript `KNOWN_KEYS` `Map`, Dart `const Map>`, Swift `static let knownKeys: [String: Set]`) and calls one helper: Python `_assert_known_keys(obj: Mapping[str, Any], allowed: frozenset[str], where: str) -> None`, TypeScript `assertKnownKeys(obj, allowed, where)`, Dart `void assertKnownKeys(Map obj, Set allowed, String where)`, Swift `func assertKnownKeys(_ obj: [String: Any], _ allowed: Set, _ where: String) throws`. + 4. Every harness writes the [executed-case manifest](#5-executed-case-manifest), and CI runs the comparer in the same job right after the tests. 3. Separate unimplemented capabilities from defects in implemented code, following [typescript/agents/a2ui_agent/tests/conformance/loader.ts](../typescript/agents/a2ui_agent/tests/conformance/loader.ts) and [dart/a2ui_core/test/conformance/conformance_harness.dart](../dart/a2ui_core/test/conformance/conformance_harness.dart). Mark unsupported versions or unbuilt modules as `[SKIPPED]`, and mark known bugs with a strict expected-failure marker (`pytest.mark.xfail(strict=True)`, Vitest `test.fails`, or a name-to-reason `expectedFailures` map). When a code change fixes a known bug, a strict expected failure fails CI right away so you delete the stale entry. ### 3. Mapping error categories @@ -106,6 +110,38 @@ Pay close attention to class inheritance if your target language supports only s The `scaling.yaml` suites and `core/message_processor_scaling.yaml` time an operation rather than check its result. A `measure_growth` case times `operation` on a `PayloadTemplate` expanded at two `sizes`, n and 4n, and expects the larger run to take at most `max_ratio` times as long. A `measure_overhead` case times streaming a payload against compiling it. Tag these tests so regular unit test runs skip them and CI runs them in an isolated job. The Dart packages tag them `performance` (`dart test -P performance`), the Python packages mark them `performance` (`uv run pytest -m performance`), and the TypeScript packages run them with `yarn test:timing` through [typescript/core/web_core/tests/timing/timing.mjs](../typescript/core/web_core/tests/timing/timing.mjs). Build catalogs and formats outside the timer and expand the template (`repeat` segments replacing `{i}` joined by `separator`). Run one untimed warmup pass, then alternate at least five measured runs of each size before comparing medians. Avoid forcing a major garbage collection inside the loop: in V8 or the JVM, a forced full GC between runs can deoptimize compiled code and push `4n` past `max_ratio`. +### 5. Executed-case manifest + +Every conformance harness records each case it runs or skips so CI can verify that no suite or case was silently dropped. When the `A2UI_CONFORMANCE_MANIFEST_DIR` environment variable is set, the harness appends one JSON Lines record per finished case to `$A2UI_CONFORMANCE_MANIFEST_DIR/-.jsonl` (where `` is the process ID or test-file name so parallel runners do not share a file handle). When `A2UI_CONFORMANCE_MANIFEST_DIR` is not set, no manifest files are written. + +Each line is validated against [executed_case_manifest.schema.json](executed_case_manifest.schema.json): + +```json +{"harness": "python-core", "suite": "core/rpc_functions.yaml", "case": "test_rpc_call_agent_function_timeout_handling", "status": "passed"} +{"harness": "dart-core", "suite": "core/common_types.yaml", "case": "test_dynamic_string_accepts_binding", "status": "skipped", "reason": "validate_common_type is not implemented in this harness yet"} +``` + +- `harness`: Harness identifier (`python-core`, `web_core`, `dart-core`, `swift-core`, or the agent/adapter harness name). +- `suite`: Suite path relative to `conformance/`, using forward slashes. +- `case`: Case `name`, or test `id` for `{macros, tests}` map-form suites. +- `status`: `passed`, `failed`, `skipped`, or `xfailed`. +- `reason`: Required and non-empty when `status` is `skipped` or `xfailed`. + +Suite corpora (`core`, `agent`, `extensions`, and `adapter`) are listed in [manifest_corpora.json](manifest_corpora.json). Every Core harness must account for every case in `core/*.yaml`, recording the adapter suites (`core/adapter_conformance.yaml` and `core/accessibility.yaml`) as `skipped` with a reason when run headlessly. After the test step finishes, CI runs the comparer: + +```bash +uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "$A2UI_CONFORMANCE_MANIFEST_DIR" \ + --harness python-core \ + --corpus core \ + [--corpus adapter] \ + [--only SUITE_GLOB] \ + [--exclude SUITE=REASON] \ + [--matrix-out FILE.md] +``` + +The comparer fails on any `MISSING` (corpus case without a record), `DUPLICATE` (two records for one suite and case), `UNKNOWN_CASE` (record names a case the suite does not contain), `UNREASONED` (`skipped` or `xfailed` without a reason), `INVALID_RECORD` (schema validation failure), or `FAILED` (`status: "failed"`) finding, including when the manifest directory is empty. Records for suites outside the requested corpora are ignored. Repeat `--only SUITE_GLOB` to narrow a corpus to suites matching a glob when a harness covers only part of a corpus (such as `--corpus extensions --only 'extensions/a2a/**'`). Repeat `--exclude SUITE=REASON` for suites that a test runner filters out before harness code executes (such as performance-tagged suites in Dart and Swift); the comparer prints each excluded suite and reason. + ## Implementation order across tiers Implement the suites in stages so each layer is tested before higher-level components depend on it: diff --git a/conformance/executed_case_manifest.schema.json b/conformance/executed_case_manifest.schema.json new file mode 100644 index 0000000000..4256ac2415 --- /dev/null +++ b/conformance/executed_case_manifest.schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/conformance/executed_case_manifest.schema.json", + "$comment": "Copyright 2024 Google LLC. Licensed under the Apache License, Version 2.0 (https://www.apache.org/licenses/LICENSE-2.0).", + "title": "A2UI Conformance Executed-Case Manifest Record", + "description": "JSON Schema validating a single JSON Lines record in an A2UI conformance executed-case manifest.", + "type": "object", + "additionalProperties": false, + "required": ["harness", "suite", "case", "status"], + "properties": { + "harness": { + "type": "string", + "minLength": 1, + "description": "Identifier of the conformance harness that executed the test case." + }, + "suite": { + "type": "string", + "minLength": 1, + "description": "Suite file path relative to conformance/, using forward slashes." + }, + "case": { + "type": "string", + "minLength": 1, + "description": "Test case name, or test id for map-form suites." + }, + "status": { + "type": "string", + "enum": ["passed", "failed", "skipped", "xfailed"], + "description": "Execution status of the test case." + }, + "reason": { + "type": "string", + "minLength": 1, + "pattern": "\\S", + "description": "Non-empty explanation required when status is skipped or xfailed." + } + }, + "if": { + "properties": { + "status": { + "enum": ["skipped", "xfailed"] + } + }, + "required": ["status"] + }, + "then": { + "required": ["reason"] + } +} diff --git a/conformance/manifest_corpora.json b/conformance/manifest_corpora.json new file mode 100644 index 0000000000..33eac8b9d9 --- /dev/null +++ b/conformance/manifest_corpora.json @@ -0,0 +1,8 @@ +{ + "core": {"include": ["core/*.yaml"]}, + "agent": {"include": ["agent/**/*.yaml"]}, + "extensions": {"include": ["extensions/**/*.yaml"]}, + "adapter": { + "include": ["core/adapter_conformance.yaml", "core/accessibility.yaml"] + } +} diff --git a/conformance/scripts/check_executed_cases.py b/conformance/scripts/check_executed_cases.py new file mode 100644 index 0000000000..7993afe480 --- /dev/null +++ b/conformance/scripts/check_executed_cases.py @@ -0,0 +1,481 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compares executed conformance case manifests against YAML suite corpora.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import json +from pathlib import Path +import re +import sys +from typing import Any + +import jsonschema +import yaml + +CONFORMANCE_ROOT = Path(__file__).resolve().parent.parent +SCHEMA_PATH = CONFORMANCE_ROOT / "executed_case_manifest.schema.json" +CORPORA_PATH = CONFORMANCE_ROOT / "manifest_corpora.json" + +STATUSES: tuple[str, ...] = ("passed", "failed", "skipped", "xfailed") +REASON_REQUIRED_STATUSES: frozenset[str] = frozenset({"skipped", "xfailed"}) + +_SCHEMA: dict[str, Any] = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) +_VALIDATOR = jsonschema.Draft202012Validator(_SCHEMA) + + +@dataclass(frozen=True) +class ManifestRecord: + """Single executed-case manifest entry.""" + + harness: str + suite: str + case: str + status: str + reason: str | None = None + error: str | None = None + + +@dataclass(frozen=True) +class Finding: + """Fatal discrepancy discovered while comparing a manifest to a corpus.""" + + kind: str + suite: str + case: str | None + message: str + + def __str__(self) -> str: + return f"{self.kind}: {self.message}" + + +def _translate_glob_segment(segment: str) -> str: + parts: list[str] = [] + for char in segment: + if char == "*": + parts.append("[^/]*") + elif char == "?": + parts.append("[^/]") + else: + parts.append(re.escape(char)) + return "".join(parts) + + +def _glob_to_regex(pattern: str) -> re.Pattern[str]: + normalized = pattern.replace("\\", "/").strip("/") + segments = [seg for seg in normalized.split("/") if seg] + regex_parts: list[str] = ["^"] + last_idx = len(segments) - 1 + for idx, seg in enumerate(segments): + is_last = idx == last_idx + if seg == "**": + regex_parts.append("(?:.+)" if is_last else "(?:[^/]+/)*") + else: + regex_parts.append(_translate_glob_segment(seg)) + if not is_last: + regex_parts.append("/") + regex_parts.append("$") + return re.compile("".join(regex_parts)) + + +def _matches_any_glob(suite: str, patterns: Sequence[str]) -> bool: + return any(_glob_to_regex(pat).match(suite) is not None for pat in patterns) + + +def _extract_suite_cases(suite_path: Path) -> set[str]: + data = yaml.safe_load(suite_path.read_text(encoding="utf-8")) + if isinstance(data, list): + return { + str(item["name"]) + for item in data + if isinstance(item, dict) and item.get("name") + } + if isinstance(data, dict) and isinstance(data.get("tests"), list): + return { + str(item.get("id") or item.get("name")) + for item in data["tests"] + if isinstance(item, dict) and (item.get("id") or item.get("name")) + } + raise ValueError(f"Unsupported suite YAML structure in {suite_path}") + + +def _iter_pattern_files(conformance_root: Path, patterns: Sequence[str]) -> list[Path]: + files: list[Path] = [] + for pattern in patterns: + files.extend( + path for path in sorted(conformance_root.glob(pattern)) if path.is_file() + ) + return files + + +def load_corpus( + conformance_root: Path, + corpus_names: Sequence[str], + only: Sequence[str] | None = None, +) -> dict[str, set[str]]: + """Loads suite-to-case sets for the named corpora from manifest_corpora.json.""" + corpora_file = conformance_root / "manifest_corpora.json" + corpora_spec: dict[str, Any] = json.loads(corpora_file.read_text(encoding="utf-8")) + corpus: dict[str, set[str]] = {} + for name in corpus_names: + if name not in corpora_spec: + raise ValueError(f"Unknown corpus {name!r} in {corpora_file}") + patterns: Sequence[str] = corpora_spec[name].get("include", []) + for suite_path in _iter_pattern_files(conformance_root, patterns): + rel_suite = suite_path.relative_to(conformance_root).as_posix() + corpus[rel_suite] = _extract_suite_cases(suite_path) + if not only: + return corpus + return { + suite: cases + for suite, cases in corpus.items() + if _matches_any_glob(suite, only) + } + + +def _is_unreasoned_only(raw: Mapping[str, Any]) -> bool: + if raw.get("status") not in REASON_REQUIRED_STATUSES: + return False + reason = raw.get("reason") + if reason is not None and not isinstance(reason, str): + return False + if isinstance(reason, str) and reason.strip(): + return False + probe = {**raw, "reason": "placeholder"} + return next(_VALIDATOR.iter_errors(probe), None) is None + + +def _parse_manifest_line( + line: str, + source: str, + harness: str, +) -> ManifestRecord | None: + try: + raw = json.loads(line) + except json.JSONDecodeError as err: + return ManifestRecord( + harness=harness, + suite="", + case="", + status="invalid", + error=f"{source}: invalid JSON: {err.msg}", + ) + if not isinstance(raw, dict): + return ManifestRecord( + harness=harness, + suite="", + case="", + status="invalid", + error=f"{source}: record must be a JSON object", + ) + rec_harness = raw.get("harness") + if isinstance(rec_harness, str) and rec_harness != harness: + return None + first_error = next(_VALIDATOR.iter_errors(raw), None) + if first_error is None or _is_unreasoned_only(raw): + raw_reason = raw.get("reason") + return ManifestRecord( + harness=str(raw["harness"]), + suite=str(raw["suite"]), + case=str(raw["case"]), + status=str(raw["status"]), + reason=raw_reason if isinstance(raw_reason, str) else None, + ) + return ManifestRecord( + harness=str(rec_harness or harness), + suite=str(raw.get("suite") or ""), + case=str(raw.get("case") or ""), + status=str(raw.get("status") or "invalid"), + reason=raw.get("reason") if isinstance(raw.get("reason"), str) else None, + error=f"{source}: {first_error.message}", + ) + + +def load_records(manifest_dir: Path, harness: str) -> list[ManifestRecord]: + """Loads and schema-validates JSONL manifest records for the given harness.""" + if not manifest_dir.exists(): + return [] + records: list[ManifestRecord] = [] + for jsonl_path in sorted(manifest_dir.glob("*.jsonl")): + lines = jsonl_path.read_text(encoding="utf-8").splitlines() + for lineno, raw_line in enumerate(lines, start=1): + stripped = raw_line.strip() + if not stripped: + continue + source = f"{jsonl_path.name}:{lineno}" + parsed = _parse_manifest_line(stripped, source, harness) + if parsed is not None: + records.append(parsed) + return records + + +def _record_schema_error(record: ManifestRecord) -> str | None: + if record.error is not None: + return record.error + payload: dict[str, Any] = { + "harness": record.harness, + "suite": record.suite, + "case": record.case, + "status": record.status, + } + if record.reason is not None: + payload["reason"] = record.reason + first_error = next(_VALIDATOR.iter_errors(payload), None) + if first_error is None or _is_unreasoned_only(payload): + return None + return first_error.message + + +def _check_single_record( + record: ManifestRecord, + corpus: Mapping[str, set[str]], + excluded_map: Mapping[str, str], + seen: dict[tuple[str, str], int], +) -> list[Finding]: + schema_err = _record_schema_error(record) + if schema_err is not None: + return [ + Finding( + kind="INVALID_RECORD", + suite=record.suite, + case=record.case or None, + message=schema_err, + ) + ] + if record.suite not in corpus or record.suite in excluded_map: + return [] + if record.case not in corpus[record.suite]: + return [ + Finding( + kind="UNKNOWN_CASE", + suite=record.suite, + case=record.case, + message=f"Unknown case {record.case!r} in suite {record.suite}", + ) + ] + findings: list[Finding] = [] + key = (record.suite, record.case) + seen[key] = seen.get(key, 0) + 1 + if seen[key] > 1: + findings.append( + Finding( + kind="DUPLICATE", + suite=record.suite, + case=record.case, + message=f"Duplicate record for {record.suite}#{record.case}", + ) + ) + if record.status in REASON_REQUIRED_STATUSES and not ( + record.reason and record.reason.strip() + ): + findings.append( + Finding( + kind="UNREASONED", + suite=record.suite, + case=record.case, + message=( + f"Status {record.status!r} requires a non-empty reason for" + f" {record.suite}#{record.case}" + ), + ) + ) + if record.status == "failed": + findings.append( + Finding( + kind="FAILED", + suite=record.suite, + case=record.case, + message=f"Case failed: {record.suite}#{record.case}", + ) + ) + return findings + + +def compare( + corpus: Mapping[str, set[str]], + records: Sequence[ManifestRecord], + excluded: Mapping[str, str] | None = None, +) -> list[Finding]: + """Compares manifest records against expected corpus suites and cases.""" + excluded_map = dict(excluded or {}) + findings: list[Finding] = [] + seen: dict[tuple[str, str], int] = {} + + for record in records: + findings.extend(_check_single_record(record, corpus, excluded_map, seen)) + + for suite in sorted(corpus): + if suite in excluded_map: + continue + for case in sorted(corpus[suite]): + if (suite, case) not in seen: + findings.append( + Finding( + kind="MISSING", + suite=suite, + case=case, + message=f"Missing record for {suite}#{case}", + ) + ) + return findings + + +def render_matrix(records: Sequence[ManifestRecord]) -> str: + """Renders a Markdown table of suite-by-status counts.""" + counts: dict[str, dict[str, int]] = {} + for record in records: + if record.error is not None or not record.suite: + continue + suite_counts = counts.setdefault( + record.suite, {status: 0 for status in STATUSES} + ) + if record.status in suite_counts: + suite_counts[record.status] += 1 + + lines = [ + "| Suite | Passed | Failed | Skipped | XFailed | Total |", + "| :--- | ---: | ---: | ---: | ---: | ---: |", + ] + totals = {status: 0 for status in STATUSES} + for suite in sorted(counts): + row = counts[suite] + row_total = sum(row.values()) + for status in STATUSES: + totals[status] += row[status] + lines.append( + f"| `{suite}` | {row['passed']} | {row['failed']} | {row['skipped']} |" + f" {row['xfailed']} | {row_total} |" + ) + grand_total = sum(totals.values()) + lines.append( + f"| **Total** | **{totals['passed']}** | **{totals['failed']}** |" + f" **{totals['skipped']}** | **{totals['xfailed']}** |" + f" **{grand_total}** |" + ) + return "\n".join(lines) + "\n" + + +def _parse_exclusions( + raw_excludes: Sequence[str], + parser: argparse.ArgumentParser, +) -> dict[str, str]: + excluded: dict[str, str] = {} + for item in raw_excludes: + suite, sep, reason = item.partition("=") + suite = suite.strip() + reason = reason.strip() + if not sep or not suite or not reason: + parser.error( + f"Invalid --exclude value {item!r}; expected SUITE=REASON with a" + " non-empty reason." + ) + excluded[suite] = reason + return excluded + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Verify that an A2UI conformance harness recorded every case in its" + " target corpora." + ) + ) + parser.add_argument( + "--manifest-dir", + type=Path, + required=True, + help="Directory containing JSONL manifest files.", + ) + parser.add_argument( + "--harness", + type=str, + required=True, + help="Harness name to check.", + ) + parser.add_argument( + "--corpus", + action="append", + required=True, + dest="corpora", + help="Corpus name from manifest_corpora.json (repeatable).", + ) + parser.add_argument( + "--only", + action="append", + default=[], + metavar="SUITE_GLOB", + help="Narrow the corpus to suites matching a glob (repeatable).", + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + metavar="SUITE=REASON", + help="Exclude a suite with a required reason (repeatable).", + ) + parser.add_argument( + "--matrix-out", + type=Path, + default=None, + help="Optional path to write the Markdown suite status matrix.", + ) + parser.add_argument( + "--conformance-root", + type=Path, + default=CONFORMANCE_ROOT, + help="Path to the conformance directory (defaults to conformance/).", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """CLI entry point for checking executed conformance cases.""" + parser = _build_parser() + args = parser.parse_args(argv) + excluded = _parse_exclusions(args.exclude, parser) + for suite, reason in sorted(excluded.items()): + print(f"Excluded {suite}: {reason}") + + corpus = load_corpus(args.conformance_root, args.corpora, only=args.only) + records = load_records(args.manifest_dir, args.harness) + findings = compare(corpus, records, excluded) + + if args.matrix_out is not None: + in_corpus_records = [ + rec for rec in records if rec.suite in corpus and rec.error is None + ] + args.matrix_out.parent.mkdir(parents=True, exist_ok=True) + args.matrix_out.write_text(render_matrix(in_corpus_records), encoding="utf-8") + + if findings: + for finding in findings: + print(finding, file=sys.stderr) + return 1 + + total_cases = sum( + len(cases) for suite, cases in corpus.items() if suite not in excluded + ) + print( + f"OK ({args.harness}): verified {total_cases} cases across" + f" {len(corpus) - len(excluded & corpus.keys())} active suites." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/tests/test_check_executed_cases.py b/conformance/tests/test_check_executed_cases.py new file mode 100644 index 0000000000..0020a741ab --- /dev/null +++ b/conformance/tests/test_check_executed_cases.py @@ -0,0 +1,548 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for conformance/scripts/check_executed_cases.py.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys +from typing import Any + +import pytest +import yaml + +CONFORMANCE_DIR = Path(__file__).resolve().parent.parent +SCRIPT_PATH = CONFORMANCE_DIR / "scripts" / "check_executed_cases.py" + +_SPEC = importlib.util.spec_from_file_location("check_executed_cases", SCRIPT_PATH) +assert _SPEC is not None and _SPEC.loader is not None +check_executed_cases = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = check_executed_cases +_SPEC.loader.exec_module(check_executed_cases) + +Finding = check_executed_cases.Finding +ManifestRecord = check_executed_cases.ManifestRecord +compare = check_executed_cases.compare +load_corpus = check_executed_cases.load_corpus +load_records = check_executed_cases.load_records +main = check_executed_cases.main +render_matrix = check_executed_cases.render_matrix + + +def _create_temp_conformance(tmp_path: Path) -> tuple[Path, Path]: + """Creates a temporary conformance root and manifest directory.""" + conf_root = tmp_path / "conformance" + (conf_root / "core").mkdir(parents=True) + (conf_root / "agent" / "builder").mkdir(parents=True) + (conf_root / "extensions" / "a2a").mkdir(parents=True) + (conf_root / "extensions" / "adk").mkdir(parents=True) + + corpora = { + "core": {"include": ["core/*.yaml"]}, + "agent": {"include": ["agent/**/*.yaml"]}, + "extensions": {"include": ["extensions/**/*.yaml"]}, + } + (conf_root / "manifest_corpora.json").write_text( + json.dumps(corpora), encoding="utf-8" + ) + + (conf_root / "core" / "alpha.yaml").write_text( + yaml.safe_dump([ + {"name": "case_one", "action": "validate"}, + {"name": "case_two", "action": "validate"}, + ]), + encoding="utf-8", + ) + (conf_root / "core" / "beta.yaml").write_text( + yaml.safe_dump([ + {"name": "case_three", "action": "validate"}, + ]), + encoding="utf-8", + ) + (conf_root / "agent" / "builder" / "builder.yaml").write_text( + yaml.safe_dump({ + "suite": "builder_suite", + "macros": [], + "tests": [ + {"id": "01_primitive"}, + {"id": "02_nested"}, + ], + }), + encoding="utf-8", + ) + (conf_root / "extensions" / "a2a" / "a2a_integration.yaml").write_text( + yaml.safe_dump([{"name": "a2a_case", "action": "convert"}]), + encoding="utf-8", + ) + (conf_root / "extensions" / "adk" / "adk_extensions.yaml").write_text( + yaml.safe_dump([{"name": "adk_case", "action": "convert"}]), + encoding="utf-8", + ) + + manifest_dir = tmp_path / "manifest" + manifest_dir.mkdir() + return conf_root, manifest_dir + + +def _write_jsonl(path: Path, lines: list[dict[str, Any] | str]) -> None: + serialized = [line if isinstance(line, str) else json.dumps(line) for line in lines] + path.write_text("\n".join(serialized) + "\n", encoding="utf-8") + + +def test_manifest_corpora_globs_match_real_files() -> None: + corpora_data = json.loads( + (CONFORMANCE_DIR / "manifest_corpora.json").read_text(encoding="utf-8") + ) + assert set(corpora_data) >= {"core", "agent", "extensions", "adapter"} + for corpus_name, spec in corpora_data.items(): + patterns = spec.get("include", []) + assert patterns, f"Corpus {corpus_name!r} has no include patterns" + for pattern in patterns: + matches = [p for p in CONFORMANCE_DIR.glob(pattern) if p.is_file()] + assert ( + matches + ), f"Glob {pattern!r} in corpus {corpus_name!r} matched 0 files" + loaded = load_corpus(CONFORMANCE_DIR, [corpus_name]) + assert loaded, f"Corpus {corpus_name!r} loaded 0 suites" + assert all(loaded.values()), f"Corpus {corpus_name!r} has an empty suite" + + +def test_passing_run_produces_no_findings(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-core-1.jsonl", + [ + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "skipped", + "reason": "feature not implemented yet", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "xfailed", + "reason": "known issue tracked separately", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + assert compare(corpus, records, {}) == [] + + +def test_empty_manifest_directory_fails_with_missing_for_every_case( + tmp_path: Path, +) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert len(findings) == 3 + assert {f.kind for f in findings} == {"MISSING"} + assert {(f.suite, f.case) for f in findings} == { + ("core/alpha.yaml", "case_one"), + ("core/alpha.yaml", "case_two"), + ("core/beta.yaml", "case_three"), + } + + +def test_finding_missing(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-core.jsonl", + [ + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert [f.kind for f in findings] == ["MISSING"] + assert findings[0].suite == "core/alpha.yaml" + assert findings[0].case == "case_two" + + +def test_finding_duplicate(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-core.jsonl", + [ + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert [f.kind for f in findings] == ["DUPLICATE"] + assert findings[0].suite == "core/alpha.yaml" + assert findings[0].case == "case_one" + + +def test_finding_unknown_case(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-core.jsonl", + [ + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "nonexistent_case", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert [f.kind for f in findings] == ["UNKNOWN_CASE"] + assert findings[0].suite == "core/alpha.yaml" + assert findings[0].case == "nonexistent_case" + + +@pytest.mark.parametrize("status", ["skipped", "xfailed"]) +@pytest.mark.parametrize("reason", [None, "", " "]) +def test_finding_unreasoned(tmp_path: Path, status: str, reason: str | None) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + bad_record: dict[str, Any] = { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": status, + } + if reason is not None: + bad_record["reason"] = reason + _write_jsonl( + manifest_dir / "python-core.jsonl", + [ + bad_record, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert [f.kind for f in findings] == ["UNREASONED"] + assert findings[0].suite == "core/alpha.yaml" + assert findings[0].case == "case_one" + + +@pytest.mark.parametrize( + "bad_line", + [ + "{not valid json", + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "unknown_status", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + "extraField": True, + }, + ], +) +def test_finding_invalid_record(tmp_path: Path, bad_line: dict[str, Any] | str) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-core.jsonl", + [ + bad_line, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert [f.kind for f in findings] == ["INVALID_RECORD"] + + +def test_finding_failed(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-core.jsonl", + [ + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "failed", + }, + { + "harness": "python-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "passed", + }, + { + "harness": "python-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + ], + ) + corpus = load_corpus(conf_root, ["core"]) + records = load_records(manifest_dir, "python-core") + findings = compare(corpus, records, {}) + assert [f.kind for f in findings] == ["FAILED"] + assert findings[0].suite == "core/alpha.yaml" + assert findings[0].case == "case_one" + + +def test_map_form_suite_uses_id(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + corpus = load_corpus(conf_root, ["agent"]) + assert corpus == {"agent/builder/builder.yaml": {"01_primitive", "02_nested"}} + _write_jsonl( + manifest_dir / "python-agent.jsonl", + [ + { + "harness": "python-agent", + "suite": "agent/builder/builder.yaml", + "case": "01_primitive", + "status": "passed", + }, + { + "harness": "python-agent", + "suite": "agent/builder/builder.yaml", + "case": "02_nested", + "status": "passed", + }, + ], + ) + records = load_records(manifest_dir, "python-agent") + assert compare(corpus, records, {}) == [] + + +def test_only_glob_narrows_corpus(tmp_path: Path) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "python-a2a.jsonl", + [ + { + "harness": "python-a2a", + "suite": "extensions/a2a/a2a_integration.yaml", + "case": "a2a_case", + "status": "passed", + }, + ], + ) + exit_code = main([ + "--conformance-root", + str(conf_root), + "--manifest-dir", + str(manifest_dir), + "--harness", + "python-a2a", + "--corpus", + "extensions", + "--only", + "extensions/a2a/**", + ]) + assert exit_code == 0 + + +def test_exclude_skips_suite_and_prints_reason( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + _write_jsonl( + manifest_dir / "dart-core.jsonl", + [ + { + "harness": "dart-core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "dart-core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "passed", + }, + ], + ) + exit_code = main([ + "--conformance-root", + str(conf_root), + "--manifest-dir", + str(manifest_dir), + "--harness", + "dart-core", + "--corpus", + "core", + "--exclude", + "core/beta.yaml=performance tagged suite", + ]) + assert exit_code == 0 + captured = capsys.readouterr() + assert "Excluded core/beta.yaml: performance tagged suite" in captured.out + + +def test_out_of_corpus_and_other_harness_records_are_ignored( + tmp_path: Path, +) -> None: + conf_root, manifest_dir = _create_temp_conformance(tmp_path) + matrix_file = tmp_path / "matrix.md" + _write_jsonl( + manifest_dir / "mixed.jsonl", + [ + { + "harness": "web_core", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "passed", + }, + { + "harness": "web_core", + "suite": "core/alpha.yaml", + "case": "case_two", + "status": "skipped", + "reason": "skip reason", + }, + { + "harness": "web_core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "passed", + }, + { + "harness": "web_core", + "suite": "agent/builder/builder.yaml", + "case": "01_primitive", + "status": "skipped", + "reason": "agent suite skipped in web_core", + }, + { + "harness": "other-harness", + "suite": "core/alpha.yaml", + "case": "case_one", + "status": "failed", + }, + ], + ) + exit_code = main([ + "--conformance-root", + str(conf_root), + "--manifest-dir", + str(manifest_dir), + "--harness", + "web_core", + "--corpus", + "core", + "--matrix-out", + str(matrix_file), + ]) + assert exit_code == 0 + matrix_text = matrix_file.read_text(encoding="utf-8") + assert "| `core/alpha.yaml` | 1 | 0 | 1 | 0 | 2 |" in matrix_text + assert "| `core/beta.yaml` | 1 | 0 | 0 | 0 | 1 |" in matrix_text + assert "agent/builder/builder.yaml" not in matrix_text From d904d49c862a51941352804118a13e9d599d113a Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 17:42:43 +0000 Subject: [PATCH 04/48] test(conformance): return a str from the manifest record schema check --- conformance/scripts/check_executed_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/scripts/check_executed_cases.py b/conformance/scripts/check_executed_cases.py index 7993afe480..ad63642858 100644 --- a/conformance/scripts/check_executed_cases.py +++ b/conformance/scripts/check_executed_cases.py @@ -238,7 +238,7 @@ def _record_schema_error(record: ManifestRecord) -> str | None: first_error = next(_VALIDATOR.iter_errors(payload), None) if first_error is None or _is_unreasoned_only(payload): return None - return first_error.message + return str(first_error.message) def _check_single_record( From a3171914abd51529e7eede9d3c250c48a8f75d13 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 05:43:50 +0000 Subject: [PATCH 05/48] test(conformance): pre-compile only globs and check is_dir in load_records --- conformance/scripts/check_executed_cases.py | 9 +++------ conformance/tests/test_check_executed_cases.py | 7 +++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/conformance/scripts/check_executed_cases.py b/conformance/scripts/check_executed_cases.py index ad63642858..1900817311 100644 --- a/conformance/scripts/check_executed_cases.py +++ b/conformance/scripts/check_executed_cases.py @@ -93,10 +93,6 @@ def _glob_to_regex(pattern: str) -> re.Pattern[str]: return re.compile("".join(regex_parts)) -def _matches_any_glob(suite: str, patterns: Sequence[str]) -> bool: - return any(_glob_to_regex(pat).match(suite) is not None for pat in patterns) - - def _extract_suite_cases(suite_path: Path) -> set[str]: data = yaml.safe_load(suite_path.read_text(encoding="utf-8")) if isinstance(data, list): @@ -141,10 +137,11 @@ def load_corpus( corpus[rel_suite] = _extract_suite_cases(suite_path) if not only: return corpus + compiled_only = [_glob_to_regex(pat) for pat in only] return { suite: cases for suite, cases in corpus.items() - if _matches_any_glob(suite, only) + if any(rx.match(suite) is not None for rx in compiled_only) } @@ -208,7 +205,7 @@ def _parse_manifest_line( def load_records(manifest_dir: Path, harness: str) -> list[ManifestRecord]: """Loads and schema-validates JSONL manifest records for the given harness.""" - if not manifest_dir.exists(): + if not manifest_dir.is_dir(): return [] records: list[ManifestRecord] = [] for jsonl_path in sorted(manifest_dir.glob("*.jsonl")): diff --git a/conformance/tests/test_check_executed_cases.py b/conformance/tests/test_check_executed_cases.py index 0020a741ab..6fef3431e2 100644 --- a/conformance/tests/test_check_executed_cases.py +++ b/conformance/tests/test_check_executed_cases.py @@ -546,3 +546,10 @@ def test_out_of_corpus_and_other_harness_records_are_ignored( assert "| `core/alpha.yaml` | 1 | 0 | 1 | 0 | 2 |" in matrix_text assert "| `core/beta.yaml` | 1 | 0 | 0 | 0 | 1 |" in matrix_text assert "agent/builder/builder.yaml" not in matrix_text + + +def test_load_records_returns_empty_when_manifest_dir_is_file(tmp_path: Path) -> None: + manifest_file = tmp_path / "not_a_dir.jsonl" + manifest_file.write_text("{}", encoding="utf-8") + assert load_records(manifest_file, "python-core") == [] + From a84ef3d8a144d7d99f6dd8cf5fd370bf768e1166 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:36:16 +0000 Subject: [PATCH 06/48] test(conformance): reject unmatched --only globs and omit excluded suites from matrix --- conformance/scripts/check_executed_cases.py | 28 ++++++++++++++----- .../tests/test_check_executed_cases.py | 20 ++++++++++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/conformance/scripts/check_executed_cases.py b/conformance/scripts/check_executed_cases.py index 1900817311..bc02f71684 100644 --- a/conformance/scripts/check_executed_cases.py +++ b/conformance/scripts/check_executed_cases.py @@ -119,6 +119,23 @@ def _iter_pattern_files(conformance_root: Path, patterns: Sequence[str]) -> list return files +def _filter_corpus_by_only( + corpus: dict[str, set[str]], + only: Sequence[str], + corpus_names: Sequence[str], +) -> dict[str, set[str]]: + matched_suites: set[str] = set() + for pat in only: + rx = _glob_to_regex(pat) + hits = {suite for suite in corpus if rx.match(suite) is not None} + if not hits: + raise ValueError( + f"--only pattern {pat!r} matched no suites in {list(corpus_names)}" + ) + matched_suites.update(hits) + return {suite: cases for suite, cases in corpus.items() if suite in matched_suites} + + def load_corpus( conformance_root: Path, corpus_names: Sequence[str], @@ -137,12 +154,7 @@ def load_corpus( corpus[rel_suite] = _extract_suite_cases(suite_path) if not only: return corpus - compiled_only = [_glob_to_regex(pat) for pat in only] - return { - suite: cases - for suite, cases in corpus.items() - if any(rx.match(suite) is not None for rx in compiled_only) - } + return _filter_corpus_by_only(corpus, only, corpus_names) def _is_unreasoned_only(raw: Mapping[str, Any]) -> bool: @@ -454,7 +466,9 @@ def main(argv: Sequence[str] | None = None) -> int: if args.matrix_out is not None: in_corpus_records = [ - rec for rec in records if rec.suite in corpus and rec.error is None + rec + for rec in records + if rec.suite in corpus and rec.suite not in excluded and rec.error is None ] args.matrix_out.parent.mkdir(parents=True, exist_ok=True) args.matrix_out.write_text(render_matrix(in_corpus_records), encoding="utf-8") diff --git a/conformance/tests/test_check_executed_cases.py b/conformance/tests/test_check_executed_cases.py index 6fef3431e2..e3b6ffe14f 100644 --- a/conformance/tests/test_check_executed_cases.py +++ b/conformance/tests/test_check_executed_cases.py @@ -449,10 +449,17 @@ def test_only_glob_narrows_corpus(tmp_path: Path) -> None: assert exit_code == 0 +def test_only_glob_matching_no_suites_raises(tmp_path: Path) -> None: + conf_root, _ = _create_temp_conformance(tmp_path) + with pytest.raises(ValueError, match="--only pattern 'extensions/missing/\\*\\*'"): + load_corpus(conf_root, ["extensions"], only=["extensions/missing/**"]) + + def test_exclude_skips_suite_and_prints_reason( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: conf_root, manifest_dir = _create_temp_conformance(tmp_path) + matrix_file = tmp_path / "matrix.md" _write_jsonl( manifest_dir / "dart-core.jsonl", [ @@ -468,6 +475,13 @@ def test_exclude_skips_suite_and_prints_reason( "case": "case_two", "status": "passed", }, + { + "harness": "dart-core", + "suite": "core/beta.yaml", + "case": "case_three", + "status": "skipped", + "reason": "excluded suite record", + }, ], ) exit_code = main([ @@ -481,10 +495,15 @@ def test_exclude_skips_suite_and_prints_reason( "core", "--exclude", "core/beta.yaml=performance tagged suite", + "--matrix-out", + str(matrix_file), ]) assert exit_code == 0 captured = capsys.readouterr() assert "Excluded core/beta.yaml: performance tagged suite" in captured.out + matrix_text = matrix_file.read_text(encoding="utf-8") + assert "| `core/alpha.yaml` | 2 | 0 | 0 | 0 | 2 |" in matrix_text + assert "core/beta.yaml" not in matrix_text def test_out_of_corpus_and_other_harness_records_are_ignored( @@ -552,4 +571,3 @@ def test_load_records_returns_empty_when_manifest_dir_is_file(tmp_path: Path) -> manifest_file = tmp_path / "not_a_dir.jsonl" manifest_file.write_text("{}", encoding="utf-8") assert load_records(manifest_file, "python-core") == [] - From b4591fff732aa28243d3e2695584a21ce74de9ac Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 18:02:06 +0000 Subject: [PATCH 07/48] test(web_core): record executed conformance cases and enforce strict dispatch --- .github/workflows/web_ci.yml | 5 + .../tests/conformance/conformance_test.mjs | 1157 +++++++++++++---- .../tests/conformance/node-resolution.ts | 297 +++-- 3 files changed, 1105 insertions(+), 354 deletions(-) diff --git a/.github/workflows/web_ci.yml b/.github/workflows/web_ci.yml index d221abc484..e71af26873 100644 --- a/.github/workflows/web_ci.yml +++ b/.github/workflows/web_ci.yml @@ -116,7 +116,12 @@ jobs: - name: Build Renderers run: eval "yarn workspaces foreach -A $RENDERERS_INC -p --topological-dev run build" - name: Test Renderers + env: + A2UI_CONFORMANCE_MANIFEST_DIR: ${{ runner.temp }}/a2ui-manifest run: eval "yarn workspaces foreach -A $RENDERERS_INC -p run test" + - uses: ./.github/actions/setup-python + - name: Check web_core executed cases + run: uv run --project conformance python conformance/scripts/check_executed_cases.py --manifest-dir "${{ runner.temp }}/a2ui-manifest" --harness web_core --corpus core - name: Test React 18 Compatibility run: yarn workspace @a2ui/react run test:react18 - name: Verify NPM Publish Script diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index 4b4804a6ba..716ba7b100 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -12,35 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +import assert from 'node:assert'; import fs from 'node:fs'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; -import assert from 'node:assert'; + import yaml from 'js-yaml'; -import {MessageProcessor, STRICT_VALIDATION} from '../../dist/src/processing/message-processor.js'; +import {z} from 'zod'; + import {Catalog, createFunctionImplementation} from '../../dist/src/catalog/types.js'; -import {PayloadValidator} from '../../dist/src/validation/index.js'; -import {DataModel} from '../../dist/src/state/data-model.js'; -import {SurfaceModel} from '../../dist/src/state/surface-model.js'; -import {SUPPORTED_PROTOCOL_VERSIONS} from '../../dist/src/processing/adapters/base.js'; -import {toCanonicalVersion} from '../../dist/src/common/semver.js'; -import { - BASIC_COMPONENTS as V0_8_BASIC_COMPONENTS, - ThemeSchema as V0_8_ThemeSchema, -} from '../../dist/src/v0_8/basic_catalog/index.js'; -import { - BASIC_COMPONENTS as V0_9_BASIC_COMPONENTS, - BASIC_FUNCTIONS as V0_9_BASIC_FUNCTIONS, - ThemeSchema as V0_9_ThemeSchema, -} from '../../dist/src/v0_9/basic_catalog/index.js'; +import {basicCatalog as V1_0_SDK_BASIC_CATALOG} from '../../dist/src/catalogs/basic/v1/catalog.js'; import { BASIC_COMPONENTS as V1_0_BASIC_COMPONENTS, BASIC_FUNCTIONS as V1_0_BASIC_FUNCTIONS, } from '../../dist/src/catalogs/basic/v1/index.js'; -import {basicCatalog as V1_0_SDK_BASIC_CATALOG} from '../../dist/src/catalogs/basic/v1/catalog.js'; -import {basicCatalog as V0_9_SDK_BASIC_CATALOG} from '../../dist/src/v0_9/basic_catalog/catalog.js'; -import {basicCatalog as V0_8_SDK_BASIC_CATALOG} from '../../dist/src/v0_8/basic_catalog/catalog.js'; -import {ExpressionParser} from '../../dist/src/expressions/expression_parser.js'; +import {toCanonicalVersion} from '../../dist/src/common/semver.js'; import { A2uiCatalogError, A2uiDataError, @@ -51,17 +37,34 @@ import { A2uiStateError, A2uiValidationError, } from '../../dist/src/errors.js'; -import {DataContext} from '../../dist/src/resolution/data-context.js'; -import {NodeResolver} from '../../dist/src/resolution/node-resolver.js'; -import {ResolvedBinding} from '../../dist/src/resolution/resolved-binding.js'; +import {ExpressionParser} from '../../dist/src/expressions/expression_parser.js'; +import {SUPPORTED_PROTOCOL_VERSIONS} from '../../dist/src/processing/adapters/base.js'; +import {MessageProcessor, STRICT_VALIDATION} from '../../dist/src/processing/message-processor.js'; import { - getValue, - peekValue, effect, + getValue, isSignal, - signal, + peekValue, setValue, + signal, } from '../../dist/src/reactivity/signals.js'; +import {DataContext} from '../../dist/src/resolution/data-context.js'; +import {NodeResolver} from '../../dist/src/resolution/node-resolver.js'; +import {ResolvedBinding} from '../../dist/src/resolution/resolved-binding.js'; +import {DataModel} from '../../dist/src/state/data-model.js'; +import {SurfaceModel} from '../../dist/src/state/surface-model.js'; +import {basicCatalog as V0_8_SDK_BASIC_CATALOG} from '../../dist/src/v0_8/basic_catalog/catalog.js'; +import { + BASIC_COMPONENTS as V0_8_BASIC_COMPONENTS, + ThemeSchema as V0_8_ThemeSchema, +} from '../../dist/src/v0_8/basic_catalog/index.js'; +import {basicCatalog as V0_9_SDK_BASIC_CATALOG} from '../../dist/src/v0_9/basic_catalog/catalog.js'; +import { + BASIC_COMPONENTS as V0_9_BASIC_COMPONENTS, + BASIC_FUNCTIONS as V0_9_BASIC_FUNCTIONS, + ThemeSchema as V0_9_ThemeSchema, +} from '../../dist/src/v0_9/basic_catalog/index.js'; +import {PayloadValidator} from '../../dist/src/validation/index.js'; import {runNodeResolutionCase} from '../../dist/tests/conformance/node-resolution.js'; // Dedicated basic catalog component definitions per specification version @@ -191,31 +194,31 @@ class CaseSkipped extends Error {} const REQUIRED_SUITES = new Set(['core/node_resolution.yaml']); /** - * Transition skip list containing specific test suite files to skip during active feature transitions. - * - * 'accessibility.yaml' tests ARIA and DOM accessibility tree rendering, which is handled - * by UI framework renderers (Lit, React, Angular, Flutter, SwiftUI) rather than headless web_core. - * - * 'adapter_conformance.yaml' tests UI framework renderers (Lit, React, Angular, Flutter, SwiftUI) - * rather than headless web_core. - * - * 'builder.yaml' covers the agent-side typesafe builder API, which web_core does not implement. + * Test suite files to skip during headless web_core conformance runs, mapped to + * the reason every case in that suite is recorded as skipped. */ -const SKIP_TEST_SUITES = new Set([ - 'accessibility.yaml', - 'adapter_conformance.yaml', - 'builder.yaml', - 'macros.yaml', +const SKIP_TEST_SUITES = new Map([ + [ + 'accessibility.yaml', + 'accessibility tree rendering is handled by UI framework renderers rather than headless web_core', + ], + [ + 'adapter_conformance.yaml', + 'UI framework renderer adapter conformance is tested in the renderer packages rather than headless web_core', + ], + ['builder.yaml', 'typesafe builder API is agent-side only'], + ['macros.yaml', 'composite macros are agent-side only'], ]); /** - * Action types the web_core runner deliberately does not implement, and why. + * Action types or `.` entries the web_core runner + * deliberately does not implement, and why. * * `web_core` is the renderer-side library. It has no inference-format parser, * payload fixer or prompt generator, so the agent-side suites cannot be - * executed here. They previously reached handlers that asserted nothing and so - * reported as passing; naming them keeps the gap countable and keeps the - * default `throw` for genuinely unrecognised actions intact. + * executed here. Naming them keeps the gap countable and keeps the default + * `throw` for genuinely unrecognised actions intact. Every entry must match at + * least one case during a run. */ const UNIMPLEMENTED_ACTIONS = new Map([ ['from_format', 'the Express inference format is implemented in the agent SDK'], @@ -256,6 +259,78 @@ const UNIMPLEMENTED_ACTIONS = new Map([ ['measure_overhead', 'web_core has no streaming inference parser'], ]); +const UNSUPPORTED = UNIMPLEMENTED_ACTIONS; + +const MANIFEST_DIR = process.env.A2UI_CONFORMANCE_MANIFEST_DIR; +const MANIFEST_FILE = MANIFEST_DIR + ? path.join(MANIFEST_DIR, `web_core-${process.pid}.jsonl`) + : null; +if (MANIFEST_DIR) { + fs.mkdirSync(MANIFEST_DIR, {recursive: true}); +} + +function recordCase(suite, name, status, reason) { + if (!MANIFEST_FILE) return; + const entry = { + harness: 'web_core', + suite: suite.replaceAll('\\', '/'), + case: name, + status, + ...(reason ? {reason} : {}), + }; + fs.appendFileSync(MANIFEST_FILE, `${JSON.stringify(entry)}\n`, 'utf8'); +} + +function assertKnownKeys(obj, allowed, where) { + if (obj === undefined || obj === null) return; + assert.ok(typeof obj === 'object' && !Array.isArray(obj), `${where}: expected object`); + const unknown = Object.keys(obj).filter(key => !allowed.has(key)); + assert.deepStrictEqual( + unknown, + [], + `${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`, + ); +} + +function assertKnownKeysList(items, allowed, where) { + if (!Array.isArray(items)) return; + for (let i = 0; i < items.length; i++) { + assertKnownKeys(items[i], allowed, `${where}[${i}]`); + } +} + +function assertKnownKeysRecord(record, allowed, where) { + if (!record || typeof record !== 'object' || Array.isArray(record)) return; + for (const [key, value] of Object.entries(record)) { + assertKnownKeys(value, allowed, `${where}.${key}`); + } +} + +function hasNestedKeyPath(obj, dottedPath) { + let cur = obj; + for (const segment of dottedPath.split('.')) { + if (!cur || typeof cur !== 'object' || !(segment in cur)) return false; + cur = cur[segment]; + } + return cur !== undefined; +} + +function findUnsupportedEntry(testCase, matchedUnsupported) { + const {action} = testCase; + if (UNSUPPORTED.has(action)) { + matchedUnsupported.add(action); + return {key: action, reason: UNSUPPORTED.get(action)}; + } + const prefix = `${action}.`; + for (const [key, reason] of UNSUPPORTED) { + if (key.startsWith(prefix) && hasNestedKeyPath(testCase, key.slice(prefix.length))) { + matchedUnsupported.add(key); + return {key, reason}; + } + } + return null; +} + function findYamlFiles(dir) { let results = []; if (!fs.existsSync(dir)) return results; @@ -276,242 +351,375 @@ function loadYamlFile(filePath) { return yaml.load(content, {maxDepth: 1000}); } -async function runConformanceHarness() { - console.log('====================================================='); - console.log('A2UI Web Core TypeScript Conformance Test Harness'); - console.log('====================================================='); - - const files = [...findYamlFiles(CORE_DIR), ...findYamlFiles(AGENT_DIR)]; - console.log(`Discovered ${files.length} conformance YAML test suite file(s).`); - - if (files.length === 0) { - console.error('✗ ERROR: No conformance test suite files discovered!'); - process.exit(1); +function recordSkippedSuite(filePath, relativePath, suiteSkipReason) { + const loaded = loadYamlFile(filePath); + const cases = Array.isArray(loaded) ? loaded : (loaded?.tests ?? []); + let count = 0; + for (const tc of cases) { + const caseName = tc?.name || tc?.id; + if (caseName) { + count++; + recordCase(relativePath, caseName, 'skipped', suiteSkipReason); + } } + return count; +} - let totalTests = 0; - let totalPassed = 0; - let totalFailed = 0; - let totalSkipped = 0; - const failures = []; - /** Count of cases skipped per `UNIMPLEMENTED_ACTIONS` entry, for the summary. */ - const unrunByAction = new Map(); - /** `KNOWN_DIVERGENCES` entries that matched a case, as `suite#name`. */ - const matchedDivergences = new Set(); - const discoveredSuites = new Set(files.map(file => path.relative(CONFORMANCE_ROOT, file))); +function checkCaseSkip(relativePath, testCase, state) { + const {name, catalog, args} = testCase; + // The top-level `protocolVersion` is the case's own declaration and wins + // over anything inferred from the catalog or the first message. + const rawVersion = testCase.protocolVersion || catalog?.protocolVersion || args?.version || '0.8'; + const version = toCanonicalVersion(rawVersion) || rawVersion; - for (const suite of REQUIRED_SUITES) { - if (!discoveredSuites.has(suite)) { - totalTests++; - totalFailed++; - const err = 'Required suite was not discovered.'; - console.error(` ✗ FAILED: ${suite}: ${err}`); - failures.push({file: suite, name: 'Required Suite', error: err}); - } + if (!SUPPORTED_PROTOCOL_VERSIONS.has(version)) { + const reason = `version ${rawVersion} not in SUPPORTED_PROTOCOL_VERSIONS`; + console.log(` ⁃ [SKIPPED] ${name} (${reason})`); + return reason; } - for (const filePath of files) { - const relativePath = path.relative(CONFORMANCE_ROOT, filePath); - if (SKIP_TEST_SUITES.has(relativePath) || SKIP_TEST_SUITES.has(path.basename(filePath))) { - continue; - } - let testCases; - try { - testCases = loadYamlFile(filePath); - } catch (err) { - totalTests++; - totalFailed++; - const failMessage = ` ✗ FAILED to load ${relativePath}: ${err.message}`; - console.error(failMessage); - failures.push({file: relativePath, name: 'YAML Parsing', error: err.message}); - continue; - } + if (SKIP_TEST_NAMES.has(name)) { + state.matchedSkippedNames.add(name); + const reason = + SKIP_TEST_NAMES instanceof Map ? SKIP_TEST_NAMES.get(name) : 'skipped via SKIP_TEST_NAMES'; + console.log(` ⁃ [SKIPPED] ${name}`); + return reason; + } - if (!Array.isArray(testCases)) { - totalTests++; - totalFailed++; - const err = `Content is not an array of test cases (got ${typeof testCases}).`; - console.error(` ✗ FAILED to load ${relativePath}: ${err}`); - failures.push({file: relativePath, name: 'Suite Structure', error: err}); - continue; - } + const divergence = KNOWN_DIVERGENCES.get(relativePath)?.get(name); + if (divergence !== undefined) { + state.matchedDivergences.add(`${relativePath}#${name}`); + const reason = `known divergence: ${divergence}`; + console.log(` ⁃ [SKIPPED] ${name} (${reason})`); + return reason; + } - if (REQUIRED_SUITES.has(relativePath) && testCases.length === 0) { - totalTests++; - totalFailed++; - const err = 'Required suite has no test cases.'; - console.error(` ✗ FAILED: ${relativePath}: ${err}`); - failures.push({file: relativePath, name: 'Required Suite', error: err}); - continue; - } + const unsupported = findUnsupportedEntry(testCase, state.matchedUnsupported); + if (unsupported !== null) { + state.unrunByAction.set(unsupported.key, (state.unrunByAction.get(unsupported.key) ?? 0) + 1); + const reason = `action '${unsupported.key}': ${unsupported.reason}`; + console.log(` ⁃ [SKIPPED] ${name} (${reason})`); + return reason; + } - console.log(`\n📄 Suite: ${relativePath} (${testCases.length} test cases)`); + return null; +} - for (const testCase of testCases) { - const {name, action, catalog, args} = testCase; - // The top-level `protocolVersion` is the case's own declaration and wins - // over anything inferred from the catalog or the first message. - const rawVersion = - testCase.protocolVersion || catalog?.protocolVersion || args?.version || '0.8'; - const version = toCanonicalVersion(rawVersion) || rawVersion; +async function dispatchTestCase(testCase) { + const {name, action} = testCase; + if (!name || !action) { + throw new Error('Test case missing required "name" or "action" property.'); + } + switch (action) { + case 'handle_rpc': + await validateRpcTestCase(testCase); + break; + case 'validate': + validateValidateTestCase(testCase); + break; + case 'process_messages': + validateProcessMessagesTestCase(testCase); + break; + case 'get_renderer_capabilities': + validateGetRendererCapabilitiesTestCase(testCase); + break; + case 'validation_schema': + validateValidationSchemaTestCase(testCase); + break; + case 'round_trip': + validateRoundTripTestCase(testCase); + break; + case 'to_json': + validateToJsonTestCase(testCase); + break; + case 'data_model': + validateDataModelTestCase(testCase); + break; + case 'resolve_path': + validateResolvePathTestCase(testCase); + break; + case 'from_json': + validateFromJsonTestCase(testCase); + break; + case 'derive': + validateDeriveTestCase(testCase); + break; + case 'select_catalog': + validateSelectCatalogTestCase(testCase); + break; + case 'parse_expression_template': + validateParseExpressionTemplateTestCase(testCase); + break; + case 'get_renderer_data_model': + validateGetRendererDataModelTestCase(testCase); + break; + case 'resolve_nodes': + await runNodeResolutionCase(testCase, CONFORMANCE_ROOT); + break; + case 'evaluate_function': + validateEvaluateFunctionTestCase(testCase); + break; + case 'dispatch_action': + await validateDispatchActionTestCase(testCase); + break; + default: + throw new Error(`Unhandled action type in conformance harness: '${action}'`); + } +} - if (!SUPPORTED_PROTOCOL_VERSIONS.has(version)) { - totalSkipped++; - console.log( - ` ⁃ [SKIPPED] ${name} (version ${rawVersion} not in SUPPORTED_PROTOCOL_VERSIONS)`, - ); - continue; - } +async function runSuiteCases(relativePath, testCases, state) { + console.log(`\n📄 Suite: ${relativePath} (${testCases.length} test cases)`); - if (SKIP_TEST_NAMES.has(name)) { - totalSkipped++; - console.log(` ⁃ [SKIPPED] ${name}`); - continue; - } + for (const testCase of testCases) { + const {name} = testCase; + const skipReason = checkCaseSkip(relativePath, testCase, state); + if (skipReason !== null) { + state.totalSkipped++; + recordCase(relativePath, name, 'skipped', skipReason); + continue; + } - const divergence = KNOWN_DIVERGENCES.get(relativePath)?.get(name); - if (divergence !== undefined) { - matchedDivergences.add(`${relativePath}#${name}`); - totalSkipped++; - console.log(` ⁃ [SKIPPED] ${name} (known divergence: ${divergence})`); - continue; - } + state.totalTests++; - if (UNIMPLEMENTED_ACTIONS.has(action)) { - totalSkipped++; - unrunByAction.set(action, (unrunByAction.get(action) ?? 0) + 1); - console.log( - ` ⁃ [SKIPPED] ${name} (action '${action}': ${UNIMPLEMENTED_ACTIONS.get(action)})`, - ); + try { + await dispatchTestCase(testCase); + state.totalPassed++; + recordCase(relativePath, name, 'passed'); + console.log(` ✓ PASSED: ${name}`); + } catch (err) { + if (err instanceof CaseSkipped) { + state.totalTests--; + state.totalSkipped++; + recordCase(relativePath, name, 'skipped', err.message); + console.log(` ⁃ [SKIPPED] ${name} (${err.message})`); continue; } + state.totalFailed++; + recordCase(relativePath, name, 'failed', err.message); + const failMessage = ` ✗ FAILED: ${name} - ${err.message}`; + console.error(failMessage); + state.failures.push({file: relativePath, name, error: err.message}); + } + } +} - totalTests++; - - try { - if (!name || !action) { - throw new Error('Test case missing required "name" or "action" property.'); - } - - // Action-specific test execution dispatch - switch (action) { - case 'handle_rpc': - await validateRpcTestCase(testCase); - break; - case 'validate': - validateValidateTestCase(testCase); - break; - case 'process_messages': - validateProcessMessagesTestCase(testCase); - break; - case 'get_renderer_capabilities': - validateGetRendererCapabilitiesTestCase(testCase); - break; - case 'validation_schema': - validateValidationSchemaTestCase(testCase); - break; - case 'round_trip': - validateRoundTripTestCase(testCase); - break; - case 'to_json': - validateToJsonTestCase(testCase); - break; - case 'data_model': - validateDataModelTestCase(testCase); - break; - case 'resolve_path': - validateResolvePathTestCase(testCase); - break; - case 'from_json': - validateFromJsonTestCase(testCase); - break; - case 'derive': - validateDeriveTestCase(testCase); - break; - case 'select_catalog': - validateSelectCatalogTestCase(testCase); - break; - case 'accessibility_check': - validateAccessibilityCheckTestCase(testCase); - break; - case 'parse_expression_template': - validateParseExpressionTemplateTestCase(testCase); - break; - case 'get_renderer_data_model': - validateGetRendererDataModelTestCase(testCase); - break; - case 'resolve_nodes': - await runNodeResolutionCase(testCase, CONFORMANCE_ROOT); - break; - case 'evaluate_function': - validateEvaluateFunctionTestCase(testCase); - break; - case 'dispatch_action': - await validateDispatchActionTestCase(testCase); - break; - default: - throw new Error(`Unhandled action type in conformance harness: '${action}'`); - } +function recordInvariantFailure(state, file, name, error) { + state.totalTests++; + state.totalFailed++; + console.error(` ✗ FAILED: ${name}: ${error}`); + state.failures.push({file, name, error}); +} - totalPassed++; - console.log(` ✓ PASSED: ${name}`); - } catch (err) { - if (err instanceof CaseSkipped) { - totalTests--; - totalSkipped++; - console.log(` ⁃ [SKIPPED] ${name} (${err.message})`); - continue; - } - totalFailed++; - const failMessage = ` ✗ FAILED: ${name} - ${err.message}`; - console.error(failMessage); - failures.push({file: relativePath, name, error: err.message}); - } +function verifyHarnessInvariants(discoveredSuites, state) { + for (const suite of REQUIRED_SUITES) { + if (!discoveredSuites.has(suite)) { + recordInvariantFailure(state, suite, 'Required Suite', 'Required suite was not discovered.'); } } - for (const [suite, cases] of KNOWN_DIVERGENCES) { for (const name of cases.keys()) { - if (!matchedDivergences.has(`${suite}#${name}`)) { - totalTests++; - totalFailed++; - const err = 'Known divergence matches no test case.'; - console.error(` ✗ FAILED: ${name}: ${err}`); - failures.push({file: suite, name, error: err}); + if (!state.matchedDivergences.has(`${suite}#${name}`)) { + recordInvariantFailure(state, suite, name, 'Known divergence matches no test case.'); } } } + for (const key of UNSUPPORTED.keys()) { + if (!state.matchedUnsupported.has(key)) { + recordInvariantFailure(state, 'UNSUPPORTED', key, 'Unsupported entry matches no test case.'); + } + } + for (const suite of SKIP_TEST_SUITES.keys()) { + if (!state.matchedSkippedSuites.has(suite)) { + recordInvariantFailure( + state, + suite, + 'SKIP_TEST_SUITES', + 'Skipped suite entry matches no suite file.', + ); + } + } +} +function loadSuiteTestCases(filePath, relativePath, state) { + const baseName = path.basename(filePath); + const suiteSkipReason = SKIP_TEST_SUITES.get(relativePath) ?? SKIP_TEST_SUITES.get(baseName); + if (suiteSkipReason !== undefined) { + state.matchedSkippedSuites.add(SKIP_TEST_SUITES.has(relativePath) ? relativePath : baseName); + state.totalSkipped += recordSkippedSuite(filePath, relativePath, suiteSkipReason); + return null; + } + let testCases; + try { + testCases = loadYamlFile(filePath); + } catch (err) { + recordInvariantFailure(state, relativePath, 'YAML Parsing', err.message); + return null; + } + if (!Array.isArray(testCases)) { + recordInvariantFailure( + state, + relativePath, + 'Suite Structure', + `Content is not an array of test cases (got ${typeof testCases}).`, + ); + return null; + } + if (REQUIRED_SUITES.has(relativePath) && testCases.length === 0) { + recordInvariantFailure( + state, + relativePath, + 'Required Suite', + 'Required suite has no test cases.', + ); + return null; + } + return testCases; +} + +function finishHarnessRun(state) { console.log('\n====================================================='); console.log( - `Conformance Summary: ${totalPassed}/${totalTests} Passed (${totalFailed} Failed, ${totalSkipped} Skipped)`, + `Conformance Summary: ${state.totalPassed}/${state.totalTests} Passed (${state.totalFailed} Failed, ${state.totalSkipped} Skipped)`, ); - if (unrunByAction.size > 0) { - const unrunTotal = [...unrunByAction.values()].reduce((a, b) => a + b, 0); + if (state.unrunByAction.size > 0) { + const unrunTotal = [...state.unrunByAction.values()].reduce((a, b) => a + b, 0); console.log(`Not run by design: ${unrunTotal} cases`); - for (const [action, count] of [...unrunByAction].sort((a, b) => b[1] - a[1])) { - console.log(` ${action}: ${count} (${UNIMPLEMENTED_ACTIONS.get(action)})`); + for (const [action, count] of [...state.unrunByAction].sort((a, b) => b[1] - a[1])) { + console.log(` ${action}: ${count} (${UNSUPPORTED.get(action)})`); } } console.log('====================================================='); - if (totalFailed > 0) { + if (state.totalFailed > 0) { console.error('\nFailures Summary:'); - for (const failure of failures) { + for (const failure of state.failures) { console.error(`- [${failure.file}] ${failure.name}: ${failure.error}`); } process.exit(1); - } else { - console.log('🎉 All Web Core conformance test vectors validated successfully!'); - process.exit(0); } + console.log('🎉 All Web Core conformance test vectors validated successfully!'); + process.exit(0); } +async function runConformanceHarness() { + console.log('====================================================='); + console.log('A2UI Web Core TypeScript Conformance Test Harness'); + console.log('====================================================='); + + const files = [...findYamlFiles(CORE_DIR), ...findYamlFiles(AGENT_DIR)]; + console.log(`Discovered ${files.length} conformance YAML test suite file(s).`); + + if (files.length === 0) { + console.error('✗ ERROR: No conformance test suite files discovered!'); + process.exit(1); + } + + const state = { + totalTests: 0, + totalPassed: 0, + totalFailed: 0, + totalSkipped: 0, + failures: [], + unrunByAction: new Map(), + matchedDivergences: new Set(), + matchedUnsupported: new Set(), + matchedSkippedSuites: new Set(), + matchedSkippedNames: new Set(), + }; + const discoveredSuites = new Set(files.map(file => path.relative(CONFORMANCE_ROOT, file))); + + for (const filePath of files) { + const relativePath = path.relative(CONFORMANCE_ROOT, filePath); + const testCases = loadSuiteTestCases(filePath, relativePath, state); + if (testCases !== null) { + await runSuiteCases(relativePath, testCases, state); + } + } + + verifyHarnessInvariants(discoveredSuites, state); + finishHarnessRun(state); +} + +const EXPECT_ERROR_KNOWN_KEYS = { + error: new Set([ + 'category', + 'code', + 'details', + 'functionCallId', + 'message', + 'path', + 'rendererError', + 'surfaceId', + ]), + detail: new Set(['code', 'message', 'path', 'surfaceId']), + rendererError: new Set(['error', 'version']), + rendererErrorPayload: new Set(['code', 'functionCallId', 'message', 'path', 'surfaceId']), +}; + +const RPC_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'description', + 'expect', + 'expectError', + 'features', + 'name', + 'protocolVersion', + ]), + args: new Set([ + 'catalogId', + 'catalogVersion', + 'dataModel', + 'functionMetadata', + 'inboundResponse', + 'message', + 'outboundCall', + 'secondOutboundCall', + 'surfaceId', + 'uiActionFallbackScenario', + 'uiBindingScenario', + 'userActivationPresent', + ]), + uiBindingScenario: new Set(['catalogId', 'component', 'inboundResponseValue', 'surfaceId']), + uiActionFallbackScenario: new Set([ + 'actionPayload', + 'catalogId', + 'inboundResponseValue', + 'surfaceId', + ]), + expect: new Set([ + 'correlatedCallId', + 'error', + 'initialValue', + 'outboundCall', + 'resolvedValue', + 'response', + 'result', + ]), +}; + async function validateRpcTestCase(testCase) { + assertKnownKeys(testCase, RPC_KNOWN_KEYS.case, 'handle_rpc'); const {args, expect, expectError} = testCase; if (!args) throw new Error('handle_rpc test requires "args" object.'); if (!expect && !expectError) throw new Error('handle_rpc test requires "expect" or "expectError" object.'); + assertKnownKeys(args, RPC_KNOWN_KEYS.args, 'handle_rpc.args'); + assertKnownKeys( + args.uiBindingScenario, + RPC_KNOWN_KEYS.uiBindingScenario, + 'handle_rpc.args.uiBindingScenario', + ); + assertKnownKeys( + args.uiActionFallbackScenario, + RPC_KNOWN_KEYS.uiActionFallbackScenario, + 'handle_rpc.args.uiActionFallbackScenario', + ); + assertKnownKeys(expect, RPC_KNOWN_KEYS.expect, 'handle_rpc.expect'); + assertKnownKeys(expect?.error, EXPECT_ERROR_KNOWN_KEYS.error, 'handle_rpc.expect.error'); + assertKnownKeys(expectError, EXPECT_ERROR_KNOWN_KEYS.error, 'handle_rpc.expectError'); const message = args.message; const outboundCall = args.outboundCall; @@ -909,12 +1117,43 @@ async function validateRpcTestCase(testCase) { } } +const SELECT_CATALOG_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'description', + 'expect', + 'expectError', + 'expectSelected', + 'features', + 'name', + 'protocolVersion', + ]), + args: new Set([ + 'acceptsInlineCatalogs', + 'catalogs', + 'clientCapabilities', + 'components', + 'functionCall', + 'supportedCatalogs', + 'surface', + ]), + surface: new Set(['defaultCatalogId', 'id', 'supportedCatalogIds']), + expect: new Set(['catalogId', 'components']), +}; + function validateSelectCatalogTestCase(testCase) { + assertKnownKeys(testCase, SELECT_CATALOG_KNOWN_KEYS.case, 'select_catalog'); const {args, expect, expectSelected, expectError} = testCase; if (!args) throw new Error('select_catalog test requires "args" object.'); if (!expect && !expectSelected && !expectError) { throw new Error('select_catalog test requires "expect", "expectSelected", or "expectError".'); } + assertKnownKeys(args, SELECT_CATALOG_KNOWN_KEYS.args, 'select_catalog.args'); + assertKnownKeys(args.surface, SELECT_CATALOG_KNOWN_KEYS.surface, 'select_catalog.args.surface'); + assertKnownKeys(expect, SELECT_CATALOG_KNOWN_KEYS.expect, 'select_catalog.expect'); + assertKnownKeys(expectError, EXPECT_ERROR_KNOWN_KEYS.error, 'select_catalog.expectError'); // Handle agent format catalog selection (supportedCatalogs + clientCapabilities) if (args.supportedCatalogs) { @@ -1123,6 +1362,18 @@ function matchesErrorCategory(err, category) { function assertExpectedConformanceError(thrown, emittedErrorMsg, expErrObj) { if (!expErrObj || typeof expErrObj !== 'object') return; + assertKnownKeys(expErrObj, EXPECT_ERROR_KNOWN_KEYS.error, 'expectError'); + assertKnownKeysList(expErrObj.details, EXPECT_ERROR_KNOWN_KEYS.detail, 'expectError.details'); + assertKnownKeys( + expErrObj.rendererError, + EXPECT_ERROR_KNOWN_KEYS.rendererError, + 'expectError.rendererError', + ); + assertKnownKeys( + expErrObj.rendererError?.error, + EXPECT_ERROR_KNOWN_KEYS.rendererErrorPayload, + 'expectError.rendererError.error', + ); if (expErrObj.category) { if (!matchesErrorCategory(thrown, expErrObj.category)) { throw new Error( @@ -1336,7 +1587,33 @@ function assertExpectedConformanceError(thrown, emittedErrorMsg, expErrObj) { } } +const VALIDATE_KNOWN_KEYS = { + case: new Set([ + 'action', + 'catalog', + 'catalogPaths', + 'catalogs', + 'description', + 'expect', + 'expectError', + 'expectValid', + 'expect_error', + 'features', + 'messages', + 'name', + 'payload', + 'protocolVersion', + 'steps', + 'strictMode', + ]), + step: new Set(['expect', 'expectError', 'expect_error', 'messages', 'payload']), + expect: new Set(['surfaces']), +}; + function validateValidateTestCase(testCase) { + assertKnownKeys(testCase, VALIDATE_KNOWN_KEYS.case, 'validate'); + assertKnownKeysList(testCase.steps, VALIDATE_KNOWN_KEYS.step, 'validate.steps'); + assertKnownKeys(testCase.expect, VALIDATE_KNOWN_KEYS.expect, 'validate.expect'); const expectError = testCase.expectError || testCase.expect_error; const {steps, payload, messages, expect, expectValid} = testCase; if (!steps && !payload && !messages) { @@ -1414,6 +1691,26 @@ function validateValidateTestCase(testCase) { } } +const GET_RENDERER_DATA_MODEL_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'catalogPaths', + 'catalogs', + 'description', + 'expect', + 'features', + 'messages', + 'name', + 'protocolVersion', + 'steps', + ]), + args: new Set(['version']), + step: new Set(['messages']), + expect: new Set(['surfaces', 'version']), +}; + /** * Runs a `get_renderer_data_model` case and compares the emitted payload. * @@ -1427,6 +1724,22 @@ function validateValidateTestCase(testCase) { * `expect` payload. */ function validateGetRendererDataModelTestCase(testCase) { + assertKnownKeys(testCase, GET_RENDERER_DATA_MODEL_KNOWN_KEYS.case, 'get_renderer_data_model'); + assertKnownKeys( + testCase.args, + GET_RENDERER_DATA_MODEL_KNOWN_KEYS.args, + 'get_renderer_data_model.args', + ); + assertKnownKeysList( + testCase.steps, + GET_RENDERER_DATA_MODEL_KNOWN_KEYS.step, + 'get_renderer_data_model.steps', + ); + assertKnownKeys( + testCase.expect, + GET_RENDERER_DATA_MODEL_KNOWN_KEYS.expect, + 'get_renderer_data_model.expect', + ); const {messages, steps, args, expect} = testCase; const processor = new MessageProcessor(getCatalogsForTestCase(testCase), undefined, { version: resolveProtocolVersion(testCase), @@ -1453,11 +1766,6 @@ function validateGetRendererDataModelTestCase(testCase) { ); } -function validateAccessibilityCheckTestCase() { - // Accessibility tree rendering is handled by UI framework renderers (Lit, React, Angular, Flutter, SwiftUI), - // not headless web_core state engines. -} - function collectRefs(node, refs = []) { if (Array.isArray(node)) { for (const item of node) collectRefs(item, refs); @@ -1546,7 +1854,28 @@ function assertMatchesBuiltinCatalog(loaded) { ); } +const FROM_JSON_KNOWN_KEYS = { + case: new Set([ + 'action', + 'catalog', + 'catalogId', + 'catalogPath', + 'catalogSchema', + 'description', + 'expect', + 'expectError', + 'features', + 'name', + 'protocolVersion', + 'schema', + ]), + expect: FROM_JSON_EXPECT_KEYS, +}; + function validateFromJsonTestCase(testCase) { + assertKnownKeys(testCase, FROM_JSON_KNOWN_KEYS.case, 'from_json'); + assertKnownKeys(testCase.expect, FROM_JSON_KNOWN_KEYS.expect, 'from_json.expect'); + assertKnownKeys(testCase.expectError, EXPECT_ERROR_KNOWN_KEYS.error, 'from_json.expectError'); const rawSchema = testCase.catalogPath ? JSON.parse( fs.readFileSync(path.resolve(CONFORMANCE_ROOT, '../', testCase.catalogPath), 'utf8'), @@ -1583,8 +1912,6 @@ function validateFromJsonTestCase(testCase) { if (testCase.expect) { const expected = testCase.expect; - const unknownKeys = Object.keys(expected).filter(key => !FROM_JSON_EXPECT_KEYS.has(key)); - assert.deepStrictEqual(unknownKeys, [], `Unknown from_json expect keys: ${unknownKeys}`); if (expected.catalogId) { assert.strictEqual(catalog.id, expected.catalogId); } @@ -1660,11 +1987,31 @@ const DERIVE_EXPECT_KEYS = new Set([ 'theme', ]); +const DERIVE_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'catalogPath', + 'description', + 'expect', + 'expectError', + 'features', + 'name', + ]), + args: new Set(['catalogId', 'components', 'functions', 'instructions']), + expect: DERIVE_EXPECT_KEYS, +}; + /** * Runs a `derive` case: loads the source catalog with `fromJson`, derives a * catalog from it, and checks the derived catalog and its `toJson()` document. */ function validateDeriveTestCase(testCase) { + assertKnownKeys(testCase, DERIVE_KNOWN_KEYS.case, 'derive'); + assertKnownKeys(testCase.args, DERIVE_KNOWN_KEYS.args, 'derive.args'); + assertKnownKeys(testCase.expect, DERIVE_KNOWN_KEYS.expect, 'derive.expect'); + assertKnownKeys(testCase.expectError, EXPECT_ERROR_KNOWN_KEYS.error, 'derive.expectError'); const source = Catalog.fromJson(readCaseCatalogDocument(testCase)); const {catalogId, components, functions, instructions} = testCase.args; // A map with one key is a new entry, built by loading it into a catalog of @@ -1697,8 +2044,6 @@ function validateDeriveTestCase(testCase) { } const expected = testCase.expect; - const unknownKeys = Object.keys(expected).filter(key => !DERIVE_EXPECT_KEYS.has(key)); - assert.deepStrictEqual(unknownKeys, [], `Unknown derive expect keys: ${unknownKeys}`); const derived = derive(); const document = derived.toJson(); if (expected.catalogId) assert.strictEqual(derived.id, expected.catalogId); @@ -1728,7 +2073,51 @@ function validateDeriveTestCase(testCase) { } } +const DATA_MODEL_KNOWN_KEYS = { + case: new Set([ + 'action', + 'catalog', + 'description', + 'expect', + 'features', + 'initial', + 'name', + 'steps', + 'watch', + ]), + step: { + get: new Set([ + 'expect', + 'expectError', + 'expect_absent', + 'expect_error', + 'expect_type', + 'op', + 'path', + ]), + set: new Set([ + 'expectError', + 'expect_error', + 'expect_notified', + 'expect_values', + 'op', + 'path', + 'value', + ]), + delete: new Set([ + 'expectError', + 'expect_error', + 'expect_notified', + 'expect_values', + 'op', + 'path', + ]), + dispose: new Set(['expectError', 'expect_error', 'op']), + }, +}; + function validateDataModelTestCase(testCase) { + assertKnownKeys(testCase, DATA_MODEL_KNOWN_KEYS.case, 'data_model'); const {initial, watch, steps, expect: topExpect} = testCase; const initialData = initial ? JSON.parse(JSON.stringify(initial)) : {}; const model = new DataModel(initialData); @@ -1758,6 +2147,7 @@ function validateDataModelTestCase(testCase) { const expectErr = step.expect_error || step.expectError; if (expectErr) { + assertKnownKeys(expectErr, EXPECT_ERROR_KNOWN_KEYS.error, 'data_model.step.expectError'); assert.throws( () => { applyDataModelOp(model, step); @@ -1817,6 +2207,7 @@ function applyDataModelOp(model, step) { const {op, path: stepPath, value, expect: stepExpect, expect_absent, expect_type} = step; switch (op) { case 'get': { + assertKnownKeys(step, DATA_MODEL_KNOWN_KEYS.step.get, 'data_model.step.get'); const actual = model.get(stepPath); if (expect_absent === true) { assert.strictEqual( @@ -1843,14 +2234,17 @@ function applyDataModelOp(model, step) { break; } case 'set': { + assertKnownKeys(step, DATA_MODEL_KNOWN_KEYS.step.set, 'data_model.step.set'); model.set(stepPath, value); break; } case 'delete': { + assertKnownKeys(step, DATA_MODEL_KNOWN_KEYS.step.delete, 'data_model.step.delete'); model.set(stepPath, undefined); break; } case 'dispose': { + assertKnownKeys(step, DATA_MODEL_KNOWN_KEYS.step.dispose, 'data_model.step.dispose'); model.dispose(); break; } @@ -1859,9 +2253,31 @@ function applyDataModelOp(model, step) { } } +const RESOLVE_PATH_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'description', + 'expect', + 'expectError', + 'features', + 'name', + 'protocolVersion', + ]), + args: new Set(['contextPath', 'context_path', 'path']), + expect: new Set(['result']), +}; + function validateResolvePathTestCase(testCase) { + assertKnownKeys(testCase, RESOLVE_PATH_KNOWN_KEYS.case, 'resolve_path'); const {args, expect, expectError} = testCase; if (!args) throw new Error('resolve_path test requires "args" object.'); + assertKnownKeys(args, RESOLVE_PATH_KNOWN_KEYS.args, 'resolve_path.args'); + assertKnownKeys(expectError, EXPECT_ERROR_KNOWN_KEYS.error, 'resolve_path.expectError'); + if (expect && typeof expect === 'object') { + assertKnownKeys(expect, RESOLVE_PATH_KNOWN_KEYS.expect, 'resolve_path.expect'); + } const targetPath = args.path || ''; const contextPath = args.contextPath || args.context_path; @@ -1882,7 +2298,35 @@ function validateResolvePathTestCase(testCase) { } } +const GET_RENDERER_CAPABILITIES_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'catalogPaths', + 'catalogs', + 'description', + 'expect', + 'features', + 'name', + 'protocolVersion', + ]), + args: new Set(['componentEnvelopeRef', 'includeInlineCatalogs', 'version', 'versions']), + expect: new Set(['1.0', 'v0.8', 'v0.9', 'v0.9.1']), +}; + function validateGetRendererCapabilitiesTestCase(testCase) { + assertKnownKeys(testCase, GET_RENDERER_CAPABILITIES_KNOWN_KEYS.case, 'get_renderer_capabilities'); + assertKnownKeys( + testCase.args, + GET_RENDERER_CAPABILITIES_KNOWN_KEYS.args, + 'get_renderer_capabilities.args', + ); + assertKnownKeys( + testCase.expect, + GET_RENDERER_CAPABILITIES_KNOWN_KEYS.expect, + 'get_renderer_capabilities.expect', + ); if (!testCase.expect) { throw new Error('get_renderer_capabilities test requires "expect" object.'); } @@ -1928,8 +2372,33 @@ function loadValidationSchemaCatalog(testCase, rawSchema, pVer) { return Catalog.fromJson(rawSchema, pVer, testCase.catalogId); } +const VALIDATION_SCHEMA_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'catalogFile', + 'catalogId', + 'catalogPath', + 'catalogSchema', + 'description', + 'expectError', + 'expectFile', + 'features', + 'name', + 'protocolVersion', + 'schema', + ]), +}; + /** Runs a `validation_schema` case against its golden `expectFile`. */ function validateValidationSchemaTestCase(testCase) { + assertKnownKeys(testCase, VALIDATION_SCHEMA_KNOWN_KEYS.case, 'validation_schema'); + assertKnownKeys( + testCase.expectError, + EXPECT_ERROR_KNOWN_KEYS.error, + 'validation_schema.expectError', + ); const pVer = testCase.protocolVersion || testCase.args?.version || resolveProtocolVersion(testCase); const cPath = testCase.catalogPath || testCase.catalogFile; @@ -2006,7 +2475,22 @@ function assertJsonSetEqual(actual, expected, message) { assert.deepStrictEqual(normalizeSetArrays(actual), normalizeSetArrays(expected), message); } +const ROUND_TRIP_KNOWN_KEYS = { + case: new Set([ + 'action', + 'catalog', + 'catalogFile', + 'catalogId', + 'catalogPath', + 'description', + 'features', + 'name', + 'protocolVersion', + ]), +}; + function validateRoundTripTestCase(testCase) { + assertKnownKeys(testCase, ROUND_TRIP_KNOWN_KEYS.case, 'round_trip'); const document = readCaseCatalogDocument(testCase); const pVer = testCase.protocolVersion; const out = Catalog.fromJson(document, pVer, testCase.catalogId).toJson(); @@ -2015,12 +2499,29 @@ function validateRoundTripTestCase(testCase) { assertJsonSetEqual(again, out, 'toJson() is not idempotent.'); } +const TO_JSON_KNOWN_KEYS = { + case: new Set([ + 'action', + 'builtinCatalog', + 'catalog', + 'catalogFile', + 'catalogId', + 'catalogPath', + 'description', + 'expectFile', + 'features', + 'name', + 'protocolVersion', + ]), +}; + /** * Runs a `to_json` case: serializes web_core's own basic catalog * (`builtinCatalog: basic`) or the catalog loaded with `fromJson`, and compares * the document exactly with `expectFile`. */ function validateToJsonTestCase(testCase) { + assertKnownKeys(testCase, TO_JSON_KNOWN_KEYS.case, 'to_json'); assert.ok(testCase.expectFile, 'to_json case requires expectFile.'); let catalog; if (testCase.builtinCatalog !== undefined) { @@ -2040,8 +2541,6 @@ function validateToJsonTestCase(testCase) { ); } -import {z} from 'zod'; - /** * Fallback component definitions with permissive schemas (`z.object({}).passthrough()`). * @@ -2572,7 +3071,26 @@ function normalizeComponentRefs(val) { return val; } +const SURFACE_EXPECT_KNOWN_KEYS = { + expect: new Set(['surfaces']), + surface: new Set([ + 'components', + 'dataModel', + 'exists', + 'isValid', + 'metadata', + 'sendDataModel', + 'theme', + 'validationErrors', + 'validationResult', + 'validationResults', + ]), + validationResult: new Set(['code', 'message', 'severity', 'valid']), +}; + function assertSurfacesMatch(processor, expect) { + assertKnownKeys(expect, SURFACE_EXPECT_KNOWN_KEYS.expect, 'expect'); + assertKnownKeysRecord(expect?.surfaces, SURFACE_EXPECT_KNOWN_KEYS.surface, 'expect.surfaces'); if (expect && expect.surfaces) { for (const [surfaceId, expectedSurface] of Object.entries(expect.surfaces)) { const surface = processor.getSurface(surfaceId); @@ -2733,6 +3251,11 @@ function assertSurfacesMatch(processor, expect) { } if (expectedSurface.validationResult !== undefined) { const checkSingleResult = (compId, expRes) => { + assertKnownKeys( + expRes, + SURFACE_EXPECT_KNOWN_KEYS.validationResult, + `expect.surfaces.${surfaceId}.validationResult.${compId}`, + ); const firstResult = Array.isArray(getPropsForId(compId).validationResults) ? getPropsForId(compId).validationResults[0] : undefined; @@ -2788,14 +3311,46 @@ function caseValidationConfig(testCase) { if (flags === undefined) { return testCase.strictMode ? {validationConfig: STRICT_VALIDATION} : {}; } - const unknown = Object.keys(flags).filter(name => !CASE_VALIDATION_FLAGS.has(name)); - if (unknown.length > 0) { - throw new Error(`Unknown validationConfig flags: ${unknown.join(', ')}`); - } + assertKnownKeys(flags, CASE_VALIDATION_FLAGS, 'process_messages.validationConfig'); return {validationConfig: {...STRICT_VALIDATION, ...flags}}; } +const PROCESS_MESSAGES_KNOWN_KEYS = { + case: new Set([ + 'action', + 'catalog', + 'catalogPaths', + 'catalogs', + 'description', + 'expect', + 'expectError', + 'expect_error', + 'features', + 'messages', + 'name', + 'payload', + 'protocolVersion', + 'steps', + 'strictMode', + 'validationConfig', + ]), + step: new Set([ + 'expect', + 'expectError', + 'expect_error', + 'message', + 'messages', + 'payload', + 'validationConfig', + ]), + expect: new Set(['surfaces']), + validationConfig: CASE_VALIDATION_FLAGS, +}; + function validateProcessMessagesTestCase(testCase) { + assertKnownKeys(testCase, PROCESS_MESSAGES_KNOWN_KEYS.case, 'process_messages'); + assertKnownKeysList(testCase.steps, PROCESS_MESSAGES_KNOWN_KEYS.step, 'process_messages.steps'); + assertKnownKeys(testCase.expect, PROCESS_MESSAGES_KNOWN_KEYS.expect, 'process_messages.expect'); const {messages, payload, steps, expect, expectError, protocolVersion} = testCase; const testCatalogs = getCatalogsForTestCase(testCase); @@ -2839,6 +3394,11 @@ function validateProcessMessagesTestCase(testCase) { stepMsgs = normalizeMsgs(stepMsgs); // A step's validationConfig applies to that call only. + assertKnownKeys( + step.validationConfig, + PROCESS_MESSAGES_KNOWN_KEYS.validationConfig, + `process_messages.steps[${i}].validationConfig`, + ); const context = step.validationConfig ? {validationConfig: step.validationConfig} : undefined; const stepExpectError = step.expectError || (i === steps.length - 1 ? expectError : undefined); @@ -2907,9 +3467,29 @@ function joinLiterals(parts) { return joined.filter(part => part !== ''); } +const PARSE_EXPRESSION_TEMPLATE_KNOWN_KEYS = { + case: new Set([ + 'action', + 'description', + 'expect', + 'expectError', + 'expect_error', + 'features', + 'input', + 'name', + 'protocolVersion', + ]), +}; + function validateParseExpressionTemplateTestCase(testCase) { + assertKnownKeys(testCase, PARSE_EXPRESSION_TEMPLATE_KNOWN_KEYS.case, 'parse_expression_template'); const {input, expect, expectError, expect_error} = testCase; const errorSpec = expect_error || expectError; + assertKnownKeys( + errorSpec, + EXPECT_ERROR_KNOWN_KEYS.error, + 'parse_expression_template.expectError', + ); const parser = new ExpressionParser(); if (errorSpec) { @@ -2952,7 +3532,53 @@ function getBasicCatalog(version) { return v0_9BasicCatalog; } +const DISPATCH_ACTION_KNOWN_KEYS = { + case: new Set([ + 'action', + 'actionPayload', + 'catalog', + 'catalogPaths', + 'catalogs', + 'dataModel', + 'description', + 'expectDataModel', + 'expectDispatched', + 'expectError', + 'expect_error', + 'features', + 'metadata', + 'name', + 'protocolVersion', + 'scope', + 'sourceComponentId', + 'surfaceId', + 'useBasicCatalog', + 'userActivation', + ]), + actionPayload: new Set(['context', 'event', 'functionCall', 'name']), + expectDispatched: new Set([ + 'context', + 'metadata', + 'name', + 'sourceComponentId', + 'surfaceId', + 'timestampIso8601', + 'userMessage', + ]), +}; + async function validateDispatchActionTestCase(testCase) { + assertKnownKeys(testCase, DISPATCH_ACTION_KNOWN_KEYS.case, 'dispatch_action'); + assertKnownKeys( + testCase.actionPayload, + DISPATCH_ACTION_KNOWN_KEYS.actionPayload, + 'dispatch_action.actionPayload', + ); + assertKnownKeys( + testCase.expectDispatched, + DISPATCH_ACTION_KNOWN_KEYS.expectDispatched, + 'dispatch_action.expectDispatched', + ); const { actionPayload, metadata, @@ -2967,6 +3593,7 @@ async function validateDispatchActionTestCase(testCase) { expect_error, } = testCase; const errorSpec = expect_error || expectError; + assertKnownKeys(errorSpec, EXPECT_ERROR_KNOWN_KEYS.error, 'dispatch_action.expectError'); const testCatalogs = getCatalogsForTestCase(testCase); const baseCat = testCatalogs[0] || getBasicCatalog(resolveProtocolVersion(testCase) || 'v0.9'); @@ -3175,7 +3802,36 @@ function matchesEvaluateFunctionError(err, errorSpec) { return true; } +const EVALUATE_FUNCTION_KNOWN_KEYS = { + case: new Set([ + 'action', + 'args', + 'catalog', + 'catalogId', + 'catalogPaths', + 'catalogs', + 'dataModel', + 'description', + 'expect', + 'expectError', + 'expect_error', + 'features', + 'function', + 'name', + 'protocolVersion', + 'surface', + 'useBasicCatalog', + ]), + surface: new Set(['catalogId']), +}; + function validateEvaluateFunctionTestCase(testCase) { + assertKnownKeys(testCase, EVALUATE_FUNCTION_KNOWN_KEYS.case, 'evaluate_function'); + assertKnownKeys( + testCase.surface, + EVALUATE_FUNCTION_KNOWN_KEYS.surface, + 'evaluate_function.surface', + ); const { function: funcName, args = {}, @@ -3185,6 +3841,7 @@ function validateEvaluateFunctionTestCase(testCase) { expect_error, } = testCase; const errorSpec = expect_error || expectError; + assertKnownKeys(errorSpec, EXPECT_ERROR_KNOWN_KEYS.error, 'evaluate_function.expectError'); if (testCase.surface) { const evaluate = () => evaluateFunctionOnSurface(testCase); diff --git a/typescript/core/web_core/tests/conformance/node-resolution.ts b/typescript/core/web_core/tests/conformance/node-resolution.ts index 1d175bc0e9..d5f23100a1 100644 --- a/typescript/core/web_core/tests/conformance/node-resolution.ts +++ b/typescript/core/web_core/tests/conformance/node-resolution.ts @@ -17,20 +17,22 @@ import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; import {join} from 'node:path'; + import yaml from 'js-yaml'; + import {Catalog, ComponentApi, createFunctionImplementation} from '../../src/catalog/types.js'; import {A2uiMetadata, ActionDispatchOptions, jsonValueEquals} from '../../src/common/metadata.js'; import {MessageProcessor} from '../../src/processing/message-processor.js'; +import {batchWrite, effect, getValue, peekValue} from '../../src/reactivity/signals.js'; import { ComponentNode, - PLACEHOLDER_TYPE, isComponentNode, + PLACEHOLDER_TYPE, } from '../../src/resolution/component-node.js'; import {NodeResolver} from '../../src/resolution/node-resolver.js'; -import {ResolvedBinding, isWritable} from '../../src/resolution/resolved-binding.js'; -import {batchWrite, effect, getValue, peekValue} from '../../src/reactivity/signals.js'; +import {isWritable, ResolvedBinding} from '../../src/resolution/resolved-binding.js'; import {ComponentModel} from '../../src/state/component-model.js'; -import {SurfaceModel, A2uiErrorPayload} from '../../src/state/surface-model.js'; +import {A2uiErrorPayload, SurfaceModel} from '../../src/state/surface-model.js'; type Component = Record & {id: string; component: string}; interface Fixture { @@ -87,6 +89,64 @@ interface Watch { const flush = () => new Promise(resolve => setTimeout(resolve, 0)); +const KNOWN_KEYS = { + case: new Set(['action', 'description', 'expect', 'features', 'fixture', 'name', 'steps']), + fixture: new Set(['catalog', 'components', 'data']), + expect: new Set([ + 'data', + 'destroyed', + 'emissions', + 'errors', + 'events', + 'functions', + 'metadata_emissions', + 'nodes', + 'replaced_nodes', + 'same_nodes', + ]), + node: new Set([ + 'component_id', + 'data_path', + 'isValid', + 'metadata', + 'props', + 'state', + 'type', + 'validationErrors', + 'validationResults', + ]), + event: new Set(['context', 'metadata', 'name', 'source_component_id']), + function: new Set(['args', 'name']), + error: new Set(['code', 'message', 'surfaceId']), + step: { + set_data: new Set(['expect', 'op', 'path', 'value']), + update_components: new Set(['components', 'expect', 'op']), + remove_component: new Set(['component_id', 'expect', 'op']), + write: new Set(['expect', 'node', 'op', 'property', 'value']), + invoke: new Set(['expect', 'metadata', 'node', 'op', 'property']), + process_messages: new Set(['expect', 'messages', 'op']), + dispose: new Set(['expect', 'op']), + }, +} as const; + +function assertKnownKeys(obj: unknown, allowed: ReadonlySet, where: string): void { + if (obj === undefined || obj === null) return; + assert.ok(typeof obj === 'object' && !Array.isArray(obj), `${where}: expected object`); + const unknown = Object.keys(obj).filter(key => !allowed.has(key)); + assert.deepEqual( + unknown, + [], + `${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`, + ); +} + +function assertKnownKeysList(items: unknown, allowed: ReadonlySet, where: string): void { + if (!Array.isArray(items)) return; + for (let i = 0; i < items.length; i++) { + assertKnownKeys(items[i], allowed, `${where}[${i}]`); + } +} + /** Detached values preserve binding snapshots and name child references by tree position. */ function normalize(value: unknown, path: string): unknown { if (isComponentNode(value)) return {node: path}; @@ -170,6 +230,7 @@ function checkNodes( `${reason}: node paths`, ); for (const [path, fields] of Object.entries(expected.nodes)) { + assertKnownKeys(fields, KNOWN_KEYS.node, `${reason}.expect.nodes.${path}`); const node = requiredNode(nodes, path); const nodeType = node.state === 'cyclic' && node.type === PLACEHOLDER_TYPE && surface @@ -253,6 +314,7 @@ function checkExpectedErrors( assert.equal(actualErrors.length, expectedErrors.length, `${reason}: errors count`); for (let i = 0; i < expectedErrors.length; i++) { const expErr: {code?: string; surfaceId?: string; message?: string} = expectedErrors[i]; + assertKnownKeys(expErr, KNOWN_KEYS.error, `${reason}.expect.errors[${i}]`); const actErr = actualErrors[i]; if (expErr.code !== undefined) { assert.equal(actErr.code, expErr.code, `${reason}: errors[${i}].code`); @@ -280,15 +342,19 @@ async function executeNodeResolutionStep( ): Promise { switch (step.op) { case 'set_data': + assertKnownKeys(step, KNOWN_KEYS.step.set_data, reason); surface.dataModel.set(step.path, step.value); break; case 'update_components': + assertKnownKeys(step, KNOWN_KEYS.step.update_components, reason); updateComponents(surface, catalog, step.components); break; case 'remove_component': + assertKnownKeys(step, KNOWN_KEYS.step.remove_component, reason); surface.componentsModel.removeComponent(step.component_id); break; case 'write': { + assertKnownKeys(step, KNOWN_KEYS.step.write, reason); const binding = peekValue(requiredNode(before, step.node).props)[step.property]; assert.ok( binding instanceof ResolvedBinding && isWritable(binding), @@ -298,6 +364,7 @@ async function executeNodeResolutionStep( break; } case 'invoke': { + assertKnownKeys(step, KNOWN_KEYS.step.invoke, reason); const action = peekValue(requiredNode(before, step.node).props)[step.property]; assert.equal(typeof action, 'function', `${reason}: not an action`); await (action as (options?: ActionDispatchOptions) => unknown)( @@ -306,9 +373,11 @@ async function executeNodeResolutionStep( break; } case 'process_messages': + assertKnownKeys(step, KNOWN_KEYS.step.process_messages, reason); processor.processMessages(step.messages as any); break; case 'dispose': + assertKnownKeys(step, KNOWN_KEYS.step.dispose, reason); resolver.dispose(); break; default: @@ -316,6 +385,120 @@ async function executeNodeResolutionStep( } } +interface ResolutionContext { + resolver: NodeResolver; + surface: SurfaceModel; + watches: Map; + events: unknown[]; + functions: unknown[]; + errors: A2uiErrorPayload[]; +} + +function watchNodes(nodes: Map, watches: Map): void { + for (const [path, node] of nodes) { + const existing = watches.get(node); + if (existing) { + existing.path = path; + continue; + } + const watch: Watch = { + path, + emissions: 0, + metadataEmissions: 0, + destroyed: 0, + stop: () => {}, + }; + const stopProps = effect(() => { + const props = getValue(node.props); + watch.emissions++; + watch.lastProps = normalize(props, watch.path); + }); + const stopMetadata = effect(() => { + getValue(node.metadata); + watch.metadataEmissions++; + }); + const stopDestroyed = node.onDestroyed.subscribe(() => { + watch.destroyed++; + }); + watch.stop = () => { + stopProps(); + stopMetadata(); + stopDestroyed.unsubscribe(); + }; + watches.set(node, watch); + } +} + +function checkExpectation( + expected: Expectation, + before: Map, + ctx: ResolutionContext, + reason: string, +): void { + assertKnownKeys(expected, KNOWN_KEYS.expect, `${reason}.expect`); + assertKnownKeysList(expected.events, KNOWN_KEYS.event, `${reason}.expect.events`); + assertKnownKeysList(expected.functions, KNOWN_KEYS.function, `${reason}.expect.functions`); + const after = mountedNodes(ctx.resolver); + checkNodes(after, expected, reason, ctx.surface); + const surviving = new Set(after.values()); + const emissions: Record = {}; + const metadataEmissions: Record = {}; + const destroyed: Record = {}; + for (const [node, watch] of ctx.watches) { + // Removal may update a binding before teardown; only surviving nodes have + // exact emission counts. Retired nodes remain watched for destruction. + if (watch.emissions && surviving.has(node)) { + emissions[watch.path] = (emissions[watch.path] ?? 0) + watch.emissions; + assert.deepEqual( + watch.lastProps, + normalize(peekValue(node.props), watch.path), + `${reason}: ${watch.path} last emission`, + ); + } + if (watch.metadataEmissions && surviving.has(node)) { + metadataEmissions[watch.path] = + (metadataEmissions[watch.path] ?? 0) + watch.metadataEmissions; + } + if (watch.destroyed) { + destroyed[watch.path] = (destroyed[watch.path] ?? 0) + watch.destroyed; + assert.equal(node.disposed, true, `${reason}: ${watch.path} destruction without disposal`); + } + } + assert.deepEqual(emissions, expected.emissions ?? {}, `${reason}: emissions`); + if (expected.metadata_emissions !== undefined) { + assert.deepEqual( + metadataEmissions, + expected.metadata_emissions, + `${reason}: metadata_emissions`, + ); + } + assert.deepEqual(destroyed, expected.destroyed ?? {}, `${reason}: destroyed`); + for (const path of expected.same_nodes ?? []) { + assert.equal( + requiredNode(after, path), + requiredNode(before, path), + `${reason}: ${path} identity`, + ); + } + for (const path of expected.replaced_nodes ?? []) { + assert.notEqual( + requiredNode(after, path), + requiredNode(before, path), + `${reason}: ${path} replacement`, + ); + } + for (const [path, value] of Object.entries(expected.data ?? {})) { + assert.deepEqual( + normalize(ctx.surface.dataModel.get(path), path), + value, + `${reason}: data ${path}`, + ); + } + assert.deepEqual(ctx.events, expected.events ?? [], `${reason}: events`); + assert.deepEqual(ctx.functions, expected.functions ?? [], `${reason}: functions`); + checkExpectedErrors(ctx.errors, expected.errors, reason); +} + /** * Runs one `resolve_nodes` case; fixture and catalog paths resolve against * `conformanceRoot`. @@ -324,10 +507,12 @@ export async function runNodeResolutionCase( testCase: ConformanceCase, conformanceRoot: string, ): Promise { + assertKnownKeys(testCase, KNOWN_KEYS.case, testCase.name || 'resolve_nodes'); assert.equal(testCase.action, 'resolve_nodes'); const fixture = yaml.load( readFileSync(join(conformanceRoot, testCase.fixture), 'utf8'), ) as Fixture; + assertKnownKeys(fixture, KNOWN_KEYS.fixture, `${testCase.name}.fixture`); const parsed = Catalog.fromJson( JSON.parse(readFileSync(join(conformanceRoot, fixture.catalog), 'utf8')), ); @@ -352,6 +537,7 @@ export async function runNodeResolutionCase( updateComponents(surface, catalog, fixture.components); const resolver = new NodeResolver(surface, catalog); const watches = new Map(); + const ctx: ResolutionContext = {resolver, surface, watches, events, functions, errors}; const stopActions = surface.onAction.subscribe(action => { events.push({ name: action.name, @@ -364,110 +550,13 @@ export async function runNodeResolutionCase( errors.push(err); }); - function watchNodes(nodes: Map): void { - for (const [path, node] of nodes) { - const existing = watches.get(node); - if (existing) { - existing.path = path; - continue; - } - const watch: Watch = { - path, - emissions: 0, - metadataEmissions: 0, - destroyed: 0, - stop: () => {}, - }; - const stopProps = effect(() => { - const props = getValue(node.props); - watch.emissions++; - watch.lastProps = normalize(props, watch.path); - }); - const stopMetadata = effect(() => { - getValue(node.metadata); - watch.metadataEmissions++; - }); - const stopDestroyed = node.onDestroyed.subscribe(() => { - watch.destroyed++; - }); - watch.stop = () => { - stopProps(); - stopMetadata(); - stopDestroyed.unsubscribe(); - }; - watches.set(node, watch); - } - } - - function check(expected: Expectation, before: Map, reason: string): void { - const after = mountedNodes(resolver); - checkNodes(after, expected, reason, surface); - const surviving = new Set(after.values()); - const emissions: Record = {}; - const metadataEmissions: Record = {}; - const destroyed: Record = {}; - for (const [node, watch] of watches) { - // Removal may update a binding before teardown; only surviving nodes have - // exact emission counts. Retired nodes remain watched for destruction. - if (watch.emissions && surviving.has(node)) { - emissions[watch.path] = (emissions[watch.path] ?? 0) + watch.emissions; - assert.deepEqual( - watch.lastProps, - normalize(peekValue(node.props), watch.path), - `${reason}: ${watch.path} last emission`, - ); - } - if (watch.metadataEmissions && surviving.has(node)) { - metadataEmissions[watch.path] = - (metadataEmissions[watch.path] ?? 0) + watch.metadataEmissions; - } - if (watch.destroyed) { - destroyed[watch.path] = (destroyed[watch.path] ?? 0) + watch.destroyed; - assert.equal(node.disposed, true, `${reason}: ${watch.path} destruction without disposal`); - } - } - assert.deepEqual(emissions, expected.emissions ?? {}, `${reason}: emissions`); - if (expected.metadata_emissions !== undefined) { - assert.deepEqual( - metadataEmissions, - expected.metadata_emissions, - `${reason}: metadata_emissions`, - ); - } - assert.deepEqual(destroyed, expected.destroyed ?? {}, `${reason}: destroyed`); - for (const path of expected.same_nodes ?? []) { - assert.equal( - requiredNode(after, path), - requiredNode(before, path), - `${reason}: ${path} identity`, - ); - } - for (const path of expected.replaced_nodes ?? []) { - assert.notEqual( - requiredNode(after, path), - requiredNode(before, path), - `${reason}: ${path} replacement`, - ); - } - for (const [path, value] of Object.entries(expected.data ?? {})) { - assert.deepEqual( - normalize(surface.dataModel.get(path), path), - value, - `${reason}: data ${path}`, - ); - } - assert.deepEqual(events, expected.events ?? [], `${reason}: events`); - assert.deepEqual(functions, expected.functions ?? [], `${reason}: functions`); - checkExpectedErrors(errors, expected.errors, reason); - } - try { await flush(); - check(testCase.expect, new Map(), `${testCase.name} initial`); + checkExpectation(testCase.expect, new Map(), ctx, `${testCase.name} initial`); for (const [index, step] of testCase.steps.entries()) { const reason = `${testCase.name} step ${index} (${step.op})`; const before = mountedNodes(resolver); - watchNodes(before); + watchNodes(before, watches); for (const watch of watches.values()) { watch.emissions = 0; watch.metadataEmissions = 0; @@ -480,7 +569,7 @@ export async function runNodeResolutionCase( const bindings = saveBindings(before); await executeNodeResolutionStep(step, before, surface, catalog, processor, resolver, reason); await flush(); - check(step.expect, before, reason); + checkExpectation(step.expect, before, ctx, reason); for (const [binding, value] of bindings) { assert.deepEqual( normalize(binding.value, ''), From 65ca982be3b1f19835a4ebbae4da368b64b73230 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 17:59:37 +0000 Subject: [PATCH 08/48] test(dart-core): record executed conformance cases and enforce strict dispatch Add suite_registry.dart and suite_registry_test.dart to map every conformance/core/*.yaml suite and action to its test file, record executed cases to A2UI_CONFORMANCE_MANIFEST_DIR, enforce known-key checks across all Dart core conformance harnesses, default matchesErrorFields onUnassertedField to fail, and verify the manifest in Flutter CI. --- .github/workflows/flutter_packages_test.yml | 13 + .../conformance/actions_conformance_test.dart | 46 ++- .../catalog_schema_conformance_test.dart | 82 ++++- .../test/conformance/conformance_harness.dart | 224 ++++++++++++-- .../conformance/conformance_harness_test.dart | 120 ++++++- .../data_model_conformance_test.dart | 156 +++++++--- .../expressions_conformance_test.dart | 17 + .../functions_conformance_test.dart | 131 +++++--- .../message_processor_conformance_test.dart | 292 +++++++++++++----- ...ge_processor_scaling_conformance_test.dart | 31 +- .../multi_catalog_conformance_test.dart | 66 +++- .../node_resolution_conformance_test.dart | 145 ++++++--- .../conformance/rpc_conformance_test.dart | 64 +++- .../test/conformance/suite_registry.dart | 116 +++++++ .../test/conformance/suite_registry_test.dart | 104 +++++++ .../validator_conformance_test.dart | 108 ++++++- 16 files changed, 1450 insertions(+), 265 deletions(-) create mode 100644 dart/a2ui_core/test/conformance/suite_registry.dart create mode 100644 dart/a2ui_core/test/conformance/suite_registry_test.dart diff --git a/.github/workflows/flutter_packages_test.yml b/.github/workflows/flutter_packages_test.yml index 3fc580b9d1..ad74816826 100644 --- a/.github/workflows/flutter_packages_test.yml +++ b/.github/workflows/flutter_packages_test.yml @@ -130,12 +130,25 @@ jobs: run: dart run tool/generate_express_parser.dart --check - name: Run tests working-directory: ${{ matrix.package.path }} + env: + A2UI_CONFORMANCE_MANIFEST_DIR: ${{ matrix.package.path == 'dart/a2ui_core' && format('{0}/a2ui-manifest', runner.temp) || '' }} run: | if [ -d "test" ]; then flutter test --test-randomize-ordering-seed=random else echo "No 'test' directory found in ${{ matrix.package.path }}, skipping tests." fi + - name: Set up Python for conformance manifest check + if: matrix.package.path == 'dart/a2ui_core' + uses: ./.github/actions/setup-python + - name: Check executed conformance cases + if: matrix.package.path == 'dart/a2ui_core' + run: | + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --harness dart-core \ + --corpus core \ + --exclude "core/message_processor_scaling.yaml=performance: runs with -P performance" # The e2e_test packages are excluded from the matrix above because their # tests call a live Gemini model, which needs credentials and quota that are diff --git a/dart/a2ui_core/test/conformance/actions_conformance_test.dart b/dart/a2ui_core/test/conformance/actions_conformance_test.dart index 95124802c4..7a618c65c7 100644 --- a/dart/a2ui_core/test/conformance/actions_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/actions_conformance_test.dart @@ -46,6 +46,32 @@ class _TestActionFunction extends FunctionImplementation { } } +const Map> _knownKeys = { + 'case:dispatch_action': { + ...caseMetadataKeys, + 'actionPayload', + 'dataModel', + 'expectDataModel', + 'expectDispatched', + 'expectError', + 'metadata', + 'protocolVersion', + 'scope', + 'sourceComponentId', + 'surfaceId', + 'userActivation', + }, + 'expectDispatched': { + 'context', + 'metadata', + 'name', + 'sourceComponentId', + 'surfaceId', + 'timestampIso8601', + 'userMessage', + }, +}; + /// Runs the shared `conformance/core/actions.yaml` suite against /// [DataContext] and [SurfaceModel]. void main() { @@ -56,11 +82,11 @@ void main() { group('conformance core/actions.yaml', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - test(testCase['name']! as String, () async { - await _runCase(testCase); - }); - } + runConformanceSuite( + cases, + _runCase, + suite: 'core/actions.yaml', + ); }); } @@ -68,6 +94,11 @@ Future _runCase(Map testCase) async { final String action = testCase['action'] as String? ?? 'dispatch_action'; switch (action) { case 'dispatch_action': + assertKnownKeys( + testCase, + _knownKeys['case:dispatch_action']!, + 'case:dispatch_action', + ); await _runDispatchActionCase(testCase); default: throw StateError('Unsupported action: $action'); @@ -94,6 +125,9 @@ Future _runDispatchActionCase(Map testCase) async { final Map? dispatchMetadata = (testCase['metadata'] as Map?)?.cast(); final expectError = testCase['expectError'] as Map?; + if (expectError != null) { + assertKnownKeys(expectError, knownExpectErrorKeys, 'expectError'); + } if (expectError != null && expectError['category'] == 'ValidationError') { await expectLater( () => _executeActionPayload( @@ -228,6 +262,8 @@ void _assertDispatchedAction( final expected = (testCase['expectDispatched'] ?? testCase['expected']) as Map?; if (expected == null) return; + assertKnownKeys( + expected, _knownKeys['expectDispatched']!, 'expectDispatched'); expect(dispatched, isNotNull); expect(dispatched!.name, equals(expected['name'])); if (expected.containsKey('surfaceId')) { diff --git a/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart b/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart index a2182a3a3e..53d3f5c52a 100644 --- a/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart @@ -32,21 +32,72 @@ void main() { group('conformance core/catalog.yaml', () { test('suite has cases', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - final name = testCase['name']! as String; - test( - name, - () => _runCase(testCase), - skip: _knownDivergences[name] ?? - (testCase['protocolVersion'] == 'v0.8' && - testCase['action'] != 'round_trip' - ? _noV08 - : null), - ); - } + runConformanceSuite( + cases, + _runCase, + suite: 'core/catalog.yaml', + unsupported: _knownDivergences, + skipReason: (testCase) => testCase['protocolVersion'] == 'v0.8' && + testCase['action'] != 'round_trip' + ? _noV08 + : null, + ); }); } +const Map> _knownKeys = { + 'case:derive': { + ...caseMetadataKeys, + 'args', + 'catalog', + 'catalogId', + 'catalogPath', + 'expect', + 'expectError', + 'protocolVersion', + }, + 'case:from_json': { + ...caseMetadataKeys, + 'catalog', + 'catalogId', + 'catalogPath', + 'expect', + 'expectError', + 'protocolVersion', + }, + 'case:round_trip': { + ...caseMetadataKeys, + 'catalog', + 'catalogId', + 'catalogPath', + 'protocolVersion', + }, + 'case:to_json': { + ...caseMetadataKeys, + 'builtinCatalog', + 'catalog', + 'catalogId', + 'catalogPath', + 'expectFile', + 'protocolVersion', + }, + 'case:validation_schema': { + ...caseMetadataKeys, + 'catalog', + 'catalogId', + 'catalogPath', + 'expectError', + 'expectFile', + 'protocolVersion', + }, + 'args:derive': { + 'catalogId', + 'components', + 'functions', + 'instructions', + }, +}; + /// Why a case naming protocol version v0.8 is skipped, unless it is a /// `round_trip`, which reads and writes the document without a v0.8 basic /// catalog or v0.8 validation rules. @@ -69,6 +120,10 @@ final Map _builtinCatalogs = { void _runCase(Map testCase) { final action = testCase['action'] as String?; + final Set? allowed = _knownKeys['case:$action']; + if (allowed != null) { + assertKnownKeys(testCase, allowed, 'case:$action'); + } switch (action) { case 'from_json': _runFromJsonCase(testCase); @@ -287,7 +342,7 @@ Map _functionSignatures(CatalogApi catalog) => { /// The protocol version a case names, or null when it names none. /// -/// A case may spell it with or without the `v` prefix. +/// A case may write it with or without the `v` prefix. A2uiProtocolVersion? _version(Map testCase) => switch (testCase['protocolVersion']) { final String version => A2uiProtocolVersion.tryParseSemVer(version), @@ -535,6 +590,7 @@ void _runDeriveCase(Map testCase) { ); final Map args = (testCase['args']! as Map).cast(); + assertKnownKeys(args, _knownKeys['args:derive']!, 'args:derive'); /// The new entries of the kind [section] in [list], keyed by name. Map added(String section, List? list) { diff --git a/dart/a2ui_core/test/conformance/conformance_harness.dart b/dart/a2ui_core/test/conformance/conformance_harness.dart index 4872f44f55..676defbc3c 100644 --- a/dart/a2ui_core/test/conformance/conformance_harness.dart +++ b/dart/a2ui_core/test/conformance/conformance_harness.dart @@ -15,6 +15,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'dart:isolate'; import 'package:a2ui_core/a2ui_core.dart'; import 'package:path/path.dart' as p; @@ -24,6 +25,70 @@ import 'package:yaml/yaml.dart'; /// A single decoded conformance test case map from a YAML suite. typedef ConformanceTestCase = Map; +/// Common metadata keys allowed at the top level of every conformance case. +const Set caseMetadataKeys = { + 'action', + 'description', + 'features', + 'name', +}; + +/// Keys recognized inside an `expectError` block by [matchesErrorFields] and +/// suite error assertions. +const Set knownExpectErrorKeys = { + 'category', + 'code', + 'details', + 'functionCallId', + 'message', + 'path', + 'rendererError', + 'surfaceId', +}; + +/// Fails the current test if [obj] contains any keys not in [allowed]. +void assertKnownKeys( + Map obj, + Set allowed, + String where, +) { + final List unknown = + obj.keys.where((k) => !allowed.contains(k)).toList()..sort(); + if (unknown.isNotEmpty) { + fail('Unhandled keys at $where: $unknown'); + } +} + +/// Appends an executed-case manifest entry when +/// `A2UI_CONFORMANCE_MANIFEST_DIR` (or [manifestDir]) is set. +void recordManifest( + String suite, + String name, + String status, [ + String? reason, + String? manifestDir, +]) { + final String? dir = + manifestDir ?? Platform.environment['A2UI_CONFORMANCE_MANIFEST_DIR']; + if (dir == null || dir.isEmpty) return; + Directory(dir).createSync(recursive: true); + final file = File( + p.join(dir, 'dart-core-$pid-${Isolate.current.hashCode}.jsonl'), + ); + final record = { + 'harness': 'dart-core', + 'suite': suite, + 'case': name, + 'status': status, + if (reason != null && reason.isNotEmpty) 'reason': reason, + }; + file.writeAsStringSync( + '${jsonEncode(record)}\n', + mode: FileMode.append, + flush: true, + ); +} + String resolveConformancePath(String relativePath) { final String root = _conformanceRoot(); final String inConformance = p.normalize(p.join(root, relativePath)); @@ -128,7 +193,8 @@ String? caseVersion(ConformanceTestCase testCase) { } /// Executes a single conformance [testCase] [body], applying [expectedFailures] -/// semantics when the case name is listed. +/// and [unsupported] semantics when the case name is listed, and recording the +/// outcome to the executed-case manifest when [suite] is provided. /// /// A case listed in [expectedFailures] still runs [body]: /// - If [body] throws (either a [TestFailure] or an unhandled error), the @@ -139,16 +205,71 @@ String? caseVersion(ConformanceTestCase testCase) { Future runConformanceCase( ConformanceTestCase testCase, FutureOr Function() body, { + String? suite, Map expectedFailures = const {}, + Map unsupported = const {}, void Function(String message) onSkip = markTestSkipped, + String? manifestDir, }) async { final name = testCase['name']! as String; + final String? unsupportedReason = unsupported[name]; + if (unsupportedReason != null) { + _recordIfSuite(suite, name, 'skipped', unsupportedReason, manifestDir); + onSkip('unsupported: $unsupportedReason'); + return; + } + final String? reason = expectedFailures[name]; if (reason == null) { - await body(); + await _runStandardCase(body, suite, name, manifestDir); return; } + await _runExpectedFailureCase( + body, + suite: suite, + name: name, + reason: reason, + onSkip: onSkip, + manifestDir: manifestDir, + ); +} + +void _recordIfSuite( + String? suite, + String name, + String status, [ + String? reason, + String? manifestDir, +]) { + if (suite != null) { + recordManifest(suite, name, status, reason, manifestDir); + } +} + +Future _runStandardCase( + FutureOr Function() body, + String? suite, + String name, + String? manifestDir, +) async { + try { + await body(); + } catch (error) { + _recordIfSuite(suite, name, 'failed', '$error', manifestDir); + rethrow; + } + _recordIfSuite(suite, name, 'passed', null, manifestDir); +} + +Future _runExpectedFailureCase( + FutureOr Function() body, { + required String? suite, + required String name, + required String reason, + required void Function(String message) onSkip, + required String? manifestDir, +}) async { Object? caughtError; try { await body(); @@ -157,39 +278,86 @@ Future runConformanceCase( } if (caughtError != null) { + _recordIfSuite(suite, name, 'xfailed', reason, manifestDir); onSkip('expected failure: $reason ($caughtError)'); return; } + _recordIfSuite( + suite, + name, + 'failed', + 'Passed unexpectedly while listed in expectedFailures.', + manifestDir, + ); fail( 'Test "$name" passed unexpectedly; remove "$name" from expectedFailures.', ); } /// Registers a `test(...)` for each entry in [cases], wrapping execution in -/// [runConformanceCase] with [expectedFailures]. +/// [runConformanceCase] with [expectedFailures] and [unsupported]. void runConformanceSuite( List cases, FutureOr Function(ConformanceTestCase testCase) runCase, { + String? suite, Map expectedFailures = const {}, + Map unsupported = const {}, String? Function(ConformanceTestCase testCase)? skipReason, void Function(String message) onSkip = markTestSkipped, + String? manifestDir, }) { + _verifyNoUnusedEntries(suite, cases, expectedFailures, unsupported); for (final testCase in cases) { final name = testCase['name']! as String; + final String? skip = skipReason?.call(testCase); test( name, - () => runConformanceCase( - testCase, - () => runCase(testCase), - expectedFailures: expectedFailures, - onSkip: onSkip, - ), - skip: skipReason?.call(testCase), + () { + if (skip != null) { + _recordIfSuite(suite, name, 'skipped', skip, manifestDir); + markTestSkipped(skip); + return null; + } + return runConformanceCase( + testCase, + () => runCase(testCase), + suite: suite, + expectedFailures: expectedFailures, + unsupported: unsupported, + onSkip: onSkip, + manifestDir: manifestDir, + ); + }, ); } } +void _verifyNoUnusedEntries( + String? suite, + List cases, + Map expectedFailures, + Map unsupported, +) { + if (suite == null) return; + final Set caseNames = { + for (final ConformanceTestCase testCase in cases) + testCase['name']! as String, + }; + final List unusedExpected = + expectedFailures.keys.where((k) => !caseNames.contains(k)).toList(); + final List unusedUnsupported = + unsupported.keys.where((k) => !caseNames.contains(k)).toList(); + if (unusedExpected.isNotEmpty || unusedUnsupported.isNotEmpty) { + test('$suite: no unused expectedFailures or unsupported entries', () { + fail( + 'Unused expectedFailures ($unusedExpected) or ' + 'unsupported ($unusedUnsupported) in $suite.', + ); + }); + } +} + /// Reads the error `code` from [error] if it exposes a non-empty string code. String? extractErrorCode(A2uiError error) { final String code = error.code; @@ -248,7 +416,7 @@ Matcher categoryMatches(String? category) => switch (category) { /// - If the thrown [A2uiError] exposes that field, asserts equality with the /// expected value. /// - If the thrown [A2uiError] does not expose the field, invokes -/// [onUnassertedField] (defaulting to [markTestSkipped]) with +/// [onUnassertedField] (defaulting to [fail]) with /// `"code/path not asserted: ..."` rather than silently passing. /// /// When `expectError.rendererError` is set, the matcher requires an outbound @@ -262,22 +430,24 @@ Matcher matchesErrorFields( String defaultVersion = 'v0.9', ErrorMessage? capturedErrorMessage, List? emittedErrors, - void Function(String message) onUnassertedField = markTestSkipped, -}) => - _ErrorFieldsMatcher( - expectedCode: expectedError['code'] as String?, - expectedPath: expectedError['path'] as String?, - expectedSurfaceId: expectedError['surfaceId'] as String?, - expectedFunctionCallId: expectedError['functionCallId'] as String?, - expectedDetails: expectedError['details'] as List?, - expectedRendererError: - expectedError['rendererError'] as Map?, - rawMessage: rawMessage, - defaultVersion: defaultVersion, - capturedErrorMessage: capturedErrorMessage, - emittedErrors: emittedErrors, - onUnassertedField: onUnassertedField, - ); + void Function(String message) onUnassertedField = fail, +}) { + assertKnownKeys(expectedError, knownExpectErrorKeys, 'expectError'); + return _ErrorFieldsMatcher( + expectedCode: expectedError['code'] as String?, + expectedPath: expectedError['path'] as String?, + expectedSurfaceId: expectedError['surfaceId'] as String?, + expectedFunctionCallId: expectedError['functionCallId'] as String?, + expectedDetails: expectedError['details'] as List?, + expectedRendererError: + expectedError['rendererError'] as Map?, + rawMessage: rawMessage, + defaultVersion: defaultVersion, + capturedErrorMessage: capturedErrorMessage, + emittedErrors: emittedErrors, + onUnassertedField: onUnassertedField, + ); +} class _ErrorFieldsMatcher extends Matcher { final String? expectedCode; diff --git a/dart/a2ui_core/test/conformance/conformance_harness_test.dart b/dart/a2ui_core/test/conformance/conformance_harness_test.dart index caf3389ce5..7126b8c2ef 100644 --- a/dart/a2ui_core/test/conformance/conformance_harness_test.dart +++ b/dart/a2ui_core/test/conformance/conformance_harness_test.dart @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'dart:convert'; +import 'dart:io'; + import 'package:a2ui_core/a2ui_core.dart'; import 'package:test/test.dart'; @@ -252,7 +255,30 @@ void main() { ); }); - test('records skip when expectError.path is absent on thrown error', () { + test('fails by default when expectError.path is absent on thrown error', + () { + final errorWithoutPath = A2uiParseError('Missing field'); + + expect( + () => expect( + errorWithoutPath, + matchesErrorFields(const {'path': 'messages.0.version'}), + ), + throwsA( + isA().having( + (e) => e.message, + 'message', + allOf( + contains('code/path not asserted'), + contains('messages.0.version'), + ), + ), + ), + ); + }); + + test('invokes custom onUnassertedField when expectError.path is absent', + () { final skippedMessages = []; final errorWithoutPath = A2uiParseError('Missing field'); @@ -273,4 +299,96 @@ void main() { ); }); }); + + group('assertKnownKeys', () { + test('succeeds when all keys are allowed', () { + assertKnownKeys( + const {'name': 'c1', 'action': 'validate'}, + const {'name', 'action', 'description'}, + 'case', + ); + }); + + test('fails when an unknown key is present', () { + expect( + () => assertKnownKeys( + const {'name': 'c1', 'bogus': true}, + const {'name'}, + 'case', + ), + throwsA( + isA().having( + (e) => e.message, + 'message', + contains('Unhandled keys at case: [bogus]'), + ), + ), + ); + }); + }); + + group('recordManifest', () { + test('writes passed, xfailed, and skipped records to manifestDir', + () async { + final Directory tempDir = Directory.systemTemp.createTempSync( + 'a2ui_manifest_test_', + ); + addTearDown(() => tempDir.deleteSync(recursive: true)); + + await runConformanceCase( + const {'name': 'passing_case'}, + () {}, + suite: 'core/sample.yaml', + manifestDir: tempDir.path, + ); + await runConformanceCase( + const {'name': 'xfailing_case'}, + () => fail('expected gap'), + suite: 'core/sample.yaml', + expectedFailures: const {'xfailing_case': 'known gap'}, + onSkip: (_) {}, + manifestDir: tempDir.path, + ); + await runConformanceCase( + const {'name': 'unsupported_case'}, + () => fail('should not run'), + suite: 'core/sample.yaml', + unsupported: const {'unsupported_case': 'not implemented'}, + onSkip: (_) {}, + manifestDir: tempDir.path, + ); + + final List files = tempDir.listSync().whereType().toList(); + expect(files, hasLength(1)); + final List> records = files.single + .readAsLinesSync() + .map((line) => jsonDecode(line) as Map) + .toList(); + expect( + records, + equals([ + { + 'harness': 'dart-core', + 'suite': 'core/sample.yaml', + 'case': 'passing_case', + 'status': 'passed', + }, + { + 'harness': 'dart-core', + 'suite': 'core/sample.yaml', + 'case': 'xfailing_case', + 'status': 'xfailed', + 'reason': 'known gap', + }, + { + 'harness': 'dart-core', + 'suite': 'core/sample.yaml', + 'case': 'unsupported_case', + 'status': 'skipped', + 'reason': 'not implemented', + }, + ]), + ); + }); + }); } diff --git a/dart/a2ui_core/test/conformance/data_model_conformance_test.dart b/dart/a2ui_core/test/conformance/data_model_conformance_test.dart index 7efca2466c..9c75cd13e5 100644 --- a/dart/a2ui_core/test/conformance/data_model_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/data_model_conformance_test.dart @@ -17,6 +17,55 @@ import 'package:test/test.dart'; import 'conformance_harness.dart'; +const Map> _knownKeys = { + 'case:data_model': { + ...caseMetadataKeys, + 'catalog', + 'initial', + 'steps', + 'watch', + }, + 'case:validate': { + ...caseMetadataKeys, + 'catalog', + 'expect', + 'steps', + }, + 'case:resolve_path': { + ...caseMetadataKeys, + 'args', + 'catalog', + 'expect', + }, + 'args:resolve_path': { + 'contextPath', + 'context_path', + 'path', + }, + 'step:data_model': { + 'expect', + 'expect_absent', + 'expect_error', + 'expect_notified', + 'expect_type', + 'expect_values', + 'op', + 'path', + 'value', + }, + 'step:validate': { + 'description', + 'expectError', + 'payload', + }, + 'expect:validate': { + 'surfaces', + }, + 'surface:validate': { + 'dataModel', + }, +}; + /// Runs the shared `conformance/core/data_model.yaml` suite against /// [DataModel]. /// @@ -40,47 +89,48 @@ void main() { group('conformance core/data_model.yaml', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - test(testCase['name']! as String, () { - _runCase(testCase); - }); - } + runConformanceSuite( + cases, + _runCase, + suite: 'core/data_model.yaml', + ); }); group('conformance core/data_model_pointers.yaml', () { test('suite is not empty', () => expect(pointerCases, isNotEmpty)); - for (final testCase in pointerCases) { - test(testCase['name']! as String, () { - _runPayloadDataModelCase(testCase); - }); - } + runConformanceSuite( + pointerCases, + _runPayloadDataModelCase, + suite: 'core/data_model_pointers.yaml', + ); }); group('conformance core/data_deletion.yaml', () { test('suite is not empty', () => expect(deletionCases, isNotEmpty)); - for (final testCase in deletionCases) { - test(testCase['name']! as String, () { - _runPayloadDataModelCase(testCase); - }); - } + runConformanceSuite( + deletionCases, + _runPayloadDataModelCase, + suite: 'core/data_deletion.yaml', + ); }); group('conformance core/data_context.yaml', () { test('suite is not empty', () => expect(contextCases, isNotEmpty)); - for (final testCase in contextCases) { - test(testCase['name']! as String, () { - _runResolvePathCase(testCase); - }); - } + runConformanceSuite( + contextCases, + _runResolvePathCase, + suite: 'core/data_context.yaml', + ); }); } /// Runs a payload case through [MessageProcessor], over catalogs that accept /// any component, so the data model is built the way a renderer builds it. void _runPayloadDataModelCase(Map testCase) { + assertKnownKeys(testCase, _knownKeys['case:validate']!, 'case:validate'); final steps = testCase['steps']! as List; final List> envelopes = [ for (final Object? step in steps) @@ -112,6 +162,7 @@ void _runPayloadDataModelCase(Map testCase) { for (var i = 0; i < steps.length; i++) { final step = steps[i]! as Map; + assertKnownKeys(step, _knownKeys['step:validate']!, 'step:validate'); final Object? payload = _deepCopy(step['payload']); final expectError = step['expectError'] as Map?; if (expectError != null) { @@ -125,24 +176,47 @@ void _runPayloadDataModelCase(Map testCase) { processor.processMessages(payload); } + _assertPayloadDataModelSurfaces(processor, testCase); +} + +void _assertPayloadDataModelSurfaces( + MessageProcessor processor, + Map testCase, +) { final expected = testCase['expect'] as Map?; - if (expected != null && expected.containsKey('surfaces')) { - final surfaces = expected['surfaces']! as Map; - surfaces.forEach((surfaceId, surfaceExp) { - final expMap = surfaceExp! as Map; - if (expMap.containsKey('dataModel')) { - expect( - processor.model.getSurface(surfaceId)!.dataModel.get('/'), - equals(expMap['dataModel']), - reason: '${testCase['name']} surface $surfaceId dataModel', - ); - } - }); - } + if (expected == null) return; + assertKnownKeys( + expected, + _knownKeys['expect:validate']!, + 'expect:validate', + ); + final surfaces = expected['surfaces'] as Map?; + if (surfaces == null) return; + surfaces.forEach((surfaceId, surfaceExp) { + final expMap = surfaceExp! as Map; + assertKnownKeys( + expMap, + _knownKeys['surface:validate']!, + 'surface:validate', + ); + if (expMap.containsKey('dataModel')) { + expect( + processor.model.getSurface(surfaceId)!.dataModel.get('/'), + equals(expMap['dataModel']), + reason: '${testCase['name']} surface $surfaceId dataModel', + ); + } + }); } void _runResolvePathCase(Map testCase) { + assertKnownKeys( + testCase, + _knownKeys['case:resolve_path']!, + 'case:resolve_path', + ); final args = testCase['args']! as Map; + assertKnownKeys(args, _knownKeys['args:resolve_path']!, 'args:resolve_path'); final String path = (args['path'] as String?) ?? ''; final String contextPath = (args['contextPath'] as String?) ?? (args['context_path'] as String?) ?? @@ -153,6 +227,7 @@ void _runResolvePathCase(Map testCase) { } void _runCase(Map testCase) { + assertKnownKeys(testCase, _knownKeys['case:data_model']!, 'case:data_model'); // The suite is shared across cases, so deep copy before mutating. final model = DataModel( _deepCopy(testCase['initial']) ?? {}, @@ -169,6 +244,7 @@ void _runCase(Map testCase) { final steps = testCase['steps']! as List; for (var i = 0; i < steps.length; i++) { final step = steps[i]! as Map; + assertKnownKeys(step, _knownKeys['step:data_model']!, 'step:data_model'); final reason = '${testCase['name']} step $i (${step['op']})'; for (final observer in watched) { observer.resetCount(); @@ -263,17 +339,13 @@ void _checkWatchedValues( } Matcher _matchesError(Map expectError) { + assertKnownKeys(expectError, knownExpectErrorKeys, 'expectError'); final category = expectError['category'] as String?; final message = expectError['message'] as String?; - Matcher matcher = switch (category) { - 'DataError' => isA(), - 'ValidationError' => isA(), - 'CatalogError' => isA(), - 'IntegrityError' => isA(), - 'RecursionError' => isA(), - 'ParseError' => isA(), - _ => isA(), - }; + Matcher matcher = allOf( + categoryMatches(category), + matchesErrorFields(expectError), + ); if (message != null) { matcher = allOf( matcher, diff --git a/dart/a2ui_core/test/conformance/expressions_conformance_test.dart b/dart/a2ui_core/test/conformance/expressions_conformance_test.dart index 5b56474032..e706d9893d 100644 --- a/dart/a2ui_core/test/conformance/expressions_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/expressions_conformance_test.dart @@ -38,6 +38,16 @@ List _joinLiterals(List parts) { return joined.where((p) => p != '').toList(); } +const Map> _knownKeys = { + 'case:parse_expression_template': { + ...caseMetadataKeys, + 'expect', + 'expectError', + 'expect_error', + 'input', + }, +}; + void main() { // The suite's `validate` cases render basic-catalog components on a // surface, so `message_processor_conformance_test.dart` runs them. @@ -57,6 +67,7 @@ void main() { runConformanceSuite( cases, (testCase) => _runCase(parser, testCase), + suite: 'core/expressions.yaml', ); }); } @@ -70,12 +81,18 @@ void _runCase(ExpressionParser parser, ConformanceTestCase testCase) { 'expressions_conformance_test.dart.', ); } + assertKnownKeys( + testCase, + _knownKeys['case:parse_expression_template']!, + 'case:parse_expression_template', + ); final input = testCase['input']! as String; final Object? expectError = testCase['expect_error'] ?? testCase['expectError']; if (expectError != null) { final error = expectError as Map; + assertKnownKeys(error, knownExpectErrorKeys, 'expectError'); final category = error['category'] as String; final String message = error['message'] as String? ?? ''; final Type expectedType = _categoryToError[category] ?? A2uiExpressionError; diff --git a/dart/a2ui_core/test/conformance/functions_conformance_test.dart b/dart/a2ui_core/test/conformance/functions_conformance_test.dart index e4fd44b2c3..db2fc99ca4 100644 --- a/dart/a2ui_core/test/conformance/functions_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/functions_conformance_test.dart @@ -145,6 +145,26 @@ Object? _evaluateOnSurface(Map testCase) { }); } +const Map> _knownKeys = { + 'case:evaluate_function': { + ...caseMetadataKeys, + 'args', + 'catalogId', + 'catalogPaths', + 'dataModel', + 'expect', + 'expectError', + 'expect_error', + 'function', + 'locale', + 'protocolVersion', + 'surface', + }, + 'surface:evaluate_function': { + 'catalogId', + }, +}; + void main() { // The suite's `validate` cases render basic-catalog components on a // surface, so `message_processor_conformance_test.dart` runs them. @@ -155,50 +175,71 @@ void main() { ]; group('functions conformance', () { - for (final testCase in cases) { - final name = testCase['name']! as String; - final action = testCase['action']! as String; - - test(name, () { - switch (action) { - case 'evaluate_function': - final Object? expectError = - testCase['expectError'] ?? testCase['expect_error']; - if (expectError is Map) { - final category = expectError['category']! as String; - final Type? expectedType = _categoryToError[category]; - if (expectedType == null) { - fail('Unmapped error category $category.'); - } - final message = expectError['message'] as String?; - expect( - () => _evaluate(testCase), - throwsA( - predicate( - (Object? e) { - final bool matchesType = switch (category) { - 'CatalogError' => e is A2uiCatalogError, - 'ExpressionError' => e is A2uiExpressionError, - 'ValidationError' => e is A2uiValidationError, - _ => e.runtimeType == expectedType, - }; - return matchesType && - (message == null || - RegExp(message) - .hasMatch((e! as A2uiError).message)); - }, - 'throws $expectedType' - '${message == null ? '' : " matching '$message'"}', - ), - ), - ); - return; - } - expect(_evaluate(testCase), equals(testCase['expect'])); - default: - fail('Unhandled action $action in functions.yaml.'); - } - }); - } + runConformanceSuite( + cases, + _runFunctionCase, + suite: 'core/functions.yaml', + ); }); } + +void _runFunctionCase(Map testCase) { + final action = testCase['action']! as String; + switch (action) { + case 'evaluate_function': + assertKnownKeys( + testCase, + _knownKeys['case:evaluate_function']!, + 'case:evaluate_function', + ); + if (testCase['surface'] case final Map surfaceBlock) { + assertKnownKeys( + surfaceBlock, + _knownKeys['surface:evaluate_function']!, + 'surface:evaluate_function', + ); + } + final Object? expectError = + testCase['expectError'] ?? testCase['expect_error']; + if (expectError is Map) { + assertKnownKeys(expectError, knownExpectErrorKeys, 'expectError'); + _assertFunctionError(testCase, expectError); + return; + } + expect(_evaluate(testCase), equals(testCase['expect'])); + default: + fail('Unhandled action $action in functions.yaml.'); + } +} + +void _assertFunctionError( + Map testCase, + Map expectError, +) { + final category = expectError['category']! as String; + final Type? expectedType = _categoryToError[category]; + if (expectedType == null) { + fail('Unmapped error category $category.'); + } + final message = expectError['message'] as String?; + expect( + () => _evaluate(testCase), + throwsA( + predicate( + (Object? e) { + final bool matchesType = switch (category) { + 'CatalogError' => e is A2uiCatalogError, + 'ExpressionError' => e is A2uiExpressionError, + 'ValidationError' => e is A2uiValidationError, + _ => e.runtimeType == expectedType, + }; + return matchesType && + (message == null || + RegExp(message).hasMatch((e! as A2uiError).message)); + }, + 'throws $expectedType' + '${message == null ? '' : " matching '$message'"}', + ), + ), + ); +} diff --git a/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart b/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart index a558c6eefd..7e89eb9fa9 100644 --- a/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart @@ -27,10 +27,99 @@ const Map _reservedKeysExpectedFailures = {}; /// reason each is currently failing. const Map _functionsExpectedFailures = {}; +const Map> _knownKeys = { + 'case:process_messages': { + ...caseMetadataKeys, + 'catalog', + 'catalogPaths', + 'catalogs', + 'expect', + 'expectError', + 'messages', + 'payload', + 'protocolVersion', + 'steps', + 'strictMode', + 'validationConfig', + }, + 'case:validate': { + ...caseMetadataKeys, + 'catalog', + 'catalogPaths', + 'catalogs', + 'expect', + 'expectError', + 'messages', + 'payload', + 'protocolVersion', + 'steps', + 'strictMode', + 'validationConfig', + }, + 'case:get_renderer_data_model': { + ...caseMetadataKeys, + 'args', + 'catalog', + 'catalogPaths', + 'catalogs', + 'expect', + 'messages', + 'payload', + 'protocolVersion', + 'steps', + }, + 'case:get_renderer_capabilities': { + ...caseMetadataKeys, + 'args', + 'catalog', + 'catalogPaths', + 'catalogs', + 'expect', + 'protocolVersion', + }, + 'case:resolve_path': { + ...caseMetadataKeys, + 'args', + 'expect', + }, + 'args:get_renderer_data_model': { + 'version', + }, + 'args:get_renderer_capabilities': { + 'includeInlineCatalogs', + 'version', + }, + 'args:resolve_path': { + 'contextPath', + 'path', + }, + 'step': { + 'description', + 'expect', + 'expectError', + 'messages', + 'payload', + 'validationConfig', + }, + 'expect:surfaces': { + 'surfaces', + }, + 'surface': { + 'catalogId', + 'components', + 'dataModel', + 'exists', + 'metadata', + 'rootId', + 'sendDataModel', + 'theme', + }, +}; + /// Runs the shared message-processor suites against [MessageProcessor] and /// [DataContext]: `core/message_processor_v0_9.yaml`, -/// `core/message_processor_v1_0.yaml`, the `process_messages` cases of -/// `core/reserved_keys.yaml` and `core/composition_constraints.yaml`, and the +/// `core/message_processor_v1_0.yaml`, `core/composition_constraints.yaml`, +/// the `process_messages` cases of `core/reserved_keys.yaml`, and the /// `validate` cases of `core/functions.yaml`, `core/expressions.yaml`, /// `core/index_function.yaml`, and `core/multi_catalog.yaml`, which render /// basic-catalog components. @@ -42,10 +131,7 @@ void main() { onlyAction: 'process_messages', expectedFailures: _reservedKeysExpectedFailures, ); - _registerSuite( - 'core/composition_constraints.yaml', - onlyAction: 'process_messages', - ); + _registerSuite('core/composition_constraints.yaml'); _registerSuite( 'core/functions.yaml', onlyAction: 'validate', @@ -68,7 +154,12 @@ void _registerSuite( group('conformance $suite', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - runConformanceSuite(cases, _runCase, expectedFailures: expectedFailures); + runConformanceSuite( + cases, + _runCase, + suite: suite, + expectedFailures: expectedFailures, + ); }); } @@ -77,6 +168,10 @@ String _actionOf(Map testCase) => Future _runCase(Map testCase) async { final String action = _actionOf(testCase); + final Set? allowed = _knownKeys['case:$action']; + if (allowed != null) { + assertKnownKeys(testCase, allowed, 'case:$action'); + } switch (action) { case 'process_messages' || 'validate': await _runProcessMessagesCase(testCase); @@ -201,6 +296,7 @@ Future _runSteps( final name = testCase['name']! as String; for (var index = 0; index < steps.length; index++) { final Map step = steps[index]; + assertKnownKeys(step, _knownKeys['step']!, 'step'); final Object? payload = step['messages'] ?? step['payload']; final Object? expectError = step['expectError'] ?? (index == steps.length - 1 ? testCase['expectError'] : null); @@ -240,6 +336,9 @@ Future _runSteps( resolveNodes: resolveNodes, validationConfig: validationConfig, ); + if (step['expect'] case final Map stepExpect) { + _checkSurfaces(processor, stepExpect, '$name: step $index'); + } } } _checkSurfaces( @@ -271,7 +370,9 @@ Future _runGetRendererDataModelCase( ); if (testCase['steps'] case final List steps) { for (final step in steps) { - await _process(processor, step! as Map); + final stepMap = step! as Map; + assertKnownKeys(stepMap, _knownKeys['step']!, 'step'); + await _process(processor, stepMap); } } else { await _process(processor, testCase); @@ -279,6 +380,11 @@ Future _runGetRendererDataModelCase( final Map args = (testCase['args'] as Map?) ?? const {}; + assertKnownKeys( + args, + _knownKeys['args:get_renderer_data_model']!, + 'args:get_renderer_data_model', + ); final Object? version = args['version']; final Map? actual = processor.getRendererDataModel( version: version is String ? A2uiProtocolVersion.parse(version) : null, @@ -299,6 +405,11 @@ void _runGetRendererCapabilitiesCase(Map testCase) { ); final Map args = (testCase['args'] as Map?) ?? const {}; + assertKnownKeys( + args, + _knownKeys['args:get_renderer_capabilities']!, + 'args:get_renderer_capabilities', + ); final Object? version = args['version']; final Map actual = processor .getRendererCapabilities( @@ -320,6 +431,7 @@ void _runResolvePathCase(Map testCase) { final name = testCase['name']! as String; final Map args = (testCase['args'] as Map?) ?? const {}; + assertKnownKeys(args, _knownKeys['args:resolve_path']!, 'args:resolve_path'); final String path = (args['path'] as String?) ?? ''; final String contextPath = (args['contextPath'] as String?) ?? '/'; final context = DataContext(DataModel(), (_, __, ___) => null, contextPath); @@ -380,16 +492,9 @@ List> _catalogsFor( } return built; } - if (testCase['catalog'] case final Map document - when document.containsKey('components') || - document.containsKey('catalogId')) { - return [ - rendererCatalog( - document, - asCatalogId: _catalogIdOf(testCase), - protocolVersion: version, - ), - ]; + if (_singleCatalogFromCase(testCase, version) + case final Catalog single) { + return [single]; } // A case naming no catalog document, or only a protocol version, runs // against the basic catalog of its version, under whichever id its @@ -407,6 +512,30 @@ List> _catalogsFor( ]; } +Catalog? _singleCatalogFromCase( + Map testCase, + String version, +) { + final Object? raw = testCase['catalog']; + if (raw is! Map) return null; + final Object? nested = raw['catalogSchema'] ?? raw['catalog_schema']; + if (nested is Map) { + return rendererCatalog( + nested, + asCatalogId: _catalogIdOf(testCase), + protocolVersion: (raw['protocolVersion'] as String?) ?? version, + ); + } + if (raw.containsKey('components') || raw.containsKey('catalogId')) { + return rendererCatalog( + raw, + asCatalogId: _catalogIdOf(testCase), + protocolVersion: version, + ); + } + return null; +} + /// The protocol version a case targets: the one it declares, else the one /// its first message declares, else v0.9. String _versionOf(Map testCase) { @@ -484,6 +613,11 @@ void _checkSurfaces( Map expected, String name, ) { + assertKnownKeys( + expected, + _knownKeys['expect:surfaces']!, + 'expect:surfaces', + ); final surfaces = expected['surfaces'] as Map?; if (surfaces == null) return; @@ -492,75 +626,77 @@ void _checkSurfaces( surfaceId, ); final expectations = raw! as Map; + assertKnownKeys(expectations, _knownKeys['surface']!, 'surface'); if (expectations['exists'] == false) { expect(surface, isNull, reason: '$name: surface $surfaceId is closed'); return; } expect(surface, isNotNull, reason: '$name: surface $surfaceId is open'); - - if (expectations.containsKey('catalogId')) { - expect( - surface!.defaultCatalog?.id, - expectations['catalogId'], - reason: '$name: $surfaceId catalogId', - ); - } - if (expectations.containsKey('rootId')) { - expect( - surface!.rootId, - expectations['rootId'], - reason: '$name: $surfaceId rootId', - ); - } - if (expectations.containsKey('metadata')) { - expect( - surface!.metadata, - equals(expectations['metadata']), - reason: '$name: $surfaceId metadata', - ); - } - if (expectations.containsKey('theme')) { - expect( - surface!.theme, - equals(expectations['theme']), - reason: '$name: $surfaceId theme', - ); - } - if (expectations.containsKey('sendDataModel')) { - expect( - surface!.sendDataModel, - expectations['sendDataModel'], - reason: '$name: $surfaceId sendDataModel', - ); - } - if (expectations.containsKey('dataModel')) { - expect( - surface!.dataModel.get('/'), - equals(expectations['dataModel']), - reason: '$name: $surfaceId data model', - ); - } - if (expectations.containsKey('metadata')) { - expect( - surface!.metadata, - equals(expectations['metadata']), - reason: '$name: $surfaceId metadata', - ); - } - if (expectations.containsKey('components')) { - _checkComponents( - surface!, - _normalizeExpectedComponents(expectations['components']), - '$name: $surfaceId', - // A map of expectations names the components it checks; a list - // names every component the surface holds. - exhaustive: expectations['components'] is List, - ); - } + _assertSurfaceExpectations(surface!, surfaceId, expectations, name); }); } +void _assertSurfaceExpectations( + SurfaceModel surface, + String surfaceId, + Map expectations, + String name, +) { + if (expectations.containsKey('catalogId')) { + expect( + surface.defaultCatalog?.id, + expectations['catalogId'], + reason: '$name: $surfaceId catalogId', + ); + } + if (expectations.containsKey('rootId')) { + expect( + surface.rootId, + expectations['rootId'], + reason: '$name: $surfaceId rootId', + ); + } + if (expectations.containsKey('metadata')) { + expect( + surface.metadata, + equals(expectations['metadata']), + reason: '$name: $surfaceId metadata', + ); + } + if (expectations.containsKey('theme')) { + expect( + surface.theme, + equals(expectations['theme']), + reason: '$name: $surfaceId theme', + ); + } + if (expectations.containsKey('sendDataModel')) { + expect( + surface.sendDataModel, + expectations['sendDataModel'], + reason: '$name: $surfaceId sendDataModel', + ); + } + if (expectations.containsKey('dataModel')) { + expect( + surface.dataModel.get('/'), + equals(expectations['dataModel']), + reason: '$name: $surfaceId data model', + ); + } + if (expectations.containsKey('components')) { + _checkComponents( + surface, + _normalizeExpectedComponents(expectations['components']), + '$name: $surfaceId', + // A map of expectations names the components it checks; a list + // names every component the surface holds. + exhaustive: expectations['components'] is List, + ); + } +} + List> _normalizeExpectedComponents(Object? raw) { if (raw is List) { return raw.cast>(); diff --git a/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart b/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart index 99747fe719..dafebd8bdc 100644 --- a/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart @@ -30,19 +30,42 @@ import '../support/renderer_catalog.dart'; import '../support/timing.dart'; import 'conformance_harness.dart'; +const Map> _knownKeys = { + 'case:measure_growth': { + ...caseMetadataKeys, + 'args', + 'catalogPaths', + 'generator', + 'max_ratio', + 'operation', + 'protocolVersion', + 'sizes', + }, + 'args:measure_growth': { + 'validation_config', + }, +}; + void main() { const suite = 'core/message_processor_scaling.yaml'; final List cases = loadConformanceSuite(suite); group('conformance $suite', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - test(testCase['name']! as String, () => _runCase(testCase)); - } + runConformanceSuite( + cases, + _runCase, + suite: suite, + ); }); } void _runCase(ConformanceTestCase testCase) { expect(testCase['action'], 'measure_growth'); + assertKnownKeys( + testCase, + _knownKeys['case:measure_growth']!, + 'case:measure_growth', + ); expect(testCase['operation'], 'process_messages'); final List> catalogs = [ for (final Object? path in testCase['catalogPaths']! as List) @@ -52,6 +75,8 @@ void _runCase(ConformanceTestCase testCase) { ), ]; final args = testCase['args']! as Map; + assertKnownKeys( + args, _knownKeys['args:measure_growth']!, 'args:measure_growth'); final ValidationConfig? config = switch (args['validation_config']) { 'none' => null, 'relaxed' => ValidationConfig.relaxed, diff --git a/dart/a2ui_core/test/conformance/multi_catalog_conformance_test.dart b/dart/a2ui_core/test/conformance/multi_catalog_conformance_test.dart index ec85aa0db7..bd21fbe727 100644 --- a/dart/a2ui_core/test/conformance/multi_catalog_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/multi_catalog_conformance_test.dart @@ -18,6 +18,32 @@ import 'package:test/test.dart'; import 'conformance_harness.dart'; +const Map> _knownKeys = { + 'case:select_catalog': { + ...caseMetadataKeys, + 'args', + 'catalog', + 'expectError', + 'expectSelected', + }, + 'args:select_catalog': { + 'catalogs', + 'components', + 'functionCall', + 'surface', + }, + 'surface:select_catalog': { + 'defaultCatalogId', + 'id', + 'supportedCatalogIds', + }, + 'functionCall:select_catalog': { + 'args', + 'call', + 'catalogId', + }, +}; + /// Runs the shared `conformance/core/multi_catalog.yaml` suite against /// [SurfaceModel.resolveCatalog] and the [DataContext] a component renders /// with. @@ -33,10 +59,11 @@ void main() { group('conformance core/multi_catalog.yaml', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - final name = testCase['name']! as String; - test(name, () => _runCase(testCase)); - } + runConformanceSuite( + cases, + _runCase, + suite: 'core/multi_catalog.yaml', + ); }); } @@ -44,12 +71,43 @@ void _runCase(Map testCase) { final action = testCase['action']! as String; switch (action) { case 'select_catalog': + _assertSelectCatalogKeys(testCase); _runSelectCatalogCase(testCase); default: throw StateError('Unsupported multi_catalog action: $action'); } } +void _assertSelectCatalogKeys(Map testCase) { + assertKnownKeys( + testCase, + _knownKeys['case:select_catalog']!, + 'case:select_catalog', + ); + final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:select_catalog']!, + 'args:select_catalog', + ); + final surfaceArgs = args['surface']! as Map; + assertKnownKeys( + surfaceArgs, + _knownKeys['surface:select_catalog']!, + 'surface:select_catalog', + ); + if (args['functionCall'] case final Map functionCall) { + assertKnownKeys( + functionCall, + _knownKeys['functionCall:select_catalog']!, + 'functionCall:select_catalog', + ); + } + if (testCase['expectError'] case final Map expectError) { + assertKnownKeys(expectError, knownExpectErrorKeys, 'expectError'); + } +} + /// Builds the surface a case describes and reports the catalog id its last /// component or its function call selects. /// diff --git a/dart/a2ui_core/test/conformance/node_resolution_conformance_test.dart b/dart/a2ui_core/test/conformance/node_resolution_conformance_test.dart index 8bd33ce7f2..81ce6b2a5a 100644 --- a/dart/a2ui_core/test/conformance/node_resolution_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/node_resolution_conformance_test.dart @@ -24,51 +24,110 @@ import 'conformance_harness.dart'; typedef _Node = ComponentNode; typedef _Json = Map; +const Map> _knownKeys = { + 'case:resolve_nodes': { + ...caseMetadataKeys, + 'expect', + 'fixture', + 'steps', + }, + 'step:resolve_nodes': { + 'component_id', + 'components', + 'description', + 'expect', + 'messages', + 'metadata', + 'node', + 'op', + 'path', + 'property', + 'value', + }, + 'expect:resolve_nodes': { + 'data', + 'destroyed', + 'emissions', + 'errors', + 'events', + 'functions', + 'metadata_emissions', + 'nodes', + 'replaced_nodes', + 'same_nodes', + }, + 'node:resolve_nodes': { + 'component_id', + 'data_path', + 'isValid', + 'metadata', + 'props', + 'state', + 'type', + 'validationErrors', + 'validationResults', + }, +}; + void main() { final List<_Json> cases = loadConformanceSuite('core/node_resolution.yaml'); group('conformance core/node_resolution.yaml', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - test(testCase['name']! as String, () async { - final fixture = normalizeYaml( - loadYaml( - File( - resolveConformancePath(testCase['fixture']! as String), - ).readAsStringSync(), - ), - )! as _Json; - final runner = _Runner(fixture); - addTearDown(runner.close); - runner.check(testCase['expect']! as _Json, {}, 'initial'); - final steps = testCase['steps']! as List; - for (var i = 0; i < steps.length; i++) { - final step = steps[i]! as _Json; - final Map before = runner.nodes(); - final bindings = Map, Object?>.identity(); - for (final _Node node in before.values) { - _collectBindings(node.props.peek(), bindings); - } - runner.beginStep(before); - await runner.apply(step, before); - runner.check( - step['expect']! as _Json, - before, - 'step $i (${step['op']})', - ); - for (final MapEntry, Object?> entry - in bindings.entries) { - expect( - entry.key.value, - entry.value, - reason: 'step $i: old binding snapshot', - ); - } - } - }); - } + runConformanceSuite( + cases, + _runCase, + suite: 'core/node_resolution.yaml', + ); }); } +Future _runCase(_Json testCase) async { + assertKnownKeys( + testCase, + _knownKeys['case:resolve_nodes']!, + 'case:resolve_nodes', + ); + final fixture = normalizeYaml( + loadYaml( + File( + resolveConformancePath(testCase['fixture']! as String), + ).readAsStringSync(), + ), + )! as _Json; + final runner = _Runner(fixture); + addTearDown(runner.close); + runner.check(testCase['expect']! as _Json, {}, 'initial'); + final steps = testCase['steps']! as List; + for (var i = 0; i < steps.length; i++) { + final step = steps[i]! as _Json; + assertKnownKeys( + step, + _knownKeys['step:resolve_nodes']!, + 'step:resolve_nodes', + ); + final Map before = runner.nodes(); + final bindings = Map, Object?>.identity(); + for (final _Node node in before.values) { + _collectBindings(node.props.peek(), bindings); + } + runner.beginStep(before); + await runner.apply(step, before); + runner.check( + step['expect']! as _Json, + before, + 'step $i (${step['op']})', + ); + for (final MapEntry, Object?> entry + in bindings.entries) { + expect( + entry.key.value, + entry.value, + reason: 'step $i: old binding snapshot', + ); + } + } +} + Object? _copy(Object? value) => jsonDecode(jsonEncode(value)); void _collectBindings( @@ -334,6 +393,11 @@ class _Runner { } void check(_Json expected, Map before, String reason) { + assertKnownKeys( + expected, + _knownKeys['expect:resolve_nodes']!, + 'expect:resolve_nodes', + ); final Map current = nodes(); final assertions = expected['nodes']! as _Json; expect( @@ -445,6 +509,11 @@ class _Runner { _Json nodeAssertions, String reason, ) { + assertKnownKeys( + nodeAssertions, + _knownKeys['node:resolve_nodes']!, + 'node:resolve_nodes', + ); final Map rawProps = node.props.peek(); final String nodeType = node.state == NodeState.cyclic && node.type == placeholderType diff --git a/dart/a2ui_core/test/conformance/rpc_conformance_test.dart b/dart/a2ui_core/test/conformance/rpc_conformance_test.dart index dd6b6e62ef..1959ae6f30 100644 --- a/dart/a2ui_core/test/conformance/rpc_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/rpc_conformance_test.dart @@ -52,6 +52,40 @@ class _CannedFunction extends FunctionImplementation { } } +const Map> _knownKeys = { + 'case:handle_rpc': { + ...caseMetadataKeys, + 'args', + 'catalog', + 'expect', + 'expectError', + 'protocolVersion', + }, + 'args:handle_rpc': { + 'catalogId', + 'catalogVersion', + 'dataModel', + 'functionMetadata', + 'inboundResponse', + 'message', + 'outboundCall', + 'secondOutboundCall', + 'surfaceId', + 'uiActionFallbackScenario', + 'uiBindingScenario', + 'userActivationPresent', + }, + 'expect:handle_rpc': { + 'correlatedCallId', + 'error', + 'initialValue', + 'outboundCall', + 'resolvedValue', + 'response', + 'result', + }, +}; + /// Runs the shared `conformance/core/rpc_functions.yaml` suite against /// [MessageProcessor] and its [RpcHandler]. void main() { @@ -62,10 +96,11 @@ void main() { group('conformance core/rpc_functions.yaml', () { test('suite is not empty', () => expect(cases, isNotEmpty)); - for (final testCase in cases) { - final name = testCase['name']! as String; - test(name, () => _runCase(testCase)); - } + runConformanceSuite( + cases, + _runCase, + suite: 'core/rpc_functions.yaml', + ); }); } @@ -77,6 +112,7 @@ Future _runCase(Map testCase) async { final args = testCase['args']! as Map; final expected = testCase['expect'] as Map?; final expectError = testCase['expectError'] as Map?; + _assertRpcKeys(testCase, args, expected, expectError); if (expected == null && expectError == null) { throw StateError('handle_rpc test requires "expect" or "expectError".'); } @@ -201,6 +237,26 @@ Future _runCase(Map testCase) async { } } +void _assertRpcKeys( + Map testCase, + Map args, + Map? expected, + Map? expectError, +) { + assertKnownKeys(testCase, _knownKeys['case:handle_rpc']!, 'case:handle_rpc'); + assertKnownKeys(args, _knownKeys['args:handle_rpc']!, 'args:handle_rpc'); + if (expected != null) { + assertKnownKeys( + expected, + _knownKeys['expect:handle_rpc']!, + 'expect:handle_rpc', + ); + } + if (expectError != null) { + assertKnownKeys(expectError, knownExpectErrorKeys, 'expectError'); + } +} + Future _runInboundMessageScenario( Map message, Map? expected, diff --git a/dart/a2ui_core/test/conformance/suite_registry.dart b/dart/a2ui_core/test/conformance/suite_registry.dart new file mode 100644 index 0000000000..b05f7d71db --- /dev/null +++ b/dart/a2ui_core/test/conformance/suite_registry.dart @@ -0,0 +1,116 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Maps each supported `core/*.yaml` conformance suite and `action` to the test +/// file in `test/conformance/` that executes it. +const Map> coreSuiteActions = { + 'core/actions.yaml': { + 'dispatch_action': 'actions_conformance_test.dart', + }, + 'core/catalog.yaml': { + 'derive': 'catalog_schema_conformance_test.dart', + 'from_json': 'catalog_schema_conformance_test.dart', + 'round_trip': 'catalog_schema_conformance_test.dart', + 'to_json': 'catalog_schema_conformance_test.dart', + 'validation_schema': 'catalog_schema_conformance_test.dart', + }, + 'core/composition_constraints.yaml': { + 'process_messages': 'message_processor_conformance_test.dart', + 'validate': 'message_processor_conformance_test.dart', + }, + 'core/data_context.yaml': { + 'resolve_path': 'data_model_conformance_test.dart', + }, + 'core/data_deletion.yaml': { + 'validate': 'data_model_conformance_test.dart', + }, + 'core/data_model.yaml': { + 'data_model': 'data_model_conformance_test.dart', + }, + 'core/data_model_pointers.yaml': { + 'validate': 'data_model_conformance_test.dart', + }, + 'core/error_responses.yaml': { + 'validate': 'validator_conformance_test.dart', + }, + 'core/expressions.yaml': { + 'parse_expression_template': 'expressions_conformance_test.dart', + 'validate': 'message_processor_conformance_test.dart', + }, + 'core/functions.yaml': { + 'evaluate_function': 'functions_conformance_test.dart', + 'validate': 'message_processor_conformance_test.dart', + }, + 'core/index_function.yaml': { + 'validate': 'message_processor_conformance_test.dart', + }, + 'core/message_processor_scaling.yaml': { + 'measure_growth': 'message_processor_scaling_conformance_test.dart', + }, + 'core/message_processor_v0_9.yaml': { + 'get_renderer_capabilities': 'message_processor_conformance_test.dart', + 'get_renderer_data_model': 'message_processor_conformance_test.dart', + 'process_messages': 'message_processor_conformance_test.dart', + 'resolve_path': 'message_processor_conformance_test.dart', + }, + 'core/message_processor_v1_0.yaml': { + 'get_renderer_capabilities': 'message_processor_conformance_test.dart', + 'get_renderer_data_model': 'message_processor_conformance_test.dart', + 'process_messages': 'message_processor_conformance_test.dart', + 'validate': 'message_processor_conformance_test.dart', + }, + 'core/multi_catalog.yaml': { + 'select_catalog': 'multi_catalog_conformance_test.dart', + 'validate': 'message_processor_conformance_test.dart', + }, + 'core/node_resolution.yaml': { + 'resolve_nodes': 'node_resolution_conformance_test.dart', + }, + 'core/reserved_keys.yaml': { + 'process_messages': 'message_processor_conformance_test.dart', + 'validate': 'validator_conformance_test.dart', + }, + 'core/rpc_functions.yaml': { + 'handle_rpc': 'rpc_conformance_test.dart', + }, + 'core/validation_result.yaml': { + 'validate': 'validator_conformance_test.dart', + }, + 'core/validator_v0_9.yaml': { + 'validate': 'validator_conformance_test.dart', + }, + 'core/validator_v1_0.yaml': { + 'validate': 'validator_conformance_test.dart', + }, +}; + +/// `core/*.yaml` suites that `a2ui_core` intentionally does not execute, paired +/// with the reason recorded in the executed-case manifest. +const Map unsupportedCoreSuites = { + 'core/accessibility.yaml': + 'Accessibility checks require a rendered UI tree and run in the ' + 'framework adapter harness.', + 'core/adapter_conformance.yaml': + 'Adapter conformance cases require a UI framework renderer and run in ' + 'the framework adapter harness.', + 'core/agent_to_renderer.yaml': + 'Protocol envelope schema generation is not implemented in a2ui_core.', + 'core/common_types.yaml': + 'Common-type schema generation and validation are not implemented in ' + 'a2ui_core.', + 'core/message_processor_v0_8.yaml': + 'Targets protocol v0.8, which a2ui_core does not implement.', + 'core/validator_v0_8.yaml': + 'Targets protocol v0.8, which a2ui_core does not implement.', +}; diff --git a/dart/a2ui_core/test/conformance/suite_registry_test.dart b/dart/a2ui_core/test/conformance/suite_registry_test.dart new file mode 100644 index 0000000000..c7004188bb --- /dev/null +++ b/dart/a2ui_core/test/conformance/suite_registry_test.dart @@ -0,0 +1,104 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +import 'conformance_harness.dart'; +import 'suite_registry.dart'; + +Set _discoverCoreSuites() { + final coreDir = Directory(resolveConformancePath('core')); + final suites = {}; + for (final FileSystemEntity entity in coreDir.listSync()) { + if (entity is File && entity.path.endsWith('.yaml')) { + suites.add('core/${p.basename(entity.path)}'); + } + } + return suites; +} + +Set _actionsInSuite(String suite) { + final List cases = loadConformanceSuite(suite); + return { + for (final ConformanceTestCase testCase in cases) + (testCase['action'] as String?) ?? 'default', + }; +} + +String _resolveTestFilePath(String fileName) { + final direct = File(p.join('test', 'conformance', fileName)); + if (direct.existsSync()) return direct.path; + return p.join('dart', 'a2ui_core', 'test', 'conformance', fileName); +} + +void main() { + group('core suite registry', () { + test('registers every conformance/core/*.yaml file exactly once', () { + final Set onDisk = _discoverCoreSuites(); + final Set active = coreSuiteActions.keys.toSet(); + final Set unsupported = unsupportedCoreSuites.keys.toSet(); + + expect( + active.intersection(unsupported), + isEmpty, + reason: 'A suite cannot be in both coreSuiteActions and ' + 'unsupportedCoreSuites.', + ); + expect( + active.union(unsupported), + equals(onDisk), + reason: 'Every conformance/core/*.yaml file must be registered in ' + 'coreSuiteActions or unsupportedCoreSuites.', + ); + }); + + for (final MapEntry> entry + in coreSuiteActions.entries) { + test('${entry.key} maps every action to an existing test file', () { + final Set actualActions = _actionsInSuite(entry.key); + expect( + entry.value.keys.toSet(), + equals(actualActions), + reason: + 'Registered actions for ${entry.key} must match actions in YAML.', + ); + for (final String testFile in entry.value.values.toSet()) { + expect( + File(_resolveTestFilePath(testFile)).existsSync(), + isTrue, + reason: 'Test file $testFile for ${entry.key} does not exist.', + ); + } + }); + } + + for (final MapEntry entry + in unsupportedCoreSuites.entries) { + test('${entry.key} records skipped manifest entries with a reason', () { + final String suite = entry.key; + final String reason = entry.value.trim(); + expect(reason, isNotEmpty); + final List cases = loadConformanceSuite(suite); + expect(cases, isNotEmpty); + for (final testCase in cases) { + final name = testCase['name']! as String; + recordManifest(suite, name, 'skipped', reason); + } + }); + } + }); +} diff --git a/dart/a2ui_core/test/conformance/validator_conformance_test.dart b/dart/a2ui_core/test/conformance/validator_conformance_test.dart index d6d8ae4795..29ef125318 100644 --- a/dart/a2ui_core/test/conformance/validator_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/validator_conformance_test.dart @@ -31,18 +31,68 @@ const Map _v10ExpectedFailures = {}; const Map _reservedKeysExpectedFailures = {}; /// Cases in `core/validation_result.yaml` expected to fail. -const Map _validationResultExpectedFailures = {}; +const Map _validationResultExpectedFailures = { + 'test_validation_result_dynamic_object_return': + 'NodeResolver does not expose a singular validationResult property on ' + 'resolved component props, and the built-in email function omits the ' + 'INVALID_EMAIL code and example suffix in its default message.', + 'test_validation_result_boolean_fallback': + 'NodeResolver does not expose a singular validationResult property on ' + 'resolved component props.', + 'test_validation_result_multiple_rules_preserve_order': + 'Built-in email validation function does not set code INVALID_EMAIL on ' + 'its ValidationResult.', + 'test_validation_result_reactive_recovery_on_data_update': + 'Built-in email validation function returns "Must be a valid email ' + 'address." rather than the message with the ' + '"(e.g. user@example.com)" suffix.', +}; /// Cases in `core/error_responses.yaml` expected to fail. const Map _errorResponsesExpectedFailures = {}; +const Map> _knownKeys = { + 'case:validate': { + ...caseMetadataKeys, + 'catalog', + 'catalogPath', + 'catalogPaths', + 'expect', + 'expectError', + 'expect_error', + 'messages', + 'payload', + 'protocolVersion', + 'steps', + 'strictMode', + 'validationConfig', + }, + 'step:validate': { + 'description', + 'expect', + 'expectError', + 'expect_error', + 'messages', + 'payload', + }, + 'expect:validate': { + 'surfaces', + }, + 'surface:validate': { + 'components', + 'isValid', + 'validationErrors', + 'validationResult', + 'validationResults', + }, +}; + /// Runs the shared validator suites against [MessageProcessor.processMessages], /// the entry point for checking a payload on its own: the `validate` cases of /// `core/validator_v0_9.yaml`, `core/validator_v1_0.yaml`, /// `core/reserved_keys.yaml`, `core/validation_result.yaml` and /// `core/error_responses.yaml`. The cases of -/// `core/composition_constraints.yaml` are all `process_messages` cases, which -/// the message-processor harness runs. +/// `core/composition_constraints.yaml` run in the message-processor harness. /// /// Cases targeting a protocol version this SDK does not implement are skipped /// with a reason, so the suite doubles as the implementation checklist. @@ -86,6 +136,7 @@ void _registerValidatorSuite( runConformanceSuite( cases, _runCase, + suite: suite, expectedFailures: expectedFailures, skipReason: _skipReason, ); @@ -102,6 +153,16 @@ String? _skipReason(ConformanceTestCase testCase) { } void _runCase(Map testCase) { + assertKnownKeys(testCase, _knownKeys['case:validate']!, 'case:validate'); + if (testCase['steps'] case final List explicitSteps) { + for (final rawStep in explicitSteps) { + assertKnownKeys( + rawStep! as Map, + _knownKeys['step:validate']!, + 'step:validate', + ); + } + } final List> steps = _steps(testCase); final List> allPayloads = _collectPayloads(steps); final String version = _versionOf(testCase, allPayloads); @@ -203,6 +264,11 @@ void _assertExpectedSurfaces( MessageProcessor processor, Map expectBlock, ) { + assertKnownKeys( + expectBlock, + _knownKeys['expect:validate']!, + 'expect:validate', + ); final Object? surfaces = expectBlock['surfaces']; if (surfaces is! Map) return; for (final MapEntry entry in surfaces.entries) { @@ -215,10 +281,42 @@ void _assertExpectedSurfaces( final Map> nodes = _collectSurfaceNodes(resolver.rootNode.peek()); final expectedSurface = entry.value as Map?; - final Object? expectedComponents = expectedSurface?['components']; + if (expectedSurface == null) continue; + assertKnownKeys( + expectedSurface, + _knownKeys['surface:validate']!, + 'surface:validate', + ); + final Object? expectedComponents = expectedSurface['components']; if (expectedComponents is Map) { _assertSurfaceComponentProps(nodes, expectedComponents); } + _assertSurfaceFieldMaps(nodes, expectedSurface); + } +} + +void _assertSurfaceFieldMaps( + Map> nodes, + Map expectedSurface, +) { + for (final field in const [ + 'validationResult', + 'isValid', + 'validationErrors', + 'validationResults', + ]) { + final Object? fieldMap = expectedSurface[field]; + if (fieldMap is! Map) continue; + for (final MapEntry entry in fieldMap.entries) { + final ComponentNode? node = nodes[entry.key]; + expect(node, isNotNull, reason: 'Component ${entry.key}'); + final Map props = node!.props.peek(); + expect( + _unwrapResolved(props[field]), + equals(entry.value), + reason: '${entry.key}.$field', + ); + } } } @@ -371,7 +469,7 @@ Map _document(Object? value) { /// The suite's fixtures name the catalog `standard` in the document but `std` /// in the payloads that use it. The processor resolves the catalog a surface /// names against the ones it supports, so the mismatch would reject those -/// payloads outright, which is not what these cases are testing — they are +/// payloads outright, which is not what these cases are testing: they are /// about the component graph. So a case declaring one document has it /// registered under every id the payload names, and catalog resolution keeps /// its own coverage in `processor_test.dart`. From d77696c12829fd0a37f09d00e70f0ba8b0ce972e Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 18:13:07 +0000 Subject: [PATCH 09/48] test(swift-core): record executed conformance cases and enforce strict dispatch --- .github/workflows/swift_build_and_test.yml | 11 + .../AccessibilityConformanceTests.swift | 269 ++++++--- .../ActionsConformanceTests.swift | 89 ++- .../BasicCatalogConformanceTests.swift | 387 +++++++++---- .../CatalogConformanceTests.swift | 100 +++- .../CatalogDeriveConformanceTests.swift | 88 ++- .../ConformanceManifest.swift | 153 ++++++ .../ConformanceTestHelper.swift | 520 ++++++++++-------- .../DataModelConformanceTests.swift | 300 ++++++---- .../DataModelPointerConformanceTests.swift | 247 ++++++--- .../ExpressionsConformanceTests.swift | 310 ++++++++--- .../MessageProcessorConformanceTests.swift | 348 +++++++++--- .../MultiCatalogConformanceTests.swift | 338 +++++++++--- .../NodeResolutionConformanceTests.swift | 304 ++++++---- .../RPCConformanceTests.swift | 262 +++++++-- .../SuiteRegistryTests.swift | 86 +++ .../ValidatorConformanceTests.swift | 268 ++++++--- 17 files changed, 2993 insertions(+), 1087 deletions(-) create mode 100644 swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift create mode 100644 swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift diff --git a/.github/workflows/swift_build_and_test.yml b/.github/workflows/swift_build_and_test.yml index 38d7987e2f..0b949407d1 100644 --- a/.github/workflows/swift_build_and_test.yml +++ b/.github/workflows/swift_build_and_test.yml @@ -54,7 +54,18 @@ jobs: with: persist-credentials: false - name: Run tests + env: + A2UI_CONFORMANCE_MANIFEST_DIR: ${{ runner.temp }}/a2ui-manifest run: swift test --enable-swift-testing + - name: Set up Python + uses: ./.github/actions/setup-python + - name: Check executed conformance cases + run: | + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --harness swift-core \ + --corpus core \ + --exclude core/message_processor_scaling.yaml="no Swift timing harness" ios-build: # `swift test` above builds for macOS only, so `#if os(iOS)` and UIKit code diff --git a/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift index 070d7fdecc..9d51b516b2 100644 --- a/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift @@ -40,55 +40,124 @@ struct AccessibilityConformanceTests { /// at the referenced paths. private static let dataModelRounds = 2 - @Test func accessibilityConformance() throws { - let rawYaml = try ConformanceTestHelper.loadYAML(filename: "core/accessibility.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYaml) + private static let knownCaseKeys: Set = [ + "action", + "assertions", + "catalogPath", + "messages", + "protocolVersion", + "steps", + "surface", + ] - #expect(!testCases.isEmpty, "Should find test cases in accessibility.yaml") + private static let knownAssertionsKeys: Set = [ + "accessibilityTree", + "axeCore", + ] - var checkedFields = 0 - var checkedReferences = 0 + private static let knownStepKeys: Set = [ + "assertions", + "messages", + ] - for testCase in testCases { - guard testCase.action == "accessibility_check" else { - Issue.record("[\(testCase.name)] Unhandled action '\(testCase.action ?? "")'") - continue - } - guard let surfaceDefinition = testCase.surface else { - Issue.record("[\(testCase.name)] The case has no surface") - continue - } - guard let expectedTree = testCase.assertions?["accessibilityTree"]?.objectValue else { - Issue.record("[\(testCase.name)] The case has no accessibilityTree assertions") - continue + @Test func accessibilityConformance() throws { + let cases = ConformanceTestHelper.cases("core/accessibility.yaml") + #expect(!cases.isEmpty, "Should find test cases in accessibility.yaml") + + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runSingleAccessibilityCase(conformanceCase) } + } + } + + private func runSingleAccessibilityCase(_ conformanceCase: ConformanceCase) throws { + try validateCaseStructure(conformanceCase) + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse accessibility case" + ) + guard testCase.action == "accessibility_check" else { + Issue.record("[\(testCase.name)] Unhandled action '\(testCase.action ?? "")'") + throw UnknownConformanceKeyError( + "[\(testCase.name)] Unhandled action '\(testCase.action ?? "")'" + ) + } + let surfaceDefinition = try #require( + testCase.surface, + "[\(testCase.name)] The case has no surface" + ) + let expectedTree = try #require( + testCase.assertions?["accessibilityTree"]?.objectValue, + "[\(testCase.name)] The case has no accessibilityTree assertions" + ) - let surface = try makeSurface(testCase: testCase, surfaceDefinition: surfaceDefinition) + let surface = try makeSurface(testCase: testCase, surfaceDefinition: surfaceDefinition) + let references = dataModelReferences(in: expectedTree, testCase: testCase) + if references.isEmpty { + let checkedFields = try check(expectedTree, on: surface, testCase: testCase) + #expect(checkedFields > 0, "[\(testCase.name)] no accessibilityTree assertion was checked") + return + } + try checkDataModelRounds( + expectedTree: expectedTree, + references: references, + surface: surface, + testCase: testCase + ) + } - let references = dataModelReferences(in: expectedTree, testCase: testCase) - guard !references.isEmpty else { - checkedFields += check(expectedTree, on: surface, testCase: testCase) - continue - } - let sortedReferences = references.sorted { $0.key < $1.key } - for round in 0.. 0, "no accessibilityTree assertion of the suite was checked") - #expect(checkedReferences > 0, "no data model reference of the suite was checked") + private func checkDataModelRounds( + expectedTree: OrderedDictionary, + references: [String: String], + surface: SurfaceViewModel, + testCase: ConformanceTestCase + ) throws { + let sortedReferences = references.sorted { $0.key < $1.key } + var checkedFields = 0 + var checkedReferences = 0 + for round in 0.. 0, "[\(testCase.name)] no accessibilityTree assertion was checked") + #expect(checkedReferences > 0, "[\(testCase.name)] no data model reference was checked") } /// Creates a surface with the case's catalogs and adds the case's components to it. @@ -148,7 +217,7 @@ struct AccessibilityConformanceTests { _ expectedTree: OrderedDictionary, on surface: SurfaceViewModel, testCase: ConformanceTestCase - ) -> Int { + ) throws -> Int { guard surface.rootNode != nil else { Issue.record("[\(testCase.name)] The surface resolved no tree") return 0 @@ -159,61 +228,92 @@ struct AccessibilityConformanceTests { Issue.record("[\(testCase.name)] Expectation for '\(componentID)' is not an object") continue } + checkedFields += try checkComponentFields( + componentID: componentID, + expectedFields: expectedFields, + on: surface, + testCase: testCase + ) + } + return checkedFields + } + + private func checkComponentFields( + componentID: String, + expectedFields: OrderedDictionary, + on surface: SurfaceViewModel, + testCase: ConformanceTestCase + ) throws -> Int { + guard + let node = surface.findNode(id: componentID) + ?? resolveDetached(componentID: componentID, on: surface) + else { + Issue.record("[\(testCase.name)] Component '\(componentID)' could not be resolved") + return 0 + } + let attributes = node.accessibilityAttributes + var checkedFields = 0 + for (field, expectedShape) in expectedFields { + guard !Self.rendererOnlyFields.contains(field) else { continue } + let actual = try actualFieldValue( + field: field, + attributes: attributes, + componentID: componentID, + testName: testCase.name + ) guard - let node = surface.findNode(id: componentID) - ?? resolveDetached(componentID: componentID, on: surface) - else { - Issue.record("[\(testCase.name)] Component '\(componentID)' could not be resolved") - continue - } - let attributes = node.accessibilityAttributes - for (field, expectedShape) in expectedFields { - guard !Self.rendererOnlyFields.contains(field) else { continue } - let actual: JSONValue? - switch field { - case "label": actual = attributes?.label.map(JSONValue.string) - case "description": actual = attributes?.description.map(JSONValue.string) - case "live": actual = attributes?.live.map(JSONValue.string) - case "hidden": actual = attributes?.hidden.map(JSONValue.boolean) - default: - Issue.record("[\(testCase.name)] Unhandled field '\(field)' for '\(componentID)'") - continue - } - guard - let expected = resolveExpectation( - expectedShape, - dataModel: surface.dataModel, - context: "[\(testCase.name)] \(componentID).\(field)" - ) - else { continue } - #expect( - actual == expected, - """ - [\(testCase.name)] Expected \(field) \(expected) for '\(componentID)', \ - got \(actual.map { "\($0)" } ?? "nil") - """ + let expected = try resolveExpectation( + expectedShape, + dataModel: surface.dataModel, + context: "[\(testCase.name)] \(componentID).\(field)" ) - checkedFields += 1 - } + else { continue } + #expect( + actual == expected, + """ + [\(testCase.name)] Expected \(field) \(expected) for '\(componentID)', \ + got \(actual.map { "\($0)" } ?? "nil") + """ + ) + checkedFields += 1 } return checkedFields } + private func actualFieldValue( + field: String, + attributes: AccessibilityAttributes?, + componentID: String, + testName: String + ) throws -> JSONValue? { + switch field { + case "label": return attributes?.label.map(JSONValue.string) + case "description": return attributes?.description.map(JSONValue.string) + case "live": return attributes?.live.map(JSONValue.string) + case "hidden": return attributes?.hidden.map(JSONValue.boolean) + default: + Issue.record("[\(testName)] Unhandled field '\(field)' for '\(componentID)'") + throw UnknownConformanceKeyError( + "[\(testName)] Unhandled field '\(field)' for '\(componentID)'" + ) + } + } + /// Returns the value an expectation stands for: a string or a boolean as is, or the data model - /// value at the path of a `{path}` or `{"@path"}` reference. Records an issue and returns `nil` - /// for any other shape, and for a reference to a path that has no value. + /// value at the path of a `{path}` or `{"@path"}` reference. Throws for any other shape, and + /// records an issue and returns `nil` for a reference to a path that has no value. private func resolveExpectation( _ shape: JSONValue, dataModel: DataModel, context: String - ) -> JSONValue? { + ) throws -> JSONValue? { switch shape { case .string, .boolean: return shape default: guard let path = referencedPath(in: shape) else { Issue.record("\(context): unhandled expectation shape \(shape)") - return nil + throw UnknownConformanceKeyError("\(context): unhandled expectation shape \(shape)") } guard let value = dataModel.get(path) else { Issue.record("\(context): the data model has no value at '\(path)'") @@ -256,14 +356,19 @@ struct AccessibilityConformanceTests { for field: String, path: String, index: Int, - round: Int - ) -> JSONValue? { + round: Int, + testName: String + ) throws -> JSONValue { let isEvenStep = (round + index).isMultiple(of: 2) switch field { case "label", "description": return .string("\(field) \(path) \(round)") case "live": return .string(isEvenStep ? "polite" : "assertive") case "hidden": return .boolean(isEvenStep) - default: return nil + default: + Issue.record("[\(testName)] No sample value for field '\(field)' bound to '\(path)'") + throw UnknownConformanceKeyError( + "[\(testName)] No sample value for field '\(field)' bound to '\(path)'" + ) } } diff --git a/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift index ac504b2735..46e06ecc2b 100644 --- a/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift @@ -85,7 +85,8 @@ private final class ActionTestFunction: FunctionImplementation, Sendable { name: String, block: @escaping @MainActor @Sendable ([String: JSONValue], DataContext) throws -> JSONValue ) { - let emptySchema = (try? Schema(instance: "{}")) + let emptySchema = + (try? Schema(instance: "{}")) ?? (try! Schema(rawSchema: .object([:]), context: Context(dialect: .draft2020_12))) self.api = FunctionAPI( name: name, @@ -104,20 +105,63 @@ private final class ActionTestFunction: FunctionImplementation, Sendable { /// Runs the shared `conformance/core/actions.yaml` suite. @MainActor struct ActionsConformanceTests { + private static let knownCaseKeys: Set = [ + "action", + "actionPayload", + "dataModel", + "expectDataModel", + "expectDispatched", + "expectError", + "metadata", + "protocolVersion", + "scope", + "sourceComponentId", + "surfaceId", + "userActivation", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "code", + "message", + "path", + "surfaceId", + ] + + private static let knownExpectDispatchedKeys: Set = [ + "context", + "metadata", + "name", + "sourceComponentId", + "surfaceId", + "timestampIso8601", + "userMessage", + ] + @Test func actionsConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/actions.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases("core/actions.yaml") #expect(!cases.isEmpty, "core/actions.yaml should hold test cases") - for testCase in cases { - let name = testCase["name"] as? String ?? "" - let action = testCase["action"] as? String ?? "dispatch_action" - guard action == "dispatch_action" else { continue } - try runDispatchActionCase(testCase, name: name) + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + let testCase = conformanceCase.rawDictionary + let action = testCase["action"] as? String ?? "" + guard action == "dispatch_action" else { + throw UnknownConformanceKeyError( + "[\(conformanceCase.name)] Unknown actions.yaml action '\(action)'" + ) + } + try runDispatchActionCase(testCase, name: conformanceCase.name) + } } } private func runDispatchActionCase(_ testCase: [String: Any], name: String) throws { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownCaseKeys, + "actions.yaml case '\(name)'" + ) let versionStr = testCase["protocolVersion"] as? String ?? "v0.9" let protocolVersion = A2UIProtocolVersion(lenientRawValue: versionStr) ?? .v09 let surfaceID = testCase["surfaceId"] as? String ?? "default" @@ -161,6 +205,12 @@ struct ActionsConformanceTests { if let expectErrorDict = testCase["expectError"] as? [String: Any], expectErrorDict["category"] as? String == "ValidationError" { + try ConformanceTestHelper.assertKnownKeys( + expectErrorDict, + Self.knownExpectErrorKeys, + "actions.yaml expectError in '\(name)'", + allowMetadata: false + ) #expect(throws: A2UIValidationError.self, "\(name): expected metadata validation error") { _ = try MetadataValidator.mergeActionMetadata(base: baseMeta, override: overrideMeta) } @@ -309,13 +359,26 @@ struct ActionsConformanceTests { } if let expectErrorDict = testCase["expectError"] as? [String: Any] { - assertActionErrorMatches(capturedErrors: capturedErrors, expectError: expectErrorDict, name: name) + try ConformanceTestHelper.assertKnownKeys( + expectErrorDict, + Self.knownExpectErrorKeys, + "actions.yaml expectError in '\(name)'", + allowMetadata: false + ) + assertActionErrorMatches( + capturedErrors: capturedErrors, + expectError: expectErrorDict, + name: name + ) } else { #expect(capturedErrors.isEmpty, "\(name): unexpected action errors \(capturedErrors)") } if testCase["expectDispatched"] is NSNull { - #expect(dispatched == nil, "\(name): expected no dispatched action, got \(String(describing: dispatched))") + #expect( + dispatched == nil, + "\(name): expected no dispatched action, got \(String(describing: dispatched))" + ) return } @@ -324,6 +387,12 @@ struct ActionsConformanceTests { else { return } + try ConformanceTestHelper.assertKnownKeys( + expDict, + Self.knownExpectDispatchedKeys, + "actions.yaml expectDispatched in '\(name)'", + allowMetadata: false + ) try assertDispatchedActionMatches(dispatched: dispatched, expected: expDict, name: name) } diff --git a/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift index 0bd2e8cc63..fa13c86b79 100644 --- a/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift @@ -38,6 +38,49 @@ private final class ConformanceMockFunctionHandler: FunctionHandler { @MainActor struct BasicCatalogConformanceTests { + private static let knownEvaluateFunctionCaseKeys: Set = [ + "action", + "args", + "catalogId", + "catalogPaths", + "dataModel", + "expect", + "expectError", + "function", + "protocolVersion", + "surface", + ] + + private static let knownValidateCaseKeys: Set = [ + "action", + "catalog", + "expect", + "steps", + ] + + private static let knownValidateStepKeys: Set = [ + "expect", + "expectError", + "payload", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + ] + + private static let knownComponentExpectKeys: Set = [ + "component", + "text", + ] + + private static let knownValidationResultSurfaceKeys: Set = [ + "isValid", + "validationErrors", + "validationResult", + "validationResults", + ] + private let functionHandler = ConformanceMockFunctionHandler() /// Handlers holding each protocol version's Basic Catalog functions. Stored here because /// `DataContext` keeps its function handler weakly. @@ -78,45 +121,24 @@ struct BasicCatalogConformanceTests { } @Test func evaluateFunctionConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/functions.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] - #expect(!cases.isEmpty, "core/functions.yaml should hold test cases") - - var executed = 0 - for testCase in cases { - guard let name = testCase["name"] as? String, - let action = testCase["action"] as? String - else { - Issue.record("core/functions.yaml holds a case without a name or an action") - continue - } - - switch action { - case "validate": - executed += 1 - try runValidateCase(testCase, name: name) - case "evaluate_function": - executed += 1 + let cases = ConformanceTestHelper.cases("core/functions.yaml", action: "evaluate_function") + #expect(!cases.isEmpty, "core/functions.yaml should hold evaluate_function cases") + + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + let testCase = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownEvaluateFunctionCaseKeys, + "functions.yaml evaluate_function case '\(conformanceCase.name)'" + ) if let surfaceDict = testCase["surface"] as? [String: Any] { try runSurfaceEvaluateFunctionCase(testCase, surfaceDict: surfaceDict) } else { try runDirectEvaluateFunctionCase(testCase) } - default: - Issue.record("\(name): unsupported action \(action)") } } - - #expect(executed == cases.count, "All cases in core/functions.yaml should be executed") - } - - /// Runs a `validate` case, which renders basic catalog components that call a function. - private func runValidateCase(_ rawCase: [String: Any], name: String) throws { - let testCase = try #require( - ConformanceTestHelper.parseTestCases(from: [rawCase]).first, - "\(name): case could not be parsed" - ) - try ConformanceTestHelper.runValidateCase(testCase) } private func runDirectEvaluateFunctionCase(_ testCase: [String: Any]) throws { @@ -137,6 +159,12 @@ struct BasicCatalogConformanceTests { let swiftArgs = buildFunctionArgs(from: testCase) if let expectError = (testCase["expectError"] ?? testCase["expect_error"]) as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "functions.yaml expectError in '\(name)'", + allowMetadata: false + ) assertDirectFunctionError( fn: fn, args: swiftArgs, @@ -267,6 +295,12 @@ struct BasicCatalogConformanceTests { let actual = context.resolveDynamicValue(callExpr) if let expectError = (testCase["expectError"] ?? testCase["expect_error"]) as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "functions.yaml surface expectError in '\(name)'", + allowMetadata: false + ) assertSurfaceFunctionError( capturedErrors: errorHandler.capturedErrors, expectError: expectError, @@ -346,67 +380,118 @@ struct BasicCatalogConformanceTests { return ConformanceTestHelper.messageMatches(msg, expected: expectedMessage) } #expect( - matches, "\(name): expected error containing '\(expectedMessage)', got \(capturedErrors)") + matches, + "\(name): expected error containing '\(expectedMessage)', got \(capturedErrors)" + ) + } + + @Test func functionsValidateConformance() throws { + let cases = ConformanceTestHelper.cases("core/functions.yaml", action: "validate") + #expect(!cases.isEmpty, "core/functions.yaml should hold validate cases") + + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runValidatedCatalogCase(conformanceCase, suiteName: "functions.yaml") + } + } } @Test func indexFunctionConformance() throws { - let rawYaml = try ConformanceTestHelper.loadYAML(filename: "core/index_function.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYaml) - #expect(!testCases.isEmpty, "Should load test cases from index_function.yaml") - - for testCase in testCases { - let errorHandler = IndexConformanceErrorHandler() - let processor = MessageProcessor( - catalogs: ConformanceTestHelper.basicCatalogs(for: .v10), - actionHandler: errorHandler, - validationConfig: ValidationConfig(targetVersion: "1.0") - ) + let cases = ConformanceTestHelper.cases("core/index_function.yaml", action: "validate") + #expect(!cases.isEmpty, "Should load test cases from index_function.yaml") - for (stepIndex, step) in testCase.steps.enumerated() { - guard let payload = step.payload else { continue } - try executeIndexStep( - step: step, - isFinalStep: stepIndex == testCase.steps.count - 1, - payload: payload, - testCase: testCase, - processor: processor, - errorHandler: errorHandler + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runValidatedCatalogCase(conformanceCase, suiteName: "index_function.yaml") + } + } + } + + private func runValidatedCatalogCase( + _ conformanceCase: ConformanceCase, + suiteName: String + ) throws { + let rawDict = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + rawDict, + Self.knownValidateCaseKeys, + "\(suiteName) case '\(conformanceCase.name)'" + ) + if let steps = rawDict["steps"] as? [[String: Any]] { + for step in steps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownValidateStepKeys, + "\(suiteName) step in '\(conformanceCase.name)'", + allowMetadata: false ) + if let errDict = step["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + errDict, + Self.knownExpectErrorKeys, + "\(suiteName) step.expectError in '\(conformanceCase.name)'", + allowMetadata: false + ) + } } } + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse test case" + ) + let errorHandler = IndexConformanceErrorHandler() + let processor = MessageProcessor( + catalogs: ConformanceTestHelper.basicCatalogs(for: .v10), + actionHandler: errorHandler, + validationConfig: ValidationConfig(targetVersion: "1.0") + ) + + for step in testCase.steps { + guard let payload = step.payload else { continue } + try executeIndexStep( + step: step, + payload: payload, + testCase: testCase, + processor: processor, + errorHandler: errorHandler + ) + } } private func executeIndexStep( step: ConformanceStep, - isFinalStep: Bool, payload: JSONValue, testCase: ConformanceTestCase, processor: MessageProcessor, errorHandler: IndexConformanceErrorHandler ) throws { - // The case's expectError applies to the last step only. - if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { + if let expectedError = step.expectError { errorHandler.capturedErrors.removeAll() - var caughtErrorMessage: String? + var caughtError: Error? do { - let messages = try ConformanceTestHelper.parsePayload(payload) - processor.process(messages: messages) + try processor.processMessages(payload) } catch { - caughtErrorMessage = error.localizedDescription + caughtError = error } assertIndexStepError( capturedErrors: errorHandler.capturedErrors, - caughtErrorMessage: caughtErrorMessage, + caughtError: caughtError, expectedError: expectedError, testName: testCase.name ) return } - let messages = try ConformanceTestHelper.parsePayload(payload) - processor.process(messages: messages) - guard let expectSurfaces = testCase.expect?["surfaces"]?.objectValue else { return } - assertIndexSurfaces( + try processor.processMessages(payload) + guard let expectObj = testCase.expect?.objectValue else { return } + try ConformanceTestHelper.assertKnownKeys( + expectObj, + ["surfaces"], + "expect in '\(testCase.name)'", + allowMetadata: false + ) + guard let expectSurfaces = expectObj["surfaces"]?.objectValue else { return } + try assertIndexSurfaces( OrderedDictionary(uniqueKeysWithValues: expectSurfaces.map { ($0.key, $0.value) }), processor: processor, testName: testCase.name @@ -415,10 +500,12 @@ struct BasicCatalogConformanceTests { private func assertIndexStepError( capturedErrors: [RendererError], - caughtErrorMessage: String?, + caughtError: Error?, expectedError: ConformanceExpectError, testName: String ) { + let caughtErrorMessage = + (caughtError as? (any A2UIError))?.message ?? caughtError?.localizedDescription let errorMessages = capturedErrors.map { err -> String in switch err { @@ -431,6 +518,14 @@ struct BasicCatalogConformanceTests { !errorMessages.isEmpty, "Expected error for \(testName), but processing succeeded without error" ) + if let expectedCategory = expectedError.category, expectedCategory == "ValidationError", + let caughtError + { + #expect( + caughtError is A2UIValidationError, + "[\(testName)] Expected A2UIValidationError, got \(type(of: caughtError))" + ) + } if let expectedMessage = expectedError.message { #expect( errorMessages.contains(where: { $0.contains(expectedMessage) }), @@ -443,65 +538,141 @@ struct BasicCatalogConformanceTests { _ expectSurfaces: OrderedDictionary, processor: MessageProcessor, testName: String - ) { + ) throws { for (surfaceID, expectedSurface) in expectSurfaces { + if let surfObj = expectedSurface.objectValue { + try ConformanceTestHelper.assertKnownKeys( + surfObj, + ["components"], + "expect.surfaces.\(surfaceID) in '\(testName)'", + allowMetadata: false + ) + } guard let surface = processor.surfaceGroupModel[surfaceID] else { Issue.record("Expected surface '\(surfaceID)' to exist") continue } guard let expectedComponents = expectedSurface["components"]?.objectValue else { continue } for (compID, compExpected) in expectedComponents { - let node = surface.findNode(id: compID) - if let expectedText = compExpected["text"]?.stringValue { - #expect( - node?.string(for: "text") == expectedText, - "[\(testName)] Expected node \(compID).text == '\(expectedText)'" - ) - } + try assertIndexSurfaceComponent( + compID: compID, + compExpected: compExpected, + surface: surface, + surfaceID: surfaceID, + testName: testName + ) } } } - @Test func validationResultConformance() throws { - let rawYaml = try ConformanceTestHelper.loadYAML(filename: "core/validation_result.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYaml) - #expect(!testCases.isEmpty, "Should load test cases from validation_result.yaml") - - for testCase in testCases { - let processor = MessageProcessor( - catalogs: ConformanceTestHelper.basicCatalogs(for: .v10), - validationConfig: ValidationConfig(targetVersion: "1.0") + private func assertIndexSurfaceComponent( + compID: String, + compExpected: JSONValue, + surface: SurfaceViewModel, + surfaceID: String, + testName: String + ) throws { + if let compObj = compExpected.objectValue { + try ConformanceTestHelper.assertKnownKeys( + compObj, + Self.knownComponentExpectKeys, + "expect.surfaces.\(surfaceID).components.\(compID) in '\(testName)'", + allowMetadata: false ) + } + let node = surface.findNode(id: compID) + if let expectedComponent = compExpected["component"]?.stringValue { + #expect( + node?.type == expectedComponent, + "[\(testName)] Expected node \(compID).type == '\(expectedComponent)'" + ) + } + let expectedText = + compExpected["text"]?.stringValue + ?? compExpected["text"]?.boolValue.map { $0 ? "true" : "false" } + if let expectedText { + #expect( + node?.string(for: "text") == expectedText, + "[\(testName)] Expected node \(compID).text == '\(expectedText)'" + ) + } + } - for step in testCase.steps { - guard let payload = step.payload else { continue } - let messages = try ConformanceTestHelper.parsePayload(payload) - processor.process(messages: messages) - if let stepExpectSurfaces = step.expect?["surfaces"]?.objectValue { - assertValidationResultSurfaces( - OrderedDictionary(uniqueKeysWithValues: stepExpectSurfaces.map { ($0.key, $0.value) }), - processor: processor, - testName: testCase.name - ) - } + @Test func validationResultConformance() throws { + let cases = ConformanceTestHelper.cases("core/validation_result.yaml", action: "validate") + #expect(!cases.isEmpty, "Should load test cases from validation_result.yaml") + + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runValidationResultCase(conformanceCase) + } + } + } + + private func runValidationResultCase(_ conformanceCase: ConformanceCase) throws { + let rawDict = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + rawDict, + Self.knownValidateCaseKeys, + "validation_result.yaml case '\(conformanceCase.name)'" + ) + if let steps = rawDict["steps"] as? [[String: Any]] { + for step in steps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownValidateStepKeys, + "validation_result.yaml step in '\(conformanceCase.name)'", + allowMetadata: false + ) } + } + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse validation_result case" + ) + let processor = MessageProcessor( + catalogs: ConformanceTestHelper.basicCatalogs(for: .v10), + validationConfig: ValidationConfig(targetVersion: "1.0") + ) - if let caseExpectSurfaces = testCase.expect?["surfaces"]?.objectValue { - assertValidationResultSurfaces( - OrderedDictionary(uniqueKeysWithValues: caseExpectSurfaces.map { ($0.key, $0.value) }), + for step in testCase.steps { + guard let payload = step.payload else { continue } + let messages = try ConformanceTestHelper.parsePayload(payload) + processor.process(messages: messages) + if let stepExpectSurfaces = step.expect?["surfaces"]?.objectValue { + try assertValidationResultSurfaces( + OrderedDictionary( + uniqueKeysWithValues: stepExpectSurfaces.map { ($0.key, $0.value) } + ), processor: processor, testName: testCase.name ) } } + + if let caseExpectSurfaces = testCase.expect?["surfaces"]?.objectValue { + try assertValidationResultSurfaces( + OrderedDictionary(uniqueKeysWithValues: caseExpectSurfaces.map { ($0.key, $0.value) }), + processor: processor, + testName: testCase.name + ) + } } private func assertValidationResultSurfaces( _ expectSurfaces: OrderedDictionary, processor: MessageProcessor, testName: String - ) { + ) throws { for (surfaceID, expectedSurface) in expectSurfaces { + if let surfObj = expectedSurface.objectValue { + try ConformanceTestHelper.assertKnownKeys( + surfObj, + Self.knownValidationResultSurfaceKeys, + "validation_result.yaml expect.surfaces.\(surfaceID) in '\(testName)'", + allowMetadata: false + ) + } guard let surface = processor.surfaceGroupModel[surfaceID] else { Issue.record("[\(testName)] Expected surface '\(surfaceID)' to exist") continue @@ -524,7 +695,10 @@ struct BasicCatalogConformanceTests { let node = surface.findNode(id: nodeID) #expect( node?.isValid == expectedBool, - "[\(testName)] Expected node \(nodeID).isValid == \(expectedBool), got \(String(describing: node?.isValid))" + """ + [\(testName)] Expected node \(nodeID).isValid == \(expectedBool), \ + got \(String(describing: node?.isValid)) + """ ) } } @@ -541,7 +715,10 @@ struct BasicCatalogConformanceTests { let node = surface.findNode(id: nodeID) #expect( node?.validationErrors == expectedStrings, - "[\(testName)] Expected node \(nodeID).validationErrors == \(expectedStrings), got \(String(describing: node?.validationErrors))" + """ + [\(testName)] Expected node \(nodeID).validationErrors == \(expectedStrings), \ + got \(String(describing: node?.validationErrors)) + """ ) } } @@ -557,7 +734,10 @@ struct BasicCatalogConformanceTests { let actualResults = surface.findNode(id: nodeID)?.validationResults ?? [] #expect( actualResults.count == expectedArray.count, - "[\(testName)] Expected \(expectedArray.count) validationResults on \(nodeID), got \(actualResults.count)" + """ + [\(testName)] Expected \(expectedArray.count) validationResults on \(nodeID), \ + got \(actualResults.count) + """ ) for (actualRes, expectedObjVal) in zip(actualResults, expectedArray) { assertSingleValidationResultMatches( @@ -612,7 +792,10 @@ struct BasicCatalogConformanceTests { if let expSeverity = expected["severity"]?.stringValue { #expect( actual.severity?.rawValue == expSeverity, - "[\(testName)] \(nodeID) severity mismatch: expected \(expSeverity), got \(String(describing: actual.severity?.rawValue))" + """ + [\(testName)] \(nodeID) severity mismatch: expected \(expSeverity), \ + got \(String(describing: actual.severity?.rawValue)) + """ ) } } diff --git a/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift index a720f72ac2..57ee80381a 100644 --- a/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift @@ -27,22 +27,63 @@ private struct LoadedCatalogInfo { let resolvedProtocolVersion: String } -/// Runs the shared `conformance/core/catalog.yaml` suite. +/// Runs the shared `conformance/core/catalog.yaml` suite (non-`derive` actions). struct CatalogConformanceTests { + static let supportedActions: Set = [ + "from_json", + "round_trip", + "to_json", + "validation_schema", + ] + + private static let knownCaseKeys: Set = [ + "action", + "builtinCatalog", + "catalog", + "catalogId", + "catalogPath", + "expect", + "expectError", + "expectFile", + "protocolVersion", + ] + + private static let knownFromJSONExpectKeys: Set = [ + "catalogId", + "componentMetadata", + "components", + "functions", + "invalidComponents", + "matchesBuiltinCatalog", + "protocolVersion", + "selfContained", + "validComponents", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + ] + @Test func catalogConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/catalog.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases("core/catalog.yaml", actions: Self.supportedActions) #expect(!cases.isEmpty, "core/catalog.yaml should hold test cases") - var executed = 0 - for testCase in cases { + for conformanceCase in cases { + let testCase = conformanceCase.rawDictionary let version = testCase["protocolVersion"] as? String - if version == "v0.8" { continue } - let action = testCase["action"] as? String ?? "from_json" - try dispatchCatalogCase(testCase, action: action) - executed += 1 + let skippedReason = + version == "v0.8" ? "protocol version v0.8 is not supported by A2UICore" : nil + try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownCaseKeys, + "catalog.yaml case '\(conformanceCase.name)'" + ) + let action = testCase["action"] as? String ?? "" + try dispatchCatalogCase(testCase, action: action) + } } - #expect(executed > 0, "Should execute catalog conformance cases") } private func dispatchCatalogCase(_ testCase: [String: Any], action: String) throws { @@ -56,7 +97,7 @@ struct CatalogConformanceTests { case "validation_schema": try runValidationSchemaCase(testCase) default: - break + throw UnknownConformanceKeyError("Unknown catalog.yaml action '\(action)'") } } @@ -64,6 +105,14 @@ struct CatalogConformanceTests { private func runFromJSONCase(_ testCase: [String: Any]) throws { let name = testCase["name"] as? String ?? "" + if let errDict = (testCase["expectError"] ?? testCase["expect_error"]) as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + errDict, + Self.knownExpectErrorKeys, + "catalog.yaml expectError in '\(name)'", + allowMetadata: false + ) + } let expectError = ConformanceTestHelper.parseExpectError( testCase["expectError"] ?? testCase["expect_error"] ) @@ -73,6 +122,12 @@ struct CatalogConformanceTests { } let loaded = try loadCatalogFromTestCase(testCase) if let expect = testCase["expect"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expect, + Self.knownFromJSONExpectKeys, + "catalog.yaml expect in '\(name)'", + allowMetadata: false + ) try assertFromJSONExpectations(loaded, expect: expect, caseName: name) } } @@ -270,7 +325,10 @@ struct CatalogConformanceTests { if let obj = node.objectValue { if let ref = obj["$ref"]?.stringValue, !isSupportedV10Ref(ref) { throw A2UICatalogError( - "Unsupported external $ref '\(ref)' at \(pointer)/$ref: a v1.0 catalog may only reference its own components and functions (#/...) or common_types.json." + """ + Unsupported external $ref '\(ref)' at \(pointer)/$ref: a v1.0 catalog may only \ + reference its own components and functions (#/...) or common_types.json. + """ ) } for (key, value) in obj { @@ -491,7 +549,8 @@ struct CatalogConformanceTests { if let expectedVersion = expect["protocolVersion"] as? String { let normalizedExpected = expectedVersion == "v1.0" ? "1.0" : expectedVersion #expect( - loaded.resolvedProtocolVersion == normalizedExpected || loaded.resolvedProtocolVersion == expectedVersion, + loaded.resolvedProtocolVersion == normalizedExpected + || loaded.resolvedProtocolVersion == expectedVersion, "[\(caseName)] protocolVersion mismatch" ) } @@ -544,7 +603,10 @@ struct CatalogConformanceTests { if let expectedNames = rawExpected as? [String] { #expect( Set(catalog.components.keys) == Set(expectedNames), - "[\(caseName)] Expected component names \(expectedNames), got \(Array(catalog.components.keys))" + """ + [\(caseName)] Expected component names \(expectedNames), \ + got \(Array(catalog.components.keys)) + """ ) return } @@ -568,10 +630,12 @@ struct CatalogConformanceTests { caseName: String ) { if let expParents = spec["allowedParents"] as? [String] { - #expect(comp.allowedParents == expParents, "[\(caseName)] \(comp.name) allowedParents mismatch") + #expect( + comp.allowedParents == expParents, "[\(caseName)] \(comp.name) allowedParents mismatch") } if let expChildren = spec["allowedChildren"] as? [String] { - #expect(comp.allowedChildren == expChildren, "[\(caseName)] \(comp.name) allowedChildren mismatch") + #expect( + comp.allowedChildren == expChildren, "[\(caseName)] \(comp.name) allowedChildren mismatch") } if let expDesc = spec["description"] as? String { let actualDesc = @@ -676,7 +740,9 @@ struct CatalogConformanceTests { if let invalidComps = expect["invalidComponents"] as? [[String: Any]] { for comp in invalidComps { let jsonComp = ConformanceTestHelper.toJSONValue(comp) - #expect(throws: (any Error).self, "[\(caseName)] Expected component \(comp) to fail validation") { + #expect( + throws: (any Error).self, "[\(caseName)] Expected component \(comp) to fail validation" + ) { try validator.validateComponent(jsonComp) } } diff --git a/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift index 9296f3546e..16b9581cbc 100644 --- a/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift @@ -25,9 +25,41 @@ import Testing /// check `absentKeys`. struct CatalogDeriveConformanceTests { /// Cases that check a catalog field the Swift catalog does not have. - private static let skippedCases: Set = [ + private static let skippedCases: [String: String] = [ // `derive` takes no instructions, since a Swift catalog has none. - "test_v10_catalog_derive_sets_instructions" + "test_v10_catalog_derive_sets_instructions": + "Swift Catalog does not store agent instructions" + ] + + private static let knownCaseKeys: Set = [ + "action", + "args", + "catalog", + "catalogPath", + "expect", + "expectError", + ] + + private static let knownArgsKeys: Set = [ + "catalogId", + "components", + "functions", + "instructions", + ] + + private static let knownExpectKeys: Set = [ + "absentKeys", + "catalogId", + "components", + "functions", + "instructions", + "protocolVersion", + "theme", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", ] private static let context = Context( @@ -36,25 +68,39 @@ struct CatalogDeriveConformanceTests { ) @Test func catalogDeriveConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/catalog.yaml") - let cases = ((rawYAML as? [[String: Any]]) ?? []).filter { - $0["action"] as? String == "derive" - } + let cases = ConformanceTestHelper.cases("core/catalog.yaml", action: "derive") #expect(!cases.isEmpty, "core/catalog.yaml should hold derive cases") + let caseNames = Set(cases.map(\.name)) + for skippedName in Self.skippedCases.keys { + #expect(caseNames.contains(skippedName), "Unused skip entry '\(skippedName)'") + } - var executed = 0 - for testCase in cases { - let name = testCase["name"] as? String ?? "" - if Self.skippedCases.contains(name) { continue } - try runDeriveCase(ConformanceTestHelper.toJSONValue(testCase), name: name) - executed += 1 + for conformanceCase in cases { + let skippedReason = Self.skippedCases[conformanceCase.name] + try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { + try runDeriveCase(conformanceCase.json, name: conformanceCase.name) + } } - #expect(executed > 0, "no derive case of core/catalog.yaml was executed") } private func runDeriveCase(_ testCase: JSONValue, name: String) throws { + if let obj = testCase.objectValue { + try ConformanceTestHelper.assertKnownKeys( + obj, + Self.knownCaseKeys, + "catalog.yaml derive case '\(name)'" + ) + } let source = try sourceCatalog(of: testCase) let args = testCase["args"] + if let argsObj = args?.objectValue { + try ConformanceTestHelper.assertKnownKeys( + argsObj, + Self.knownArgsKeys, + "catalog.yaml derive args in '\(name)'", + allowMetadata: false + ) + } let catalogID = try #require(args?["catalogId"]?.stringValue, "\(name): no catalogId") let components = try args?["components"]?.arrayValue.map { entries in try entries.map(componentEntry) @@ -64,6 +110,14 @@ struct CatalogDeriveConformanceTests { } if let expectError = testCase["expectError"] { + if let errObj = expectError.objectValue { + try ConformanceTestHelper.assertKnownKeys( + errObj, + Self.knownExpectErrorKeys, + "catalog.yaml derive expectError in '\(name)'", + allowMetadata: false + ) + } #expect(expectError["category"]?.stringValue == "CatalogError", "\(name)") let message = expectError["message"]?.stringValue ?? "" do { @@ -77,6 +131,14 @@ struct CatalogDeriveConformanceTests { let derived = try source.derive(id: catalogID, components: components, functions: functions) let expect = try #require(testCase["expect"], "\(name): no expect") + if let expectObj = expect.objectValue { + try ConformanceTestHelper.assertKnownKeys( + expectObj, + Self.knownExpectKeys, + "catalog.yaml derive expect in '\(name)'", + allowMetadata: false + ) + } if let expectedID = expect["catalogId"]?.stringValue { #expect(derived.id == expectedID, "\(name)") } diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift new file mode 100644 index 0000000000..d0991c93ef --- /dev/null +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift @@ -0,0 +1,153 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import Testing + +/// Appends executed conformance test case records to a JSON Lines manifest file when +/// `A2UI_CONFORMANCE_MANIFEST_DIR` is set in the process environment. +public enum ConformanceManifest { + /// Execution outcome recorded for a single conformance test case. + public enum Status: String, Sendable { + case passed + case failed + case skipped + case xfailed + } + + private static let harnessName = "swift-core" + private static let lock = NSLock() + + /// Records the execution status of `testCase` to the manifest file if configured. + public static func record( + _ testCase: ConformanceCase, + status: Status, + reason: String? = nil + ) { + record(suite: testCase.suite, name: testCase.name, status: status, reason: reason) + } + + /// Records the execution status of `(suite, name)` to the manifest file if configured. + public static func record( + suite: String, + name: String, + status: Status, + reason: String? = nil + ) { + guard + let manifestDir = ProcessInfo.processInfo.environment["A2UI_CONFORMANCE_MANIFEST_DIR"], + !manifestDir.isEmpty + else { + return + } + + var entry: [String: String] = [ + "harness": harnessName, + "suite": suite, + "case": name, + "status": status.rawValue, + ] + if let reason, !reason.isEmpty { + entry["reason"] = reason + } + + guard + let jsonData = try? JSONSerialization.data(withJSONObject: entry, options: [.sortedKeys]), + var lineData = String(data: jsonData, encoding: .utf8)?.data(using: .utf8) + else { + return + } + lineData.append(0x0A) + + lock.lock() + defer { lock.unlock() } + + let dirURL = URL(fileURLWithPath: manifestDir, isDirectory: true) + try? FileManager.default.createDirectory(at: dirURL, withIntermediateDirectories: true) + let pid = ProcessInfo.processInfo.processIdentifier + let fileURL = dirURL.appendingPathComponent("\(harnessName)-\(pid).jsonl") + + if FileManager.default.fileExists(atPath: fileURL.path) { + if let handle = try? FileHandle(forWritingTo: fileURL) { + defer { try? handle.close() } + _ = try? handle.seekToEnd() + try? handle.write(contentsOf: lineData) + } + } else { + try? lineData.write(to: fileURL, options: .atomic) + } + } + + /// Executes `body` for `testCase` and records the outcome in the conformance manifest. + public static func run( + _ testCase: ConformanceCase, + skippedReason: String? = nil, + knownIssueReason: String? = nil, + body: () throws -> Void + ) throws { + if let skippedReason { + record(testCase, status: .skipped, reason: skippedReason) + return + } + if let knownIssueReason { + withKnownIssue(Comment(rawValue: knownIssueReason)) { + do { + try body() + } catch { + Issue.record(error) + } + } + record(testCase, status: .xfailed, reason: knownIssueReason) + return + } + do { + try body() + record(testCase, status: .passed) + } catch { + record(testCase, status: .failed, reason: String(describing: error)) + throw error + } + } + + /// Executes an asynchronous `body` for `testCase` and records the outcome in the manifest. + public static func runAsync( + _ testCase: ConformanceCase, + skippedReason: String? = nil, + knownIssueReason: String? = nil, + body: () async throws -> Void + ) async throws { + if let skippedReason { + record(testCase, status: .skipped, reason: skippedReason) + return + } + if let knownIssueReason { + await withKnownIssue(Comment(rawValue: knownIssueReason)) { + do { + try await body() + } catch { + Issue.record(error) + } + } + record(testCase, status: .xfailed, reason: knownIssueReason) + return + } + do { + try await body() + record(testCase, status: .passed) + } catch { + record(testCase, status: .failed, reason: String(describing: error)) + throw error + } + } +} diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift index 53c30a008d..0dc4544455 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift @@ -137,53 +137,68 @@ public enum ConformanceTestHelper { /// specification version. public static func buildCatalogs(for testCase: ConformanceTestCase) throws -> [AnyCatalog] { if let inlineCatalog = testCase.inlineCatalog { - let version = - testCase.protocolVersion - ?? inlineCatalog["protocolVersion"]?.stringValue - ?? inlineCatalog["protocol_version"]?.stringValue - ?? "v0.9" - let commonTypes = try? commonTypesSchema( - forProtocolVersion: version + return buildInlineCatalogs( + inlineCatalog: inlineCatalog, + protocolVersion: testCase.protocolVersion ) - var catalogs: [AnyCatalog] = [] - if inlineCatalog["components"] != nil || inlineCatalog["functions"] != nil - || inlineCatalog["theme"] != nil - { - catalogs.append( - buildCatalog( - catalogSchema: inlineCatalog, - commonTypes: commonTypes, - protocolVersion: version - ) - ) - } - if inlineCatalog["components"] == nil { - let basicVersion: A2UIProtocolVersion = - (version.hasPrefix("v1") || version.hasPrefix("1")) ? .v10 : .v091 - catalogs.append(contentsOf: basicCatalogs(for: basicVersion)) - } - return catalogs } return try testCase.catalogPaths.map { path in - if path == "catalogs/basic/v1/catalog.json" { - return BasicCatalog.makeCatalog(version: .v10) - } - let fallbackVersion: String? - if path.contains("/v0_9_1/") { - fallbackVersion = "v0.9.1" - } else if path.contains("/v0_9/") { - fallbackVersion = "v0.9" - } else if path.contains("/v0_8/") { - fallbackVersion = "v0.8" - } else { - fallbackVersion = testCase.protocolVersion - } - return buildCatalog( - catalogSchema: try loadRepositoryJSON(path: path), - commonTypes: try commonTypesSchema(forCatalogPath: path), - protocolVersion: fallbackVersion + try buildCatalogFromPath(path, defaultProtocolVersion: testCase.protocolVersion) + } + } + + private static func buildInlineCatalogs( + inlineCatalog: JSONValue, + protocolVersion: String? + ) -> [AnyCatalog] { + let version = + protocolVersion + ?? inlineCatalog["protocolVersion"]?.stringValue + ?? inlineCatalog["protocol_version"]?.stringValue + ?? "v0.9" + let commonTypes = try? commonTypesSchema(forProtocolVersion: version) + var catalogs: [AnyCatalog] = [] + if inlineCatalog["components"] != nil || inlineCatalog["functions"] != nil + || inlineCatalog["theme"] != nil + { + catalogs.append( + buildCatalog( + catalogSchema: inlineCatalog, + commonTypes: commonTypes, + protocolVersion: version + ) ) } + if inlineCatalog["components"] == nil { + let basicVersion: A2UIProtocolVersion = + (version.hasPrefix("v1") || version.hasPrefix("1")) ? .v10 : .v091 + catalogs.append(contentsOf: basicCatalogs(for: basicVersion)) + } + return catalogs + } + + private static func buildCatalogFromPath( + _ path: String, + defaultProtocolVersion: String? + ) throws -> AnyCatalog { + if path == "catalogs/basic/v1/catalog.json" { + return BasicCatalog.makeCatalog(version: .v10) + } + let fallbackVersion: String? + if path.contains("/v0_9_1/") { + fallbackVersion = "v0.9.1" + } else if path.contains("/v0_9/") { + fallbackVersion = "v0.9" + } else if path.contains("/v0_8/") { + fallbackVersion = "v0.8" + } else { + fallbackVersion = defaultProtocolVersion + } + return buildCatalog( + catalogSchema: try loadRepositoryJSON(path: path), + commonTypes: try commonTypesSchema(forCatalogPath: path), + protocolVersion: fallbackVersion + ) } /// Returns the basic catalog for the given protocol version with `id: "basic"` @@ -694,16 +709,18 @@ public enum ConformanceTestHelper { let stepDicts = (dictionary["steps"] as? [[String: Any]]) ?? (dictionary["validate"] as? [[String: Any]]) - // Explicit steps carry only their own expectations. The case's `expectError` and `expect` - // apply to the final step, which each runner checks against `ConformanceTestCase`. if let stepDicts { - return stepDicts.map { stepDictionary in + let lastIndex = stepDicts.count - 1 + return stepDicts.enumerated().map { index, stepDictionary in let stepPayload = (stepDictionary["payload"] ?? stepDictionary["messages"]).map { toJSONValue($0) } + let fallbackError = index == lastIndex ? defaultExpectError : nil let stepError = parseExpectError(stepDictionary["expect_error"] ?? stepDictionary["expectError"]) - let stepExpect = (stepDictionary["expect"] as? [String: Any]).map { toJSONValue($0) } + ?? fallbackError + let stepExpect = + (stepDictionary["expect"] as? [String: Any]).map { toJSONValue($0) } ?? defaultExpect return ConformanceStep(payload: stepPayload, expectError: stepError, expect: stepExpect) } } @@ -1076,197 +1093,258 @@ public struct ConformanceExpectError: Sendable { public typealias ConformanceExpectRendererError = ConformanceExpectError.RendererError +/// A single conformance case loaded from a shared YAML suite for parameterized test execution. +public struct ConformanceCase: Sendable, CustomTestStringConvertible { + public let suite: String + public let name: String + public let json: JSONValue + + public init(suite: String, name: String, json: JSONValue) { + self.suite = suite + self.name = name + self.json = json + } + + public var testDescription: String { + "\(suite)::\(name)" + } + + public var rawDictionary: [String: Any] { + ConformanceTestHelper.rawCaseDictionary(suite: suite, name: name) + ?? (ConformanceTestHelper.fromJSONValue(json) as? [String: Any]) + ?? [:] + } + + public func parsedTestCase() -> ConformanceTestCase? { + ConformanceTestHelper.parseTestCases(from: [rawDictionary]).first + } +} + +/// Thrown when a conformance YAML dictionary contains an unrecognized key or dispatch value. +public struct UnknownConformanceKeyError: Error, CustomStringConvertible, Sendable { + public let message: String + + public init(_ message: String) { + self.message = message + } + + public var description: String { message } +} + extension ConformanceTestHelper { - /// Runs a `validate` case that renders components on a surface: processes each step's payload, - /// checks the errors a step expects, and compares the resolved properties of the expected - /// components. - /// - /// The case-level `expectError` and `expect` apply to the final step only. - @MainActor - public static func runValidateCase(_ testCase: ConformanceTestCase) throws { - // Without a declared version, `PayloadValidator` and `MessageProcessor` infer it from the - // catalog and the surface. - let targetVersion = - testCase.protocolVersion - ?? testCase.catalogConfiguration?["protocolVersion"]?.stringValue - ?? testCase.catalogConfiguration?["protocol_version"]?.stringValue - let processor = MessageProcessor( - catalogs: try ConformanceTestHelper.buildCatalogs(for: testCase), - validationConfig: ValidationConfig(targetVersion: targetVersion) - ) - // Expression errors raised while resolving nodes reach the surface as `EXPRESSION_ERROR` - // rather than being thrown out of `processMessages`. - var capturedErrors: [RendererError] = [] - processor.outboundListener = { message in - MainActor.assumeIsolated { - if case .error(let error) = message { - capturedErrors.append(error) - } - } - } + /// Maps each supported `conformance/core/*.yaml` suite to the actions and test files that run it. + public static let coreSuiteActions: [String: [String: String]] = [ + "core/accessibility.yaml": [ + "accessibility_check": "AccessibilityConformanceTests.swift" + ], + "core/actions.yaml": [ + "dispatch_action": "ActionsConformanceTests.swift" + ], + "core/catalog.yaml": [ + "derive": "CatalogDeriveConformanceTests.swift", + "from_json": "CatalogConformanceTests.swift", + "round_trip": "CatalogConformanceTests.swift", + "to_json": "CatalogConformanceTests.swift", + "validation_schema": "CatalogConformanceTests.swift", + ], + "core/composition_constraints.yaml": [ + "process_messages": "ValidatorConformanceTests.swift", + "validate": "ValidatorConformanceTests.swift", + ], + "core/data_context.yaml": [ + "resolve_path": "DataModelPointerConformanceTests.swift" + ], + "core/data_deletion.yaml": [ + "validate": "DataModelPointerConformanceTests.swift" + ], + "core/data_model.yaml": [ + "data_model": "DataModelConformanceTests.swift" + ], + "core/data_model_pointers.yaml": [ + "validate": "DataModelPointerConformanceTests.swift" + ], + "core/error_responses.yaml": [ + "validate": "ValidatorConformanceTests.swift" + ], + "core/expressions.yaml": [ + "parse_expression_template": "ExpressionsConformanceTests.swift", + "validate": "ExpressionsConformanceTests.swift", + ], + "core/functions.yaml": [ + "evaluate_function": "BasicCatalogConformanceTests.swift", + "validate": "BasicCatalogConformanceTests.swift", + ], + "core/index_function.yaml": [ + "validate": "BasicCatalogConformanceTests.swift" + ], + "core/message_processor_scaling.yaml": [ + "measure_growth": "MessageProcessorScalingConformanceTests.swift" + ], + "core/message_processor_v0_9.yaml": [ + "get_renderer_capabilities": "MessageProcessorConformanceTests.swift", + "get_renderer_data_model": "MessageProcessorConformanceTests.swift", + "process_messages": "MessageProcessorConformanceTests.swift", + "resolve_path": "MessageProcessorConformanceTests.swift", + ], + "core/message_processor_v1_0.yaml": [ + "get_renderer_capabilities": "MessageProcessorConformanceTests.swift", + "get_renderer_data_model": "MessageProcessorConformanceTests.swift", + "process_messages": "MessageProcessorConformanceTests.swift", + "validate": "MessageProcessorConformanceTests.swift", + ], + "core/multi_catalog.yaml": [ + "select_catalog": "MultiCatalogConformanceTests.swift", + "validate": "MultiCatalogConformanceTests.swift", + ], + "core/node_resolution.yaml": [ + "resolve_nodes": "NodeResolutionConformanceTests.swift" + ], + "core/reserved_keys.yaml": [ + "process_messages": "ValidatorConformanceTests.swift", + "validate": "ValidatorConformanceTests.swift", + ], + "core/rpc_functions.yaml": [ + "handle_rpc": "RPCConformanceTests.swift" + ], + "core/validation_result.yaml": [ + "validate": "BasicCatalogConformanceTests.swift" + ], + "core/validator_v0_9.yaml": [ + "validate": "ValidatorConformanceTests.swift" + ], + "core/validator_v1_0.yaml": [ + "validate": "ValidatorConformanceTests.swift" + ], + ] - for (stepIndex, step) in testCase.steps.enumerated() { - guard let payload = step.payload else { - continue - } - capturedErrors.removeAll() - - let isFinalStep = stepIndex == testCase.steps.count - 1 - if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { - assertValidateStepFails( - payload: payload, - processor: processor, - expectedError: expectedError, - capturedErrors: { capturedErrors }, - context: "\(testCase.name) step \(stepIndex)" - ) - continue - } + /// Maps each unsupported `conformance/core/*.yaml` suite to the reason it is skipped in Swift. + public static let unsupportedCoreSuites: [String: String] = [ + "core/adapter_conformance.yaml": + "render_surface cases require the SwiftUI framework adapter harness", + "core/agent_to_renderer.yaml": + "agent_to_renderer_schema is not implemented in the Swift core harness", + "core/common_types.yaml": + "common_types_schema and validate_common_type are not implemented in the Swift core harness", + "core/message_processor_v0_8.yaml": + "protocol version v0.8 is not supported by A2UICore", + "core/validator_v0_8.yaml": + "protocol version v0.8 is not supported by A2UICore", + ] - try processor.processMessages(payload) - #expect( - capturedErrors.isEmpty, - "\(testCase.name): unexpected error at step \(stepIndex): \(capturedErrors)" - ) + /// Case-level documentation keys permitted in every conformance test case dictionary. + public static let metadataCaseKeys: Set = ["name", "description", "features"] - let expectedSurfaces = - (step.expect ?? (isFinalStep ? testCase.expect : nil))?["surfaces"]?.objectValue ?? [:] - for (surfaceID, expectedSurface) in expectedSurfaces { - let surface = try #require( - processor.surfaceGroupModel[surfaceID], - "\(testCase.name): expected surface '\(surfaceID)'" - ) - let expectedComponents = expectedSurface["components"]?.objectValue ?? [:] - for (componentID, expectedComponent) in expectedComponents { - let node = try #require( - surface.findNode(id: componentID), - "\(testCase.name): expected resolved node '\(componentID)'" - ) - try assertValidateNode( - node, - matches: expectedComponent, - context: "\(testCase.name): '\(componentID)'" - ) - } - } - } + private static let cacheLock = NSLock() + nonisolated(unsafe) private static var suiteCasesCache: [String: [ConformanceCase]] = [:] + nonisolated(unsafe) private static var rawCasesCache: [String: [String: [String: Any]]] = [:] + + /// Loads and caches all cases from `suite`, optionally filtered to a single `action`. + public static func cases(_ suite: String, action: String? = nil) -> [ConformanceCase] { + let all = loadAllSuiteCases(suite) + guard let action else { return all } + return all.filter { $0.json["action"]?.stringValue == action } } - /// Processes a `validate` step that must fail and checks the error against `expectedError`. - /// - /// A thrown error must be an `A2UIError` of the expected category. An error raised while - /// resolving nodes, such as an expression that fails to parse, is reported on the surface - /// instead of thrown; like the Dart runner, it counts as a `ValidationError`. - @MainActor - private static func assertValidateStepFails( - payload: JSONValue, - processor: MessageProcessor, - expectedError: ConformanceExpectError, - capturedErrors: () -> [RendererError], - context: String - ) { - var thrownError: (any Error)? - do { - try processor.processMessages(payload) - } catch { - thrownError = error - } - let reportedMessages = capturedErrors().map { error in - switch error { - case .validationFailed(let validationError): validationError.message - case .generic(let genericError): genericError.message - } - } + /// Loads and caches all cases from `suite` whose `action` is in `actions`. + public static func cases(_ suite: String, actions: Set) -> [ConformanceCase] { + let all = loadAllSuiteCases(suite) + return all.filter { actions.contains($0.json["action"]?.stringValue ?? "") } + } + + private static func loadAllSuiteCases(_ suite: String) -> [ConformanceCase] { + cacheLock.lock() + if let cached = suiteCasesCache[suite] { + cacheLock.unlock() + return cached + } + cacheLock.unlock() + + guard let rawYAML = try? loadYAML(filename: suite) else { return [] } + let rawDicts: [[String: Any]] + if let array = rawYAML as? [[String: Any]] { + rawDicts = array + } else if let dict = rawYAML as? [String: Any], let list = dict["cases"] as? [[String: Any]] { + rawDicts = list + } else { + rawDicts = [] + } + + var builtCases: [ConformanceCase] = [] + var rawByName: [String: [String: Any]] = [:] + for dict in rawDicts { + guard let name = dict["name"] as? String else { continue } + rawByName[name] = dict + builtCases.append(ConformanceCase(suite: suite, name: name, json: toJSONValue(dict))) + } + + cacheLock.lock() + suiteCasesCache[suite] = builtCases + rawCasesCache[suite] = rawByName + cacheLock.unlock() + return builtCases + } - var errorMessages: [String] = [] - if let thrownError { - guard let a2uiError = thrownError as? any A2UIError else { - Issue.record("\(context): unexpected non-A2UI error: \(thrownError)") - return + fileprivate static func rawCaseDictionary(suite: String, name: String) -> [String: Any]? { + _ = loadAllSuiteCases(suite) + cacheLock.lock() + defer { cacheLock.unlock() } + return rawCasesCache[suite]?[name] + } + + /// Converts a `JSONValue` back into Foundation-style values (`[String: Any]`, `[Any]`, scalars). + public static func fromJSONValue(_ value: JSONValue) -> Any { + switch value { + case .object(let ordered): + var dict: [String: Any] = [:] + for (key, val) in ordered { + dict[key] = fromJSONValue(val) } - errorMessages.append(a2uiError.message) - if let category = expectedError.category { - #expect( - errorMatchesCategory(a2uiError, category: category), - "\(context): expected a \(category), got \(type(of: a2uiError)): \(a2uiError.message)" - ) + return dict + case .array(let items): + return items.map { fromJSONValue($0) } + case .string(let str): + return str + case .boolean(let bool): + return bool + case .number(let double): + if let exactInt = Int(exactly: double) { + return exactInt } - } else if !reportedMessages.isEmpty, let category = expectedError.category { - #expect( - category == "ValidationError", - "\(context): expected a thrown \(category), got surface errors \(reportedMessages)" - ) - } - errorMessages += reportedMessages - - #expect(!errorMessages.isEmpty, "\(context): expected an error") - if let expectedMessage = expectedError.message { - #expect( - errorMessages.contains { messageMatches($0, expected: expectedMessage) }, - "\(context): messages \(errorMessages) do not match '\(expectedMessage)'" - ) + return double + case .null: + return NSNull() } } - /// Whether `error` belongs to a conformance `expectError.category`, as in the Dart runner. - private static func errorMatchesCategory(_ error: any A2UIError, category: String) -> Bool { - switch category { - case "ValidationError": - error is A2UIValidationError || error is A2UIIntegrityError - || error is A2UIRecursionError || error is A2UICatalogError - case "CatalogError": error is A2UICatalogError - case "IntegrityError": error is A2UIIntegrityError - case "RecursionError": error is A2UIRecursionError - case "DataError": error is A2UIDataError - case "ParseError": error is A2UIParseError - default: true + /// Verifies that every key in `dict` belongs to `allowed` (or `metadataCaseKeys` when + /// `allowMetadata` is true), recording an issue and throwing on the first unknown key. + public static func assertKnownKeys( + _ dict: [String: Any], + _ allowed: Set, + _ location: String, + allowMetadata: Bool = true + ) throws { + let permitted = allowMetadata ? allowed.union(metadataCaseKeys) : allowed + for key in dict.keys.sorted() where !permitted.contains(key) { + let msg = "Unknown \(location) key '\(key)' (allowed: \(permitted.sorted()))" + Issue.record(Comment(rawValue: msg)) + throw UnknownConformanceKeyError(msg) } } - /// Checks a resolved node against the expected component of a `validate` case: its type and - /// the text of each listed string property. - @MainActor - private static func assertValidateNode( - _ node: A2UICore.Node, - matches expectedComponent: JSONValue, - context: String + /// Verifies that every key in `dict` belongs to `allowed` (or `metadataCaseKeys` when + /// `allowMetadata` is true), recording an issue and throwing on the first unknown key. + public static func assertKnownKeys( + _ dict: OrderedDictionary, + _ allowed: Set, + _ location: String, + allowMetadata: Bool = true ) throws { - if let expectedType = expectedComponent["component"]?.stringValue { - #expect( - node.type == expectedType, - "\(context) has type '\(node.type)', expected '\(expectedType)'" - ) - } - for (property, expectedValue) in expectedComponent.objectValue ?? [:] - where property != "component" { - let binding = try #require( - node.properties[property] as? DataBinding, - "\(context).\(property) is not a string binding" - ) - // A `Text` renders a scalar result as its JSON text. - let expectedText: String? - switch expectedValue { - case .string(let string): - expectedText = string - case .boolean(let boolean): - expectedText = String(boolean) - case .integer(let integer): - expectedText = String(integer) - case .number(let number): - expectedText = Int(exactly: number).map(String.init) ?? String(number) - case .null: - expectedText = nil - case .array, .object: - Issue.record("\(context).\(property): unsupported expected value \(expectedValue)") - continue - } - #expect( - binding.value == expectedText, - """ - \(context).\(property) is \(String(describing: binding.value)), \ - expected \(String(describing: expectedText)) - """ - ) + let permitted = allowMetadata ? allowed.union(metadataCaseKeys) : allowed + for key in dict.keys.sorted() where !permitted.contains(key) { + let msg = "Unknown \(location) key '\(key)' (allowed: \(permitted.sorted()))" + Issue.record(Comment(rawValue: msg)) + throw UnknownConformanceKeyError(msg) } } diff --git a/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift index d098502a00..bfcd621597 100644 --- a/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift @@ -19,9 +19,34 @@ import Foundation import OrderedJSON import Testing -/// Runs the shared `conformance/core/data_model.yaml` and `conformance/core/data_deletion.yaml` suites. +/// Runs the shared `conformance/core/data_model.yaml` suite. @MainActor struct DataModelConformanceTests { + private static let knownCaseKeys: Set = [ + "action", + "catalog", + "initial", + "steps", + "watch", + ] + + private static let knownStepKeys: Set = [ + "expect", + "expect_absent", + "expect_error", + "expect_notified", + "expect_type", + "expect_values", + "op", + "path", + "value", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + ] + @MainActor private final class Observer { let path: String @@ -44,82 +69,145 @@ struct DataModelConformanceTests { } @Test func dataModelConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/data_model.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases("core/data_model.yaml") #expect(!cases.isEmpty, "core/data_model.yaml should hold test cases") - for testCase in cases { - let name = testCase["name"] as? String ?? "" - let steps = testCase["steps"] as? [[String: Any]] ?? [] + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runDataModelCase(conformanceCase.rawDictionary, name: conformanceCase.name) + } + } + } - let initial = - testCase["initial"].map { ConformanceTestHelper.toJSONValue($0) } ?? .object([:]) - let model = DataModel(initial: initial) + private func runDataModelCase(_ testCase: [String: Any], name: String) throws { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownCaseKeys, + "data_model.yaml case '\(name)'" + ) + let action = testCase["action"] as? String ?? "" + guard action == "data_model" else { + throw UnknownConformanceKeyError("[\(name)] Unknown data_model.yaml action '\(action)'") + } + let steps = testCase["steps"] as? [[String: Any]] ?? [] + let initial = testCase["initial"].map { ConformanceTestHelper.toJSONValue($0) } ?? .object([:]) + let model = DataModel(initial: initial) - var observers: [Observer] = [] - if let watchPaths = testCase["watch"] as? [String] { - for watchPath in watchPaths { - observers.append(try Observer(model: model, path: watchPath)) - } + var observers: [Observer] = [] + if let watchPaths = testCase["watch"] as? [String] { + for watchPath in watchPaths { + observers.append(try Observer(model: model, path: watchPath)) } + } - for (index, step) in steps.enumerated() { - for observer in observers { - observer.changeCount = 0 - } + for (index, step) in steps.enumerated() { + try runDataModelStep( + step, + index: index, + model: model, + observers: observers, + caseName: name + ) + } + } - let operation = step["op"] as? String ?? "" - let path = step["path"] as? String ?? "/" - let location = "\(name) step \(index) (\(operation) \(path))" - - if let expectError = step["expect_error"] as? [String: Any] { - let expectedSubstring = expectError["message"] as? String - do { - try applyOp(model: model, step: step, location: location) - Issue.record("\(location): expected DataError, but succeeded") - } catch let error as A2UIDataError { - if let expectedSubstring { - #expect( - error.message.contains(expectedSubstring), - "\(location): expected error containing '\(expectedSubstring)', got '\(error.message)'" - ) - } - } catch { - Issue.record("\(location): expected A2UIDataError, got \(error)") - } - continue - } + private func runDataModelStep( + _ step: [String: Any], + index: Int, + model: DataModel, + observers: [Observer], + caseName: String + ) throws { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownStepKeys, + "data_model.yaml step \(index) in '\(caseName)'", + allowMetadata: false + ) + for observer in observers { + observer.changeCount = 0 + } - try applyOp(model: model, step: step, location: location) - - if let expectedNotified = step["expect_notified"] as? [String] { - var actualNotified: [String] = [] - for observer in observers where observer.changeCount > 0 { - for _ in 0.. 0 { + for _ in 0.. = [ + "action", + "catalog", + "expect", + "steps", + ] + + private static let knownPayloadStepKeys: Set = [ + "expectError", + "payload", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + ] + + private static let knownDataContextCaseKeys: Set = [ + "action", + "args", + "catalog", + "expect", + ] + + private static let knownDataContextArgsKeys: Set = [ + "contextPath", + "path", + ] + @Test func dataModelPointersConformance() throws { try runPayloadDataModelSuite(filename: "core/data_model_pointers.yaml") } @@ -30,89 +59,173 @@ struct DataModelPointerConformanceTests { } @Test func dataContextPathResolutionConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/data_context.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases("core/data_context.yaml") #expect(!cases.isEmpty, "core/data_context.yaml should hold test cases") - for testCase in cases { - let name = testCase["name"] as? String ?? "" - let args = testCase["args"] as? [String: Any] ?? [:] - let path = args["path"] as? String ?? "" - let contextPath = (args["contextPath"] as? String) ?? (args["context_path"] as? String) - let expected = testCase["expect"] as? String ?? "" - let actual = JSONValue.absolutePath(for: path, in: contextPath) - #expect(actual == expected, "\(name): got \(actual), expected \(expected)") + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runDataContextCase(conformanceCase.rawDictionary, name: conformanceCase.name) + } } } + private func runDataContextCase(_ testCase: [String: Any], name: String) throws { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownDataContextCaseKeys, + "data_context.yaml case '\(name)'" + ) + let action = testCase["action"] as? String ?? "" + guard action == "resolve_path" else { + throw UnknownConformanceKeyError("[\(name)] Unknown data_context.yaml action '\(action)'") + } + let args = testCase["args"] as? [String: Any] ?? [:] + try ConformanceTestHelper.assertKnownKeys( + args, + Self.knownDataContextArgsKeys, + "data_context.yaml args in '\(name)'", + allowMetadata: false + ) + let path = args["path"] as? String ?? "" + let contextPath = (args["contextPath"] as? String) ?? (args["context_path"] as? String) + let expected = testCase["expect"] as? String ?? "" + let actual = JSONValue.absolutePath(for: path, in: contextPath) + #expect(actual == expected, "\(name): got \(actual), expected \(expected)") + } + private func runPayloadDataModelSuite(filename: String) throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: filename) - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases(filename) #expect(!cases.isEmpty, "\(filename) should hold test cases") - for testCase in cases { - let name = testCase["name"] as? String ?? "" - let steps = testCase["steps"] as? [[String: Any]] ?? [] - var models: [String: DataModel] = [:] - - for (index, step) in steps.enumerated() { - let payload = step["payload"] as? [[String: Any]] ?? [] - let expectError = step["expectError"] as? [String: Any] - let location = "\(name) step \(index)" - - let applyStep = { - for msg in payload { - if let createSurface = msg["createSurface"] as? [String: Any] { - let surfaceID = createSurface["surfaceId"] as? String ?? "s1" - let initial = - createSurface["dataModel"].map { ConformanceTestHelper.toJSONValue($0) } - ?? .object([:]) - models[surfaceID] = DataModel(initial: initial) - } else if let updateDataModel = msg["updateDataModel"] as? [String: Any] { - let surfaceID = updateDataModel["surfaceId"] as? String ?? "s1" - let path = updateDataModel["path"] as? String ?? "/" - let value = updateDataModel["value"].flatMap { rawValue -> JSONValue? in - rawValue is NSNull ? nil : ConformanceTestHelper.toJSONValue(rawValue) - } - try models[surfaceID]?.setThrowing(path, value: value) - } - } - } + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runPayloadDataModelCase( + conformanceCase.rawDictionary, + name: conformanceCase.name, + suite: filename + ) + } + } + } - if let expectError { - let expectedSubstring = expectError["message"] as? String - do { - try applyStep() - Issue.record("\(location): expected DataError, but succeeded") - } catch let error as A2UIDataError { - if let expectedSubstring { - #expect( - error.message.contains(expectedSubstring), - "\(location): expected '\(expectedSubstring)', got '\(error.message)'" - ) - } - } - continue - } + private func runPayloadDataModelCase( + _ testCase: [String: Any], + name: String, + suite: String + ) throws { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownPayloadCaseKeys, + "\(suite) case '\(name)'" + ) + let action = testCase["action"] as? String ?? "" + guard action == "validate" else { + throw UnknownConformanceKeyError("[\(name)] Unknown \(suite) action '\(action)'") + } + let steps = testCase["steps"] as? [[String: Any]] ?? [] + var models: [String: DataModel] = [:] - try applyStep() + for (index, step) in steps.enumerated() { + try runPayloadStep(step, index: index, models: &models, name: name, suite: suite) + } + + if let expect = testCase["expect"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expect, + ["surfaces"], + "\(suite) expect in '\(name)'", + allowMetadata: false + ) + if let surfaces = expect["surfaces"] as? [String: Any] { + try assertExpectedSurfaceModels(surfaces, models: models, name: name, suite: suite) } + } + } - if let expect = testCase["expect"] as? [String: Any], - let surfaces = expect["surfaces"] as? [String: Any] - { - for (surfaceID, rawSurfaceExp) in surfaces { - guard let surfaceExp = rawSurfaceExp as? [String: Any], - let expectedDataModel = surfaceExp["dataModel"] - else { continue } - let actual = models[surfaceID]?.get("/") - let expectedJSON = ConformanceTestHelper.toJSONValue(expectedDataModel) + private func runPayloadStep( + _ step: [String: Any], + index: Int, + models: inout [String: DataModel], + name: String, + suite: String + ) throws { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownPayloadStepKeys, + "\(suite) step \(index) in '\(name)'", + allowMetadata: false + ) + let payload = step["payload"] as? [[String: Any]] ?? [] + let location = "\(name) step \(index)" + + if let expectError = step["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "\(suite) expectError at \(location)", + allowMetadata: false + ) + let expectedSubstring = expectError["message"] as? String + do { + try applyPayloadMessages(payload, to: &models) + Issue.record("\(location): expected DataError, but succeeded") + } catch let error as A2UIDataError { + if let expectedSubstring { #expect( - actual == expectedJSON, - "\(name) surface \(surfaceID): got \(String(describing: actual)), expected \(expectedJSON)" + error.message.contains(expectedSubstring), + "\(location): expected '\(expectedSubstring)', got '\(error.message)'" ) } } + return + } + + try applyPayloadMessages(payload, to: &models) + } + + private func applyPayloadMessages( + _ payload: [[String: Any]], + to models: inout [String: DataModel] + ) throws { + for msg in payload { + if let createSurface = msg["createSurface"] as? [String: Any] { + let surfaceID = createSurface["surfaceId"] as? String ?? "s1" + let initial = + createSurface["dataModel"].map { ConformanceTestHelper.toJSONValue($0) } + ?? .object([:]) + models[surfaceID] = DataModel(initial: initial) + } else if let updateDataModel = msg["updateDataModel"] as? [String: Any] { + let surfaceID = updateDataModel["surfaceId"] as? String ?? "s1" + let path = updateDataModel["path"] as? String ?? "/" + let value = updateDataModel["value"].flatMap { rawValue -> JSONValue? in + rawValue is NSNull ? nil : ConformanceTestHelper.toJSONValue(rawValue) + } + try models[surfaceID]?.setThrowing(path, value: value) + } + } + } + + private func assertExpectedSurfaceModels( + _ surfaces: [String: Any], + models: [String: DataModel], + name: String, + suite: String + ) throws { + for (surfaceID, rawSurfaceExp) in surfaces { + guard let surfaceExp = rawSurfaceExp as? [String: Any] else { continue } + try ConformanceTestHelper.assertKnownKeys( + surfaceExp, + ["dataModel"], + "\(suite) expect.surfaces.\(surfaceID) in '\(name)'", + allowMetadata: false + ) + guard let expectedDataModel = surfaceExp["dataModel"] else { continue } + let actual = models[surfaceID]?.get("/") + let expectedJSON = ConformanceTestHelper.toJSONValue(expectedDataModel) + #expect( + actual == expectedJSON, + "\(name) surface \(surfaceID): got \(String(describing: actual)), expected \(expectedJSON)" + ) } } } diff --git a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift index e1a500dd6b..6dc0583494 100644 --- a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift @@ -20,91 +20,277 @@ import Testing /// Runs the shared `conformance/core/expressions.yaml` suite. struct ExpressionsConformanceTests { + private static let knownParseCaseKeys: Set = [ + "action", + "expect", + "expect_error", + "input", + ] + + private static let knownValidateCaseKeys: Set = [ + "action", + "catalog", + "expect", + "steps", + ] + + private static let knownValidateStepKeys: Set = [ + "expectError", + "payload", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + ] + private let parser = ExpressionParser() @Test func expressionsConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/expressions.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] - #expect(!cases.isEmpty, "core/expressions.yaml should hold test cases") + let cases = ConformanceTestHelper.cases( + "core/expressions.yaml", + action: "parse_expression_template" + ) + #expect(!cases.isEmpty, "core/expressions.yaml should hold parse_expression_template cases") - var executed = 0 - - for testCase in cases { - guard let action = testCase["action"] as? String, action == "parse_expression_template" else { - continue + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runParseExpressionCase(conformanceCase.rawDictionary, name: conformanceCase.name) } + } + } - guard let name = testCase["name"] as? String, - let input = testCase["input"] as? String - else { - continue - } + private func runParseExpressionCase(_ testCase: [String: Any], name: String) throws { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownParseCaseKeys, + "expressions.yaml parse case '\(name)'" + ) + let input = try #require(testCase["input"] as? String, "\(name): missing input") - executed += 1 + if let expectError = (testCase["expect_error"] ?? testCase["expectError"]) as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "expressions.yaml expect_error in '\(name)'", + allowMetadata: false + ) + assertParseExpressionThrows(input: input, expectError: expectError, name: name) + } else if let expectedRaw = testCase["expect"] as? [Any] { + let actual = try parser.parse(input) + let joinedActual = joinLiterals(actual) + let expectedValues = expectedRaw.map { ConformanceTestHelper.toJSONValue($0) } + #expect( + looselyEqual(joinedActual, expectedValues), + "\(name) mismatch: got \(joinedActual), expected \(expectedValues)" + ) + } else { + Issue.record("\(name): test case does not define 'expect' or 'expect_error'") + } + } - if let expectError = (testCase["expect_error"] ?? testCase["expectError"]) as? [String: Any] { - let expectedMessage = expectError["message"] as? String - let expectedCategory = expectError["category"] as? String + private func assertParseExpressionThrows( + input: String, + expectError: [String: Any], + name: String + ) { + let expectedMessage = expectError["message"] as? String + let expectedCategory = expectError["category"] as? String - do { - _ = try parser.parse(input) - Issue.record("\(name): expected an error for input: \(input)") - } catch let error as FunctionError { - if let expectedCategory { - #expect(!expectedCategory.isEmpty, "\(name): expected category '\(expectedCategory)'") - } - if let expectedMessage { - switch error { - case .executionFailed(_, let message): - let matches = - message.localizedCaseInsensitiveContains(expectedMessage) - || ((try? NSRegularExpression(pattern: expectedMessage).firstMatch( - in: message, - range: NSRange(message.startIndex..., in: message) - )) != nil) - #expect( - matches, - "\(name): message '\(message)' does not match pattern '\(expectedMessage)'" + do { + _ = try parser.parse(input) + Issue.record("\(name): expected an error for input: \(input)") + } catch let error as FunctionError { + if let expectedCategory { + #expect(!expectedCategory.isEmpty, "\(name): expected category '\(expectedCategory)'") + } + if let expectedMessage { + switch error { + case .executionFailed(_, let message): + let matches = + message.localizedCaseInsensitiveContains(expectedMessage) + || ((try? NSRegularExpression(pattern: expectedMessage).firstMatch( + in: message, + range: NSRange(message.startIndex..., in: message) + )) != nil) + #expect( + matches, + "\(name): message '\(message)' does not match pattern '\(expectedMessage)'" + ) + default: + Issue.record("\(name): expected executionFailed error, but got \(error)") + } + } + } catch { + Issue.record("\(name): threw unexpected error \(error)") + } + } + + /// Runs the `validate` cases of `core/expressions.yaml`, which render `formatString` templates + /// on a v1.0 surface and compare the resolved component properties. + @MainActor + @Test func expressionsValidateConformance() throws { + let cases = ConformanceTestHelper.cases("core/expressions.yaml", action: "validate") + #expect(!cases.isEmpty, "core/expressions.yaml should hold validate cases") + + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + let rawDict = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + rawDict, + Self.knownValidateCaseKeys, + "expressions.yaml validate case '\(conformanceCase.name)'" + ) + if let steps = rawDict["steps"] as? [[String: Any]] { + for step in steps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownValidateStepKeys, + "expressions.yaml step in '\(conformanceCase.name)'", + allowMetadata: false + ) + if let errDict = step["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + errDict, + Self.knownExpectErrorKeys, + "expressions.yaml step.expectError in '\(conformanceCase.name)'", + allowMetadata: false ) - default: - Issue.record("\(name): expected executionFailed error, but got \(error)") } } - } catch { - Issue.record("\(name): threw unexpected error \(error)") } - } else if let expectedRaw = testCase["expect"] as? [Any] { - let actual = try parser.parse(input) - let joinedActual = joinLiterals(actual) - let expectedValues = expectedRaw.map { ConformanceTestHelper.toJSONValue($0) } - - #expect( - looselyEqual(joinedActual, expectedValues), - "\(name) mismatch: got \(joinedActual), expected \(expectedValues)" + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse validate case" ) - } else { - Issue.record("\(name): test case does not define 'expect' or 'expect_error'") + try runValidateCase(testCase) } } - - #expect(executed > 0, "No cases of expressions.yaml could be executed") } - /// Runs the `validate` cases of `core/expressions.yaml`, which render `formatString` templates - /// on a v1.0 surface and compare the resolved component properties. @MainActor - @Test func expressionsValidateConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/expressions.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYAML) + private func runValidateCase(_ testCase: ConformanceTestCase) throws { + let processor = MessageProcessor( + catalogs: try ConformanceTestHelper.buildCatalogs(for: testCase), + validationConfig: ValidationConfig(targetVersion: "1.0") + ) + // Expression errors raised while resolving nodes reach the surface as `EXPRESSION_ERROR` + // rather than being thrown out of `processMessages`. + var capturedErrors: [RendererError] = [] + processor.outboundListener = { message in + MainActor.assumeIsolated { + if case .error(let error) = message { + capturedErrors.append(error) + } + } + } + + for (stepIndex, step) in testCase.steps.enumerated() { + guard let payload = step.payload else { continue } + capturedErrors.removeAll() - var executed = 0 + let isFinalStep = stepIndex == testCase.steps.count - 1 + if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { + assertValidateStepError( + payload: payload, + processor: processor, + capturedErrors: &capturedErrors, + expectedError: expectedError, + testName: testCase.name, + stepIndex: stepIndex + ) + continue + } + + try processor.processMessages(payload) + #expect( + capturedErrors.isEmpty, + "\(testCase.name): unexpected error at step \(stepIndex): \(capturedErrors)" + ) + + let expectedSurfaces = (step.expect ?? testCase.expect)?["surfaces"]?.objectValue ?? [:] + try assertValidateSurfaces( + OrderedDictionary(uniqueKeysWithValues: expectedSurfaces.map { ($0.key, $0.value) }), + on: processor, + testName: testCase.name + ) + } + } - for testCase in testCases where testCase.action == "validate" { - executed += 1 - try ConformanceTestHelper.runValidateCase(testCase) + @MainActor + private func assertValidateStepError( + payload: JSONValue, + processor: MessageProcessor, + capturedErrors: inout [RendererError], + expectedError: ConformanceExpectError, + testName: String, + stepIndex: Int + ) { + var errorMessages: [String] = [] + do { + try processor.processMessages(payload) + } catch { + let message = (error as? (any A2UIError))?.message ?? error.localizedDescription + errorMessages.append(message) + } + for error in capturedErrors { + switch error { + case .validationFailed(let validationError): + errorMessages.append(validationError.message) + case .generic(let genericError): + errorMessages.append(genericError.message) + } } + #expect( + !errorMessages.isEmpty, + "\(testName): expected an error at step \(stepIndex)" + ) + if let expectedMessage = expectedError.message { + #expect( + errorMessages.contains { $0.localizedCaseInsensitiveContains(expectedMessage) }, + "\(testName): messages \(errorMessages) do not contain '\(expectedMessage)'" + ) + } + } - #expect(executed > 0, "No validate case of expressions.yaml could be executed") + @MainActor + private func assertValidateSurfaces( + _ expectedSurfaces: OrderedDictionary, + on processor: MessageProcessor, + testName: String + ) throws { + for (surfaceID, expectedSurface) in expectedSurfaces { + if let surfObj = expectedSurface.objectValue { + try ConformanceTestHelper.assertKnownKeys( + surfObj, + ["components"], + "expressions.yaml expect.surfaces.\(surfaceID) in '\(testName)'", + allowMetadata: false + ) + } + let surface = try #require( + processor.surfaceGroupModel[surfaceID], + "\(testName): expected surface '\(surfaceID)'" + ) + let expectedComponents = expectedSurface["components"]?.objectValue ?? [:] + for (componentID, expectedComponent) in expectedComponents { + let node = try #require( + surface.findNode(id: componentID), + "\(testName): expected resolved node '\(componentID)'" + ) + for (property, expectedValue) in expectedComponent.objectValue ?? [:] + where property != "component" { + let binding = try #require( + node.properties[property] as? DataBinding, + "\(testName): '\(componentID).\(property)' is not a string binding" + ) + #expect( + binding.value == expectedValue.stringValue, + "\(testName): '\(componentID).\(property)' is '\(String(describing: binding.value))'" + ) + } + } + } } /// Joins adjacent literal strings and drops empty string literals. diff --git a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift index d1bc150da1..4b3d599187 100644 --- a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift @@ -48,10 +48,68 @@ struct MessageProcessorConformanceTests { /// Cases of the optional agent_renderer_app feature /// (blueprints/features/agent_renderer_app.blueprint.md), which passes a validation config to /// one call. This SDK has no agent, so it does not implement the feature. - private static let skippedCases: Set = [ - "test_v10_call_validation_config_relaxes_strict_processor", - "test_v10_call_validation_config_tightens_permissive_processor", - "test_v10_call_validation_config_leaves_out_rules_as_strict", + private static let skippedCases: [String: String] = [ + "test_v10_call_validation_config_relaxes_strict_processor": + "per-call validationConfig is part of the optional agent_renderer_app feature", + "test_v10_call_validation_config_tightens_permissive_processor": + "per-call validationConfig is part of the optional agent_renderer_app feature", + "test_v10_call_validation_config_leaves_out_rules_as_strict": + "per-call validationConfig is part of the optional agent_renderer_app feature", + ] + + private static let knownCaseKeys: Set = [ + "action", + "args", + "catalogPath", + "catalogPaths", + "catalogs", + "expect", + "expectError", + "messages", + "protocolVersion", + "steps", + "strictMode", + "validationConfig", + ] + + private static let knownStepKeys: Set = [ + "expect", + "expectError", + "messages", + "validationConfig", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + "rendererError", + "surfaceId", + ] + + private static let knownSurfaceExpectKeys: Set = [ + "components", + "dataModel", + "defaultCatalogId", + "exists", + "metadata", + "sendDataModel", + "theme", + ] + + private static let knownDataModelArgsKeys: Set = [ + "version" + ] + + private static let knownCapabilitiesArgsKeys: Set = [ + "componentEnvelopeRef", + "includeInlineCatalogs", + "version", + "versions", + ] + + private static let knownResolvePathArgsKeys: Set = [ + "contextPath", + "path", ] /// A runtime error a case expects rendering to report through the action handler. @@ -69,6 +127,11 @@ struct MessageProcessorConformanceTests { } @Test func messageProcessorV10Conformance() throws { + let cases = ConformanceTestHelper.cases("core/message_processor_v1_0.yaml") + let caseNames = Set(cases.map(\.name)) + for skippedName in Self.skippedCases.keys { + #expect(caseNames.contains(skippedName), "Unused skip entry '\(skippedName)'") + } try runMessageProcessorSuite( filename: "core/message_processor_v1_0.yaml", defaultVersion: "1.0" @@ -79,52 +142,66 @@ struct MessageProcessorConformanceTests { filename: String, defaultVersion: String ) throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: filename) - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases(filename) #expect(!cases.isEmpty, "\(filename) should hold test cases") - var executed = 0 - - for testCase in cases { - let name = testCase["name"] as? String ?? "" - if Self.skippedCases.contains(name) { continue } - let action = testCase["action"] as? String ?? "process_messages" - - switch action { - case "process_messages", "validate": - try runProcessMessagesCase( - testCase, - name: name, - defaultVersion: defaultVersion - ) - executed += 1 - case "get_renderer_data_model": - try runGetRendererDataModelCase( - testCase, - name: name, - defaultVersion: defaultVersion - ) - executed += 1 - case "get_renderer_capabilities": - try runGetRendererCapabilitiesCase( - testCase, - name: name, + for conformanceCase in cases { + let skippedReason = Self.skippedCases[conformanceCase.name] + try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { + try dispatchMessageProcessorCase( + conformanceCase.rawDictionary, + name: conformanceCase.name, defaultVersion: defaultVersion ) - executed += 1 - case "resolve_path": - runResolvePathCase(testCase, name: name) - executed += 1 - default: - continue } } + } - #expect(executed > 0, "no case of \(filename) was executed") + private func dispatchMessageProcessorCase( + _ testCase: [String: Any], + name: String, + defaultVersion: String + ) throws { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownCaseKeys, + "message_processor case '\(name)'" + ) + let action = testCase["action"] as? String ?? "" + switch action { + case "process_messages", "validate": + try runProcessMessagesCase( + testCase, + name: name, + defaultVersion: defaultVersion + ) + case "get_renderer_data_model": + try runGetRendererDataModelCase( + testCase, + name: name, + defaultVersion: defaultVersion + ) + case "get_renderer_capabilities": + try runGetRendererCapabilitiesCase( + testCase, + name: name, + defaultVersion: defaultVersion + ) + case "resolve_path": + try runResolvePathCase(testCase, name: name) + default: + throw UnknownConformanceKeyError("[\(name)] Unknown message_processor action '\(action)'") + } } - private func runResolvePathCase(_ testCase: [String: Any], name: String) { + private func runResolvePathCase(_ testCase: [String: Any], name: String) throws { let args = testCase["args"] as? [String: Any] ?? [:] + try ConformanceTestHelper.assertKnownKeys( + args, + Self.knownResolvePathArgsKeys, + "message_processor resolve_path args in '\(name)'", + allowMetadata: false + ) let path = args["path"] as? String ?? "" let contextPath = args["contextPath"] as? String ?? "" let actual = JSONValue.absolutePath(for: path, in: contextPath) @@ -397,36 +474,77 @@ struct MessageProcessorConformanceTests { validationConfig: validationConfig ) let expectedRuntimeError = Self.expectedRuntimeErrors[name] - let steps = (testCase["steps"] as? [[String: Any]]) ?? [testCase] - - for step in steps { - let payload = ConformanceTestHelper.toJSONValue(step["messages"] ?? [:]) - handler.capturedErrors.removeAll() - - if let expectError = step["expectError"] as? [String: Any] { - assertStepError( - payload: payload, - expectError: expectError, - processor: processor, - handler: handler, - name: name + let explicitSteps = testCase["steps"] as? [[String: Any]] + if let explicitSteps { + for step in explicitSteps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownStepKeys, + "message_processor step in '\(name)'", + allowMetadata: false ) - continue } + } + let steps = explicitSteps ?? [testCase] - try processor.processMessages(payload) - assertStepRuntimeErrors( - capturedErrors: handler.capturedErrors, - expected: expectedRuntimeError, + for step in steps { + try executeProcessMessagesStep( + step, + processor: processor, + handler: handler, + expectedRuntimeError: expectedRuntimeError, name: name ) + } + } + + private func executeProcessMessagesStep( + _ step: [String: Any], + processor: MessageProcessor, + handler: ConformanceActionCaptureHandler, + expectedRuntimeError: ConformanceExpectRendererError?, + name: String + ) throws { + let payload = ConformanceTestHelper.toJSONValue(step["messages"] ?? [:]) + handler.capturedErrors.removeAll() + + if let expectError = step["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "message_processor expectError in '\(name)'", + allowMetadata: false + ) + assertStepError( + payload: payload, + expectError: expectError, + processor: processor, + handler: handler, + name: name + ) + return + } + + try processor.processMessages(payload) + assertStepRuntimeErrors( + capturedErrors: handler.capturedErrors, + expected: expectedRuntimeError, + name: name + ) - let expectedDict = (step["expect"] as? [String: Any]) ?? [:] - let expectedSurfaces = - (expectedDict["surfaces"] as? [String: Any]) - ?? (step["expectedSurfaces"] as? [String: Any]) - try verifyExpectedSurfaces(expectedSurfaces, on: processor, name: name) + if let expectedDict = step["expect"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectedDict, + ["surfaces"], + "message_processor expect in '\(name)'", + allowMetadata: false + ) } + let expectedDict = (step["expect"] as? [String: Any]) ?? [:] + let expectedSurfaces = + (expectedDict["surfaces"] as? [String: Any]) + ?? (step["expectedSurfaces"] as? [String: Any]) + try verifyExpectedSurfaces(expectedSurfaces, on: processor, name: name) } private func assertStepError( @@ -477,7 +595,7 @@ struct MessageProcessorConformanceTests { case "ValidationError": #expect(error is A2UIValidationError, "[\(name)] Expected ValidationError, got \(error)") default: - break + Issue.record("[\(name)] Unknown error category '\(category)'") } } @@ -538,6 +656,12 @@ struct MessageProcessorConformanceTests { return } guard let expectedSurfaceDict = rawExpected as? [String: Any] else { return } + try ConformanceTestHelper.assertKnownKeys( + expectedSurfaceDict, + Self.knownSurfaceExpectKeys, + "message_processor expect.surfaces.\(surfaceID) in '\(name)'", + allowMetadata: false + ) if expectedSurfaceDict["exists"] as? Bool == false { #expect( processor.surfaceGroupModel.surfacesMap[surfaceID] == nil, @@ -567,6 +691,36 @@ struct MessageProcessorConformanceTests { "[\(name)] DataModel mismatch on surface '\(surfaceID)'" ) } + verifySurfaceMetadataAndTheme( + expectedSurfaceDict, + on: surface, + surfaceID: surfaceID, + name: name + ) + if let expectedSendDataModel = expectedSurfaceDict["sendDataModel"] as? Bool { + #expect( + surface.sendDataModel == expectedSendDataModel, + "[\(name)] sendDataModel mismatch on surface '\(surfaceID)'" + ) + } + if let rawDefaultCatalogID = expectedSurfaceDict["defaultCatalogId"] { + if rawDefaultCatalogID is NSNull { + #expect(surface.defaultCatalogID == nil, "[\(name)] Expected nil defaultCatalogId") + } else if let expectedDefaultCatalogID = rawDefaultCatalogID as? String { + #expect( + surface.defaultCatalogID == expectedDefaultCatalogID, + "[\(name)] defaultCatalogId mismatch on surface '\(surfaceID)'" + ) + } + } + } + + private func verifySurfaceMetadataAndTheme( + _ expectedSurfaceDict: [String: Any], + on surface: SurfaceViewModel, + surfaceID: String, + name: String + ) { if expectedSurfaceDict.keys.contains("metadata") { let expectedSurfaceMeta = expectedSurfaceDict["metadata"].map { ConformanceTestHelper.toJSONValue($0) } ?? .null @@ -597,22 +751,6 @@ struct MessageProcessorConformanceTests { "[\(name)] Theme mismatch on surface '\(surfaceID)'" ) } - if let expectedSendDataModel = expectedSurfaceDict["sendDataModel"] as? Bool { - #expect( - surface.sendDataModel == expectedSendDataModel, - "[\(name)] sendDataModel mismatch on surface '\(surfaceID)'" - ) - } - if let rawDefaultCatalogID = expectedSurfaceDict["defaultCatalogId"] { - if rawDefaultCatalogID is NSNull { - #expect(surface.defaultCatalogID == nil, "[\(name)] Expected nil defaultCatalogId") - } else if let expectedDefaultCatalogID = rawDefaultCatalogID as? String { - #expect( - surface.defaultCatalogID == expectedDefaultCatalogID, - "[\(name)] defaultCatalogId mismatch on surface '\(surfaceID)'" - ) - } - } } private func verifySurfaceComponents( @@ -658,7 +796,10 @@ struct MessageProcessorConformanceTests { if let expectedType = compObj["component"] as? String { #expect( actualComp.type == expectedType, - "[\(name)] Component '\(id)' type mismatch: expected '\(expectedType)', got '\(actualComp.type)'" + """ + [\(name)] Component '\(id)' type mismatch: \ + expected '\(expectedType)', got '\(actualComp.type)' + """ ) } if let expectedCatalogID = compObj["catalogId"] as? String { @@ -683,7 +824,9 @@ struct MessageProcessorConformanceTests { } } for (propKey, propVal) in compObj - where propKey != "id" && propKey != "component" && propKey != "catalogId" && propKey != "metadata" { + where propKey != "id" && propKey != "component" && propKey != "catalogId" + && propKey != "metadata" + { #expect( actualComp.properties[propKey] == ConformanceTestHelper.toJSONValue(propVal), "[\(name)] Component '\(id)' property '\(propKey)' mismatch" @@ -698,6 +841,16 @@ struct MessageProcessorConformanceTests { ) throws { let catalogs = try buildProcessorCatalogs(from: testCase, defaultVersion: defaultVersion) let processor = MessageProcessor(catalogs: catalogs) + if let explicitSteps = testCase["steps"] as? [[String: Any]] { + for step in explicitSteps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownStepKeys, + "message_processor get_renderer_data_model step in '\(name)'", + allowMetadata: false + ) + } + } let steps = (testCase["steps"] as? [[String: Any]]) ?? [testCase] for step in steps { let payload = ConformanceTestHelper.toJSONValue(step["messages"] ?? [:]) @@ -705,16 +858,33 @@ struct MessageProcessorConformanceTests { } let args = testCase["args"] as? [String: Any] ?? [:] + try ConformanceTestHelper.assertKnownKeys( + args, + Self.knownDataModelArgsKeys, + "message_processor get_renderer_data_model args in '\(name)'", + allowMetadata: false + ) let versionStr = args["version"] as? String let protocolVersion = versionStr.flatMap { A2UIProtocolVersion(lenientRawValue: $0) } let actual = try processor.getRendererDataModel(version: protocolVersion) let expectedDict = (testCase["expect"] as? [String: Any]) ?? (testCase["expectedDataModel"] as? [String: Any]) + if let expectedDict { + try ConformanceTestHelper.assertKnownKeys( + expectedDict, + ["surfaces", "version"], + "message_processor get_renderer_data_model expect in '\(name)'", + allowMetadata: false + ) + } if let expectedSurfaces = expectedDict?["surfaces"] { #expect( actual?["surfaces"] == ConformanceTestHelper.toJSONValue(expectedSurfaces), - "[\(name)] Expected renderer data model \(expectedSurfaces), got \(String(describing: actual))" + """ + [\(name)] Expected renderer data model \(expectedSurfaces), \ + got \(String(describing: actual)) + """ ) } else { #expect( @@ -732,9 +902,21 @@ struct MessageProcessorConformanceTests { let catalogs = try buildProcessorCatalogs(from: testCase, defaultVersion: defaultVersion) let processor = MessageProcessor(catalogs: catalogs) let args = testCase["args"] as? [String: Any] ?? [:] + try ConformanceTestHelper.assertKnownKeys( + args, + Self.knownCapabilitiesArgsKeys, + "message_processor get_renderer_capabilities args in '\(name)'", + allowMetadata: false + ) let options = buildCapabilitiesOptions(from: args) if let expectError = testCase["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "message_processor get_renderer_capabilities expectError in '\(name)'", + allowMetadata: false + ) do { _ = try processor.getRendererCapabilities(options: options) Issue.record("[\(name)] Expected getRendererCapabilities to throw") diff --git a/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift index fff5fefd21..fc11f4c34d 100644 --- a/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift @@ -24,33 +24,78 @@ import Testing /// Runs the shared `conformance/core/multi_catalog.yaml` suite. @MainActor struct MultiCatalogConformanceTests { + private static let knownSelectCaseKeys: Set = [ + "action", + "args", + "catalog", + "expectError", + "expectSelected", + ] + + private static let knownSelectArgsKeys: Set = [ + "catalogs", + "components", + "functionCall", + "surface", + ] + + private static let knownValidateCaseKeys: Set = [ + "action", + "catalogs", + "expect", + "steps", + ] + + private static let knownValidateStepKeys: Set = [ + "payload" + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "message", + ] + @Test func multiCatalogConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/multi_catalog.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases("core/multi_catalog.yaml") #expect(!cases.isEmpty, "core/multi_catalog.yaml should hold test cases") - var executed = 0 - for testCase in cases { - let name = testCase["name"] as? String ?? "" - let action = testCase["action"] as? String ?? "" - - switch action { - case "select_catalog": - try runSelectCatalogCase(testCase, name: name) - executed += 1 - case "validate": - try runValidateCase(testCase, name: name) - executed += 1 - default: - continue + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try dispatchMultiCatalogCase(conformanceCase.rawDictionary, name: conformanceCase.name) } } + } - #expect(executed == cases.count, "All cases in core/multi_catalog.yaml should be executed") + private func dispatchMultiCatalogCase(_ testCase: [String: Any], name: String) throws { + let action = testCase["action"] as? String ?? "" + switch action { + case "select_catalog": + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownSelectCaseKeys, + "multi_catalog.yaml select_catalog case '\(name)'" + ) + try runSelectCatalogCase(testCase, name: name) + case "validate": + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownValidateCaseKeys, + "multi_catalog.yaml validate case '\(name)'" + ) + try runValidateCase(testCase, name: name) + default: + throw UnknownConformanceKeyError("[\(name)] Unknown multi_catalog.yaml action '\(action)'") + } } private func runSelectCatalogCase(_ testCase: [String: Any], name: String) throws { let args = (testCase["args"] as? [String: Any]) ?? [:] + try ConformanceTestHelper.assertKnownKeys( + args, + Self.knownSelectArgsKeys, + "multi_catalog.yaml args in '\(name)'", + allowMetadata: false + ) let surfaceDict = (args["surface"] as? [String: Any]) ?? [:] let surfaceID = (surfaceDict["id"] as? String) ?? "main_surface" let defaultCatalogID = surfaceDict["defaultCatalogId"] as? String @@ -59,12 +104,76 @@ struct MultiCatalogConformanceTests { let defaultVersion = (catalogConfig["protocolVersion"] as? String) ?? "1.0" let explicitCatalogs = (args["catalogs"] as? [String: Any]) ?? [:] + let catalogs = try buildSelectCatalogs( + supportedCatalogIDs: supportedCatalogIDs, + explicitCatalogs: explicitCatalogs, + defaultVersion: defaultVersion + ) + let processor = MessageProcessor(catalogs: catalogs) + let createPayload = buildCreateSurfacePayload( + surfaceID: surfaceID, + defaultCatalogID: defaultCatalogID, + defaultVersion: defaultVersion + ) + + let rawComponents = (args["components"] as? [String: Any]) ?? [:] + let sortedKeys = rawComponents.keys.sorted() + let componentList = buildSelectComponentList( + rawComponents: rawComponents, + sortedKeys: sortedKeys, + explicitCatalogs: explicitCatalogs, + supportedCatalogIDs: supportedCatalogIDs + ) + + if let expectError = testCase["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "multi_catalog.yaml expectError in '\(name)'", + allowMetadata: false + ) + assertSelectCatalogError( + processor: processor, + createPayload: createPayload, + componentList: componentList, + surfaceID: surfaceID, + defaultVersion: defaultVersion, + expectError: expectError, + name: name + ) + return + } + + try processSelectPayloads( + processor: processor, + createPayload: createPayload, + componentList: componentList, + surfaceID: surfaceID, + defaultVersion: defaultVersion + ) + + let surface = try #require(processor.surfaceGroupModel[surfaceID]) + if let expectedSelected = testCase["expectSelected"] as? String { + assertSelectedCatalog( + expectedSelected: expectedSelected, + sortedKeys: sortedKeys, + args: args, + surface: surface, + name: name + ) + } + } + + private func buildSelectCatalogs( + supportedCatalogIDs: [String], + explicitCatalogs: [String: Any], + defaultVersion: String + ) throws -> [AnyCatalog] { let openSchema = try Schema(instance: "{\"type\": \"object\"}") let dummyFn = ConformanceFunctionImplementation( api: FunctionAPI(name: "add", returnType: .number, schema: openSchema) ) - - let catalogs: [AnyCatalog] = supportedCatalogIDs.map { catID in + return supportedCatalogIDs.map { catID in let catDict = explicitCatalogs[catID] as? [String: Any] let version = (catDict?["protocolVersion"] as? String) ?? defaultVersion return Catalog( @@ -81,22 +190,31 @@ struct MultiCatalogConformanceTests { functions: [dummyFn] ) } + } - let processor = MessageProcessor(catalogs: catalogs) - + private func buildCreateSurfacePayload( + surfaceID: String, + defaultCatalogID: String?, + defaultVersion: String + ) -> JSONValue { var createSurfaceDict: OrderedDictionary = [ "surfaceId": .string(surfaceID) ] if let defaultCatalogID { createSurfaceDict["catalogId"] = .string(defaultCatalogID) } - let createPayload: JSONValue = .object([ + return .object([ "version": .string(defaultVersion), "createSurface": .object(createSurfaceDict), ]) + } - let rawComponents = (args["components"] as? [String: Any]) ?? [:] - let sortedKeys = rawComponents.keys.sorted() + private func buildSelectComponentList( + rawComponents: [String: Any], + sortedKeys: [String], + explicitCatalogs: [String: Any], + supportedCatalogIDs: [String] + ) -> [JSONValue] { var componentList: [JSONValue] = [] for key in sortedKeys { guard let compDict = rawComponents[key] as? [String: Any] else { continue } @@ -104,7 +222,6 @@ struct MultiCatalogConformanceTests { obj["id"] = .string(key) componentList.append(.object(obj)) } - if componentList.isEmpty && !explicitCatalogs.isEmpty { for (index, catID) in supportedCatalogIDs.enumerated() { componentList.append( @@ -116,29 +233,16 @@ struct MultiCatalogConformanceTests { ) } } + return componentList + } - if let expectError = testCase["expectError"] as? [String: Any] { - do { - try processor.processMessages(createPayload) - if !componentList.isEmpty { - let updatePayload: JSONValue = .object([ - "version": .string(defaultVersion), - "updateComponents": .object([ - "surfaceId": .string(surfaceID), - "components": .array(componentList), - ]), - ]) - try processor.processMessages(updatePayload) - } - Issue.record("[\(name)] Expected CatalogError, but succeeded") - } catch { - if let category = expectError["category"] as? String, category == "CatalogError" { - #expect(error is A2UICatalogError, "[\(name)] Expected A2UICatalogError, got \(error)") - } - } - return - } - + private func processSelectPayloads( + processor: MessageProcessor, + createPayload: JSONValue, + componentList: [JSONValue], + surfaceID: String, + defaultVersion: String + ) throws { try processor.processMessages(createPayload) if !componentList.isEmpty { let updatePayload: JSONValue = .object([ @@ -150,28 +254,63 @@ struct MultiCatalogConformanceTests { ]) try processor.processMessages(updatePayload) } + } - let surface = try #require(processor.surfaceGroupModel[surfaceID]) - if let expectedSelected = testCase["expectSelected"] as? String { - if let lastKey = sortedKeys.last, - let compModel = surface.componentsModel.get(lastKey) - { - let resolvedCatalog = surface.getCatalog(id: compModel.catalogID) - #expect( - resolvedCatalog?.id == expectedSelected, - "[\(name)] Expected catalog '\(expectedSelected)', got '\(resolvedCatalog?.id ?? "nil")'" - ) - } else if let fnCall = args["functionCall"] as? [String: Any] { - let fnCatalogID = fnCall["catalogId"] as? String - let resolvedCatalog = surface.getCatalog(id: fnCatalogID) - #expect( - resolvedCatalog?.id == expectedSelected, - "[\(name)] Expected catalog '\(expectedSelected)', got '\(resolvedCatalog?.id ?? "nil")'" - ) + private func assertSelectCatalogError( + processor: MessageProcessor, + createPayload: JSONValue, + componentList: [JSONValue], + surfaceID: String, + defaultVersion: String, + expectError: [String: Any], + name: String + ) { + do { + try processSelectPayloads( + processor: processor, + createPayload: createPayload, + componentList: componentList, + surfaceID: surfaceID, + defaultVersion: defaultVersion + ) + Issue.record("[\(name)] Expected CatalogError, but succeeded") + } catch { + if let category = expectError["category"] as? String { + switch category { + case "CatalogError": + #expect(error is A2UICatalogError, "[\(name)] Expected A2UICatalogError, got \(error)") + default: + Issue.record("[\(name)] Unknown expected error category '\(category)'") + } } } } + private func assertSelectedCatalog( + expectedSelected: String, + sortedKeys: [String], + args: [String: Any], + surface: SurfaceViewModel, + name: String + ) { + if let lastKey = sortedKeys.last, + let compModel = surface.componentsModel.get(lastKey) + { + let resolvedCatalog = surface.getCatalog(id: compModel.catalogID) + #expect( + resolvedCatalog?.id == expectedSelected, + "[\(name)] Expected catalog '\(expectedSelected)', got '\(resolvedCatalog?.id ?? "nil")'" + ) + } else if let fnCall = args["functionCall"] as? [String: Any] { + let fnCatalogID = fnCall["catalogId"] as? String + let resolvedCatalog = surface.getCatalog(id: fnCatalogID) + #expect( + resolvedCatalog?.id == expectedSelected, + "[\(name)] Expected catalog '\(expectedSelected)', got '\(resolvedCatalog?.id ?? "nil")'" + ) + } + } + private func runValidateCase(_ testCase: [String: Any], name: String) throws { let commonTypes = try? ConformanceTestHelper.commonTypesSchema(forProtocolVersion: "1.0") var catalogs = ConformanceTestHelper.basicCatalogs(for: .v10) @@ -190,33 +329,62 @@ struct MultiCatalogConformanceTests { let processor = MessageProcessor(catalogs: catalogs) let steps = (testCase["steps"] as? [[String: Any]]) ?? [] for step in steps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownValidateStepKeys, + "multi_catalog.yaml validate step in '\(name)'", + allowMetadata: false + ) guard let rawPayload = step["payload"] else { continue } let payload = ConformanceTestHelper.toJSONValue(rawPayload) try processor.processMessages(payload) } - if let expectDict = testCase["expect"] as? [String: Any], - let expectSurfaces = expectDict["surfaces"] as? [String: Any] - { - for (surfaceID, rawSurface) in expectSurfaces { - guard let expectedSurface = rawSurface as? [String: Any] else { continue } - let surface = try #require( - processor.surfaceGroupModel[surfaceID], - "[\(name)] Expected surface '\(surfaceID)' to exist" - ) - if let expectedComponents = expectedSurface["components"] as? [String: Any] { - for (compID, rawComp) in expectedComponents { - guard let compDict = rawComp as? [String: Any] else { continue } - let node = try #require( - surface.findNode(id: compID), - "[\(name)] Expected node '\(compID)' to exist" + if let expectDict = testCase["expect"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectDict, + ["surfaces"], + "multi_catalog.yaml validate expect in '\(name)'", + allowMetadata: false + ) + if let expectSurfaces = expectDict["surfaces"] as? [String: Any] { + try assertMultiCatalogSurfaces(expectSurfaces, on: processor, name: name) + } + } + } + + private func assertMultiCatalogSurfaces( + _ expectSurfaces: [String: Any], + on processor: MessageProcessor, + name: String + ) throws { + for (surfaceID, rawSurface) in expectSurfaces { + guard let expectedSurface = rawSurface as? [String: Any] else { continue } + try ConformanceTestHelper.assertKnownKeys( + expectedSurface, + ["components"], + "multi_catalog.yaml expect.surfaces.\(surfaceID) in '\(name)'", + allowMetadata: false + ) + let surface = try #require( + processor.surfaceGroupModel[surfaceID], + "[\(name)] Expected surface '\(surfaceID)' to exist" + ) + if let expectedComponents = expectedSurface["components"] as? [String: Any] { + for (compID, rawComp) in expectedComponents { + guard let compDict = rawComp as? [String: Any] else { continue } + let node = try #require( + surface.findNode(id: compID), + "[\(name)] Expected node '\(compID)' to exist" + ) + if let expectedText = compDict["text"] as? String { + #expect( + node.string(for: "text") == expectedText, + """ + [\(name)] Expected text '\(expectedText)', \ + got '\(node.string(for: "text") ?? "nil")' + """ ) - if let expectedText = compDict["text"] as? String { - #expect( - node.string(for: "text") == expectedText, - "[\(name)] Expected text '\(expectedText)', got '\(node.string(for: "text") ?? "nil")'" - ) - } } } } diff --git a/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift index fa4b605893..fe30574afb 100644 --- a/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift @@ -65,22 +65,85 @@ private final class RecordingFunctionImplementation: FunctionImplementation, @un /// Runs the shared `conformance/core/node_resolution.yaml` suite. @MainActor struct NodeResolutionConformanceTests { + static let placeholderKnownIssues: [String: String] = [ + "cyclic_component_reference_same_scope": "NodeResolver omits placeholder nodes", + "pending_child_replacement": "NodeResolver omits placeholder nodes", + "shared_definition_independent_instances": "NodeResolver omits placeholder nodes", + "unknown_component_type_placeholder": "NodeResolver omits placeholder nodes", + ] + + private static let knownCaseKeys: Set = [ + "action", + "expect", + "fixture", + "steps", + ] + + private static let knownStepKeys: Set = [ + "component_id", + "components", + "expect", + "messages", + "metadata", + "node", + "op", + "path", + "property", + "value", + ] + + private static let knownExpectKeys: Set = [ + "data", + "destroyed", + "emissions", + "errors", + "events", + "functions", + "metadata_emissions", + "nodes", + "replaced_nodes", + "same_nodes", + ] + + private static let knownNodeSpecKeys: Set = [ + "component_id", + "data_path", + "isValid", + "metadata", + "props", + "state", + "type", + "validationErrors", + "validationResults", + ] + @Test func nodeResolutionConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/node_resolution.yaml") - let cases = (rawYAML as? [[String: Any]]) ?? [] + let cases = ConformanceTestHelper.cases("core/node_resolution.yaml") #expect(!cases.isEmpty, "core/node_resolution.yaml should hold test cases") + let caseNames = Set(cases.map(\.name)) + for knownIssueName in Self.placeholderKnownIssues.keys { + #expect(caseNames.contains(knownIssueName), "Unused known issue '\(knownIssueName)'") + } - var executed = 0 - for testCase in cases { - guard (testCase["action"] as? String) == "resolve_nodes" else { continue } - try runResolveNodesCase(testCase) - executed += 1 + for conformanceCase in cases { + let knownIssueReason = Self.placeholderKnownIssues[conformanceCase.name] + try ConformanceManifest.run(conformanceCase, knownIssueReason: knownIssueReason) { + try runResolveNodesCase(conformanceCase.rawDictionary) + } } - #expect(executed > 0, "Should execute at least one resolve_nodes case") } private func runResolveNodesCase(_ testCase: [String: Any]) throws { let name = testCase["name"] as? String ?? "" + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownCaseKeys, + "node_resolution.yaml case '\(name)'" + ) + let action = testCase["action"] as? String ?? "" + guard action == "resolve_nodes" else { + throw UnknownConformanceKeyError("[\(name)] Unknown node_resolution.yaml action '\(action)'") + } let fixturePath = try #require( testCase["fixture"] as? String, "[\(name)] Missing fixture path" @@ -111,7 +174,7 @@ struct NodeResolutionConformanceTests { applyFixture(fixture, to: surface) if let initialExpect = testCase["expect"] as? [String: Any] { - assertStepExpectations( + try assertStepExpectations( initialExpect, surface: processor.surfaceGroupModel["s"], capture: capture, @@ -124,7 +187,7 @@ struct NodeResolutionConformanceTests { capture.clearStep() try executeStep(step, processor: processor, caseName: name) if let stepExpect = step["expect"] as? [String: Any] { - assertStepExpectations( + try assertStepExpectations( stepExpect, surface: processor.surfaceGroupModel["s"], capture: capture, @@ -225,7 +288,13 @@ struct NodeResolutionConformanceTests { processor: MessageProcessor, caseName: String ) throws { - guard let op = step["op"] as? String else { return } + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownStepKeys, + "node_resolution.yaml step in '\(caseName)'", + allowMetadata: false + ) + let op = step["op"] as? String ?? "" switch op { case "write": executeWriteStep(step, surface: processor.surfaceGroupModel["s"], caseName: caseName) @@ -253,7 +322,7 @@ struct NodeResolutionConformanceTests { case "dispose": processor.surfaceGroupModel.removeSurface(id: "s") default: - break + throw UnknownConformanceKeyError("[\(caseName)] Unknown node_resolution step op '\(op)'") } } @@ -356,30 +425,13 @@ struct NodeResolutionConformanceTests { return node.properties[segment] } if let nodes = current as? [Node], let idx = Int(segment) { - if let parentNode, let propertyKey, let surface { - let parentDefID = - parentNode.id.split(separator: "_").first.map(String.init) ?? parentNode.id - if let rawChildren = surface.componentsModel.components[parentDefID]?.properties[ - propertyKey - ]?.arrayValue, - rawChildren.indices.contains(idx), - let childDefID = rawChildren[idx].stringValue - { - guard surface.componentsModel.components[childDefID] != nil else { - return nil - } - if let match = nodes.first(where: { - $0.id == childDefID || $0.id.hasPrefix("\(childDefID)_") - || $0.id.hasPrefix("\(childDefID):") - }) { - return match - } - } - } - if nodes.indices.contains(idx) { - return nodes[idx] - } - return nil + return stepNodeArrayElement( + nodes: nodes, + idx: idx, + parentNode: parentNode, + propertyKey: propertyKey, + surface: surface + ) } if let arr = current as? ResolvedArray, let idx = Int(segment), @@ -393,25 +445,34 @@ struct NodeResolutionConformanceTests { return nil } - private func isPendingChildSlot(_ path: String, surface: SurfaceViewModel?) -> Bool { - guard let surface, let root = surface.rootNode else { return false } - let segments = path.split(separator: "/").map(String.init) - guard segments.count >= 3, let idx = Int(segments.last ?? "") else { return false } - let propKey = segments[segments.count - 2] - let parentPath = segments.dropLast(2).joined(separator: "/") - guard let parentNode = findNodeByPath(parentPath, from: root, surface: surface) else { - return false - } - let parentDefID = parentNode.id.split(separator: "_").first.map(String.init) ?? parentNode.id - guard - let rawChildren = surface.componentsModel.components[parentDefID]?.properties[propKey]? - .arrayValue, - rawChildren.indices.contains(idx), - let childDefID = rawChildren[idx].stringValue - else { - return false + private func stepNodeArrayElement( + nodes: [Node], + idx: Int, + parentNode: Node?, + propertyKey: String?, + surface: SurfaceViewModel? + ) -> Node? { + if let parentNode, let propertyKey, let surface { + let parentDefID = + parentNode.id.split(separator: "_").first.map(String.init) ?? parentNode.id + if let rawChildren = surface.componentsModel.components[parentDefID]?.properties[ + propertyKey + ]?.arrayValue, + rawChildren.indices.contains(idx), + let childDefID = rawChildren[idx].stringValue + { + guard surface.componentsModel.components[childDefID] != nil else { + return nil + } + if let match = nodes.first(where: { + $0.id == childDefID || $0.id.hasPrefix("\(childDefID)_") + || $0.id.hasPrefix("\(childDefID):") + }) { + return match + } + } } - return surface.componentsModel.components[childDefID] == nil + return nodes.indices.contains(idx) ? nodes[idx] : nil } private func assertStepExpectations( @@ -419,9 +480,15 @@ struct NodeResolutionConformanceTests { surface: SurfaceViewModel?, capture: NodeResolutionActionCapture, caseName: String - ) { + ) throws { + try ConformanceTestHelper.assertKnownKeys( + expect, + Self.knownExpectKeys, + "node_resolution.yaml expect in '\(caseName)'", + allowMetadata: false + ) if let expectedNodes = expect["nodes"] as? [String: Any] { - assertExpectedNodes(expectedNodes, surface: surface, caseName: caseName) + try assertExpectedNodes(expectedNodes, surface: surface, caseName: caseName) } if let expectedData = expect["data"] as? [String: Any], let surface { for (path, rawVal) in expectedData { @@ -443,23 +510,20 @@ struct NodeResolutionConformanceTests { _ expectedNodes: [String: Any], surface: SurfaceViewModel?, caseName: String - ) { + ) throws { if expectedNodes.isEmpty { #expect(surface?.rootNode == nil, "[\(caseName)] Expected empty tree (rootNode == nil)") return } for (nodePath, rawSpec) in expectedNodes { guard let spec = rawSpec as? [String: Any] else { continue } - let state = spec["state"] as? String ?? "resolved" - let resolvedNode = findNodeByPath(nodePath, from: surface?.rootNode, surface: surface) - if state == "pending" || state == "unknown-type" || state == "cyclic" - || (resolvedNode == nil && isPendingChildSlot(nodePath, surface: surface)) - { - // Swift's value-type NodeResolver omits unresolved/cyclic placeholders or resolves known - // component definitions; only verify resolved nodes. - continue - } - guard let node = resolvedNode else { + try ConformanceTestHelper.assertKnownKeys( + spec, + Self.knownNodeSpecKeys, + "node_resolution.yaml node spec '\(nodePath)' in '\(caseName)'", + allowMetadata: false + ) + guard let node = findNodeByPath(nodePath, from: surface?.rootNode, surface: surface) else { Issue.record("[\(caseName)] Expected resolved node at '\(nodePath)'") continue } @@ -473,12 +537,20 @@ struct NodeResolutionConformanceTests { nodePath: String, caseName: String ) { + let expectedState = spec["state"] as? String ?? "resolved" + #expect( + expectedState == "resolved", + "[\(caseName)] \(nodePath) state mismatch: got 'resolved', expected '\(expectedState)'" + ) if let expectedCompID = spec["component_id"] as? String { let baseID = node.id.split(separator: ":").first.map(String.init) ?? node.id let strippedID = baseID.split(separator: "_").first.map(String.init) ?? baseID #expect( node.id == expectedCompID || baseID == expectedCompID || strippedID == expectedCompID, - "[\(caseName)] \(nodePath) component_id mismatch: got '\(node.id)', expected '\(expectedCompID)'" + """ + [\(caseName)] \(nodePath) component_id mismatch: \ + got '\(node.id)', expected '\(expectedCompID)' + """ ) } if let expectedType = spec["type"] as? String { @@ -507,7 +579,8 @@ struct NodeResolutionConformanceTests { } } if let expectedProps = spec["props"] as? [String: Any] { - assertNodePropsMatch(node: node, expectedProps: expectedProps, nodePath: nodePath, caseName: caseName) + assertNodePropsMatch( + node: node, expectedProps: expectedProps, nodePath: nodePath, caseName: caseName) } } @@ -527,8 +600,11 @@ struct NodeResolutionConformanceTests { nodePath: nodePath, caseName: caseName ) - } else if let propSpec = rawExpectedProp as? [String: Any], propSpec["action"] as? Bool == true { - #expect(node.action(for: propKey) != nil, "[\(caseName)] \(nodePath).\(propKey) expected action") + } else if let propSpec = rawExpectedProp as? [String: Any], + propSpec["action"] as? Bool == true + { + #expect( + node.action(for: propKey) != nil, "[\(caseName)] \(nodePath).\(propKey) expected action") } else if let childSpecs = rawExpectedProp as? [[String: Any]], childSpecs.first?["node"] != nil { @@ -550,7 +626,10 @@ struct NodeResolutionConformanceTests { if let expStr = expectedVal.stringValue { #expect( node.string(for: propKey) == expStr, - "[\(caseName)] \(nodePath).\(propKey) string mismatch: got '\(String(describing: node.string(for: propKey)))', expected '\(expStr)'" + """ + [\(caseName)] \(nodePath).\(propKey) string mismatch: \ + got '\(String(describing: node.string(for: propKey)))', expected '\(expStr)' + """ ) } else if let expBool = expectedVal.boolValue { #expect( @@ -567,41 +646,56 @@ struct NodeResolutionConformanceTests { ) { #expect(actual.count == expectedEvents.count, "[\(caseName)] Event count mismatch") for (actualEvent, expSpec) in zip(actual, expectedEvents) { - guard case .event(let name, let context, _) = actualEvent.identity else { - Issue.record("[\(caseName)] Expected .event identity") - continue - } - if let expName = expSpec["name"] as? String { - #expect(name == expName, "[\(caseName)] Event name mismatch") - } - if let expSource = expSpec["source_component_id"] as? String { - #expect(actualEvent.sourceComponentID == expSource, "[\(caseName)] source_component_id mismatch") - } - if let expCtx = expSpec["context"] as? [String: Any] { - for (key, val) in expCtx { - #expect( - context?[key] == ConformanceTestHelper.toJSONValue(val), - "[\(caseName)] Event context[\(key)] mismatch" - ) - } - } - if expSpec.keys.contains("metadata") { - if expSpec["metadata"] is NSNull { - #expect(actualEvent.metadata == nil, "[\(caseName)] Expected nil event metadata") - } else { - let expectedMeta = try? MetadataValidator.validate( - ConformanceTestHelper.toJSONValue(expSpec["metadata"] as Any), - pathPrefix: "action.metadata" - ) - #expect( - MetadataValidator.areEqual(actualEvent.metadata, expectedMeta), - "[\(caseName)] Event metadata mismatch" - ) - } - } else { - #expect(actualEvent.metadata == nil, "[\(caseName)] Expected nil event metadata") + assertSingleEventMatches(actualEvent: actualEvent, expSpec: expSpec, caseName: caseName) + } + } + + private func assertSingleEventMatches( + actualEvent: ResolvedAction, + expSpec: [String: Any], + caseName: String + ) { + guard case .event(let name, let context, _) = actualEvent.identity else { + Issue.record("[\(caseName)] Expected .event identity") + return + } + if let expName = expSpec["name"] as? String { + #expect(name == expName, "[\(caseName)] Event name mismatch") + } + if let expSource = expSpec["source_component_id"] as? String { + #expect( + actualEvent.sourceComponentID == expSource, + "[\(caseName)] source_component_id mismatch" + ) + } + if let expCtx = expSpec["context"] as? [String: Any] { + for (key, val) in expCtx { + #expect( + context?[key] == ConformanceTestHelper.toJSONValue(val), + "[\(caseName)] Event context[\(key)] mismatch" + ) } } + assertEventMetadata(actualEvent: actualEvent, expSpec: expSpec, caseName: caseName) + } + + private func assertEventMetadata( + actualEvent: ResolvedAction, + expSpec: [String: Any], + caseName: String + ) { + guard expSpec.keys.contains("metadata"), !(expSpec["metadata"] is NSNull) else { + #expect(actualEvent.metadata == nil, "[\(caseName)] Expected nil event metadata") + return + } + let expectedMeta = try? MetadataValidator.validate( + ConformanceTestHelper.toJSONValue(expSpec["metadata"] as Any), + pathPrefix: "action.metadata" + ) + #expect( + MetadataValidator.areEqual(actualEvent.metadata, expectedMeta), + "[\(caseName)] Event metadata mismatch" + ) } private func assertExpectedFunctions( diff --git a/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift index 3cc52638df..9f36e57695 100644 --- a/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift @@ -38,49 +38,225 @@ private final class OutboundMessageBox: @unchecked Sendable { /// Runs the shared `conformance/core/rpc_functions.yaml` suite. @MainActor struct RPCConformanceTests { + private static let knownCaseKeys: Set = [ + "action", + "args", + "catalog", + "expect", + "expectError", + ] + + private static let knownArgsKeys: Set = [ + "catalogId", + "catalogVersion", + "dataModel", + "functionMetadata", + "inboundResponse", + "message", + "outboundCall", + "secondOutboundCall", + "surfaceId", + "uiActionFallbackScenario", + "uiBindingScenario", + "userActivationPresent", + ] + + private static let knownFunctionMetaKeys: Set = [ + "allowedCallers", + "parameters", + "requiresUserActivation", + ] + + private static let knownOutboundCallKeys: Set = [ + "callFunction", + "functionCallId", + "surfaceId", + "timeoutMs", + ] + + private static let knownInboundResponseKeys: Set = [ + "agentFunctionResponse", + "version", + ] + + private static let knownUIBindingKeys: Set = [ + "catalogId", + "component", + "inboundResponseValue", + "surfaceId", + ] + + private static let knownUIActionKeys: Set = [ + "actionPayload", + "catalogId", + "inboundResponseValue", + "surfaceId", + ] + + private static let knownExpectKeys: Set = [ + "correlatedCallId", + "error", + "initialValue", + "outboundCall", + "resolvedValue", + "response", + "result", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "code", + "functionCallId", + "rendererError", + ] + @Test func rpcFunctionsConformance() async throws { - let rawYaml = try ConformanceTestHelper.loadYAML(filename: "core/rpc_functions.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYaml) - #expect(!testCases.isEmpty, "Should load test cases from rpc_functions.yaml") + let cases = ConformanceTestHelper.cases("core/rpc_functions.yaml") + #expect(!cases.isEmpty, "Should load test cases from rpc_functions.yaml") + + for conformanceCase in cases { + try await ConformanceManifest.runAsync(conformanceCase) { + try await runSingleRPCCase(conformanceCase) + } + } + } + private func runSingleRPCCase(_ conformanceCase: ConformanceCase) async throws { + let rawDict = conformanceCase.rawDictionary + try validateRPCCaseKeys(rawDict, name: conformanceCase.name) + let action = rawDict["action"] as? String ?? "" + guard action == "handle_rpc" else { + throw UnknownConformanceKeyError( + "[\(conformanceCase.name)] Unknown rpc_functions.yaml action '\(action)'" + ) + } + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse rpc_functions case" + ) let rpcHandler = RPCHandler() + let catalogs = buildRPCCatalogs(for: testCase) - for testCase in testCases where testCase.action == "handle_rpc" { - let catalogs = buildRPCCatalogs(for: testCase) + if let messageJSON = testCase.args?["message"] { + try await runInboundMessageCase( + testCase: testCase, + messageJSON: messageJSON, + catalogs: catalogs, + rpcHandler: rpcHandler + ) + } - if let messageJSON = testCase.args?["message"] { - try await runInboundMessageCase( - testCase: testCase, - messageJSON: messageJSON, - catalogs: catalogs, - rpcHandler: rpcHandler - ) - } + if let outboundCall = testCase.args?["outboundCall"] { + try await runOutboundCallCase( + testCase: testCase, + outboundCall: outboundCall, + rpcHandler: rpcHandler + ) + } - if let outboundCall = testCase.args?["outboundCall"] { - try await runOutboundCallCase( - testCase: testCase, - outboundCall: outboundCall, - rpcHandler: rpcHandler - ) - } + if let uiBinding = testCase.args?["uiBindingScenario"] { + try await runUIBindingScenario( + testCase: testCase, + scenario: uiBinding, + rpcHandler: rpcHandler + ) + } - if let uiBinding = testCase.args?["uiBindingScenario"] { - try await runUIBindingScenario( - testCase: testCase, - scenario: uiBinding, - rpcHandler: rpcHandler - ) - } + if let uiActionFallback = testCase.args?["uiActionFallbackScenario"] { + try await runUIActionFallbackScenario( + testCase: testCase, + scenario: uiActionFallback, + rpcHandler: rpcHandler + ) + } + } - if let uiActionFallback = testCase.args?["uiActionFallbackScenario"] { - try await runUIActionFallbackScenario( - testCase: testCase, - scenario: uiActionFallback, - rpcHandler: rpcHandler - ) + private func validateRPCCaseKeys(_ rawDict: [String: Any], name: String) throws { + try ConformanceTestHelper.assertKnownKeys( + rawDict, + Self.knownCaseKeys, + "rpc_functions.yaml case '\(name)'" + ) + if let args = rawDict["args"] as? [String: Any] { + try validateRPCArgsKeys(args, name: name) + } + if let expect = rawDict["expect"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expect, + Self.knownExpectKeys, + "rpc_functions.yaml expect in '\(name)'", + allowMetadata: false + ) + } + if let expectError = rawDict["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + expectError, + Self.knownExpectErrorKeys, + "rpc_functions.yaml expectError in '\(name)'", + allowMetadata: false + ) + } + } + + private func validateRPCArgsKeys(_ args: [String: Any], name: String) throws { + try ConformanceTestHelper.assertKnownKeys( + args, + Self.knownArgsKeys, + "rpc_functions.yaml args in '\(name)'", + allowMetadata: false + ) + if let fnMeta = args["functionMetadata"] as? [String: Any] { + for (fnName, rawMeta) in fnMeta { + if let metaDict = rawMeta as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + metaDict, + Self.knownFunctionMetaKeys, + "rpc_functions.yaml functionMetadata.\(fnName) in '\(name)'", + allowMetadata: false + ) + } } } + if let outbound = args["outboundCall"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + outbound, + Self.knownOutboundCallKeys, + "rpc_functions.yaml outboundCall in '\(name)'", + allowMetadata: false + ) + } + if let secondOutbound = args["secondOutboundCall"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + secondOutbound, + Self.knownOutboundCallKeys, + "rpc_functions.yaml secondOutboundCall in '\(name)'", + allowMetadata: false + ) + } + if let inbound = args["inboundResponse"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + inbound, + Self.knownInboundResponseKeys, + "rpc_functions.yaml inboundResponse in '\(name)'", + allowMetadata: false + ) + } + if let uiBinding = args["uiBindingScenario"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + uiBinding, + Self.knownUIBindingKeys, + "rpc_functions.yaml uiBindingScenario in '\(name)'", + allowMetadata: false + ) + } + if let uiAction = args["uiActionFallbackScenario"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + uiAction, + Self.knownUIActionKeys, + "rpc_functions.yaml uiActionFallbackScenario in '\(name)'", + allowMetadata: false + ) + } } private func buildRPCCatalogs(for testCase: ConformanceTestCase) -> [String: AnyCatalog] { @@ -260,8 +436,12 @@ struct RPCConformanceTests { #expect(response.error?.code == expectedCode, "[\(testCase.name)] error.code mismatch") if let expectedMsg { #expect( - ConformanceTestHelper.messageMatches(response.error?.message ?? "", expected: expectedMsg), - "[\(testCase.name)] error.message mismatch: \(String(describing: response.error?.message))" + ConformanceTestHelper.messageMatches( + response.error?.message ?? "", expected: expectedMsg), + """ + [\(testCase.name)] error.message mismatch: \ + \(String(describing: response.error?.message)) + """ ) } } @@ -284,7 +464,8 @@ struct RPCConformanceTests { } catch { caughtError = error } - #expect(caughtError != nil, "[\(testCase.name)] Expected processor to throw on invalid RPC message") + #expect( + caughtError != nil, "[\(testCase.name)] Expected processor to throw on invalid RPC message") if let expectedRendererError = testCase.expectError?.rendererError { ConformanceTestHelper.assertRendererErrorMatches( @@ -295,7 +476,8 @@ struct RPCConformanceTests { } else if let expectedErr = testCase.expect?["error"]?.objectValue, let msgSub = expectedErr["message"]?.stringValue { - let desc = caughtError.map { ($0 as? (any A2UIError))?.message ?? $0.localizedDescription } ?? "" + let desc = + caughtError.map { ($0 as? (any A2UIError))?.message ?? $0.localizedDescription } ?? "" #expect( ConformanceTestHelper.messageMatches(desc, expected: msgSub), "[\(testCase.name)] Expected error message containing '\(msgSub)', got '\(desc)'" @@ -518,7 +700,8 @@ struct RPCConformanceTests { ) ) let resolved = try await callTask.value - #expect(resolved == testCase.expect?["resolvedValue"], "[\(testCase.name)] resolvedValue mismatch") + #expect( + resolved == testCase.expect?["resolvedValue"], "[\(testCase.name)] resolvedValue mismatch") } private func runUIActionFallbackScenario( @@ -578,7 +761,8 @@ struct RPCConformanceTests { ) { guard let expectedCall = expectedOutbound?["callAgentFunction"] else { return } guard case .callAgentFunction(let actualCall) = captured else { - Issue.record("[\(testName)] Expected outbound callAgentFunction, got \(String(describing: captured))") + Issue.record( + "[\(testName)] Expected outbound callAgentFunction, got \(String(describing: captured))") return } if let expSurfaceID = expectedCall["surfaceId"]?.stringValue { diff --git a/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift b/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift new file mode 100644 index 0000000000..eff428455c --- /dev/null +++ b/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift @@ -0,0 +1,86 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import Testing + +/// Verifies that every `conformance/core/*.yaml` suite and action is registered in +/// `ConformanceTestHelper` and records skipped manifest entries for unsupported suites. +struct SuiteRegistryTests { + @Test func allCoreSuitesAndActionsAreRegistered() throws { + let coreDir = ConformanceTestHelper.repoRoot + .appendingPathComponent("conformance") + .appendingPathComponent("core") + let entries = try FileManager.default.contentsOfDirectory( + at: coreDir, + includingPropertiesForKeys: nil + ) + let onDiskSuites = Set( + entries + .filter { $0.pathExtension == "yaml" } + .map { "core/\($0.lastPathComponent)" } + ) + #expect(!onDiskSuites.isEmpty, "conformance/core should contain YAML suites") + + let supportedSuites = Set(ConformanceTestHelper.coreSuiteActions.keys) + let unsupportedSuites = Set(ConformanceTestHelper.unsupportedCoreSuites.keys) + let overlap = supportedSuites.intersection(unsupportedSuites) + #expect(overlap.isEmpty, "Suites registered as both supported and unsupported: \(overlap)") + + let registeredSuites = supportedSuites.union(unsupportedSuites) + #expect( + registeredSuites == onDiskSuites, + "Registered suites \(registeredSuites.sorted()) != on-disk suites \(onDiskSuites.sorted())" + ) + + try verifySupportedSuiteActions() + recordUnsupportedSuiteSkips() + } + + private func verifySupportedSuiteActions() throws { + for (suite, actionMap) in ConformanceTestHelper.coreSuiteActions.sorted(by: { $0.key < $1.key }) + { + let cases = ConformanceTestHelper.cases(suite) + #expect(!cases.isEmpty, "\(suite) should contain at least one test case") + let actualActions = Set(cases.compactMap { $0.json["action"]?.stringValue }) + let expectedActions = Set(actionMap.keys) + #expect( + actualActions == expectedActions, + "\(suite) actions \(actualActions.sorted()) != registered \(expectedActions.sorted())" + ) + } + } + + private func recordUnsupportedSuiteSkips() { + for (suite, reason) in ConformanceTestHelper.unsupportedCoreSuites.sorted(by: { + $0.key < $1.key + }) { + #expect(!reason.isEmpty, "Unsupported suite \(suite) must provide a skip reason") + let cases = ConformanceTestHelper.cases(suite) + #expect(!cases.isEmpty, "Unsupported suite \(suite) should contain test cases") + for testCase in cases { + ConformanceManifest.record(testCase, status: .skipped, reason: reason) + } + } + if !ConformanceTiming.isEnabled { + for testCase in ConformanceTestHelper.cases("core/message_processor_scaling.yaml") { + ConformanceManifest.record( + testCase, + status: .skipped, + reason: "set A2UI_RUN_TIMING=1 to run timing cases" + ) + } + } + } +} diff --git a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift index d01cebab41..83365a8c72 100644 --- a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift @@ -33,6 +33,38 @@ private final class ValidatorErrorCaptureHandler: ActionHandling, @unchecked Sen @MainActor struct ValidatorConformanceTests { + private static let knownCaseKeys: Set = [ + "action", + "catalog", + "catalogPaths", + "expect", + "expectError", + "messages", + "payload", + "protocolVersion", + "steps", + "strictMode", + "validationConfig", + ] + + private static let knownStepKeys: Set = [ + "expect", + "expectError", + "messages", + "payload", + ] + + private static let knownExpectErrorKeys: Set = [ + "category", + "code", + "details", + "functionCallId", + "message", + "path", + "rendererError", + "surfaceId", + ] + @Test func validatorConformance() throws { try runValidatorSuite(filename: "core/validator_v0_9.yaml", versionPrefix: "v0.9") } @@ -44,47 +76,85 @@ struct ValidatorConformanceTests { @Test func errorResponsesConformance() throws { try runValidatorSuite(filename: "core/error_responses.yaml", versionPrefix: nil) } - private func runValidatorSuite(filename: String, versionPrefix: String?) throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: filename) - let testCases = ConformanceTestHelper.parseTestCases(from: rawYAML) - #expect(!testCases.isEmpty, "\(filename) should hold test cases") - var executedSteps = 0 + private func runValidatorSuite(filename: String, versionPrefix: String?) throws { + let cases = ConformanceTestHelper.cases(filename) + #expect(!cases.isEmpty, "\(filename) should hold test cases") - for testCase in testCases where matchesSuiteFilter(testCase, versionPrefix: versionPrefix) { - let targetVersion = resolveTargetVersion(for: testCase, versionPrefix: versionPrefix) - let config = makeValidationConfig( - strictMode: testCase.strictMode, - targetVersion: targetVersion - ) - let catalogs = try buildCatalogsWithAliases(for: testCase) - let handler = ValidatorErrorCaptureHandler() - let processor = MessageProcessor( - catalogs: catalogs, - actionHandler: handler, - validationConfig: config - ) + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runSingleValidatorCase(conformanceCase, versionPrefix: versionPrefix) + } + } + } - executedSteps += try executeValidationSteps( - for: testCase, - processor: processor, - handler: handler + private func runSingleValidatorCase( + _ conformanceCase: ConformanceCase, + versionPrefix: String? + ) throws { + try validateCaseStructure(conformanceCase) + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse validator case" + ) + guard testCase.action == "validate" else { + throw UnknownConformanceKeyError( + "[\(conformanceCase.name)] Unknown validator action '\(testCase.action)'" ) } + let targetVersion = resolveTargetVersion(for: testCase, versionPrefix: versionPrefix) + let config = makeValidationConfig( + strictMode: testCase.strictMode, + targetVersion: targetVersion + ) + let catalogs = try buildCatalogsWithAliases(for: testCase) + let handler = ValidatorErrorCaptureHandler() + let processor = MessageProcessor( + catalogs: catalogs, + actionHandler: handler, + validationConfig: config + ) - #expect(executedSteps > 0, "no step of \(filename) was executed") + _ = try executeValidationSteps( + for: testCase, + processor: processor, + handler: handler + ) } - private func matchesSuiteFilter(_ testCase: ConformanceTestCase, versionPrefix: String?) -> Bool { - guard testCase.action == "validate" else { return false } - guard let versionPrefix else { return true } - if let pv = testCase.protocolVersion { - if versionPrefix == "v1.0" || versionPrefix == "1.0" { - return pv.hasPrefix("1.0") || pv.hasPrefix("v1.0") + private func validateCaseStructure(_ conformanceCase: ConformanceCase) throws { + let rawDict = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + rawDict, + Self.knownCaseKeys, + "\(conformanceCase.suite) case '\(conformanceCase.name)'" + ) + if let errDict = rawDict["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + errDict, + Self.knownExpectErrorKeys, + "\(conformanceCase.suite) expectError in '\(conformanceCase.name)'", + allowMetadata: false + ) + } + if let steps = rawDict["steps"] as? [[String: Any]] { + for step in steps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownStepKeys, + "\(conformanceCase.suite) step in '\(conformanceCase.name)'", + allowMetadata: false + ) + if let stepErr = step["expectError"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + stepErr, + Self.knownExpectErrorKeys, + "\(conformanceCase.suite) step.expectError in '\(conformanceCase.name)'", + allowMetadata: false + ) + } } - return pv.hasPrefix(versionPrefix) } - return true } private func resolveTargetVersion( @@ -138,7 +208,7 @@ struct ValidatorConformanceTests { testName: testCase.name, stepIndex: stepIndex ) - assertErrorMatches( + try assertErrorMatches( error: error, capturedRendererError: handler.capturedErrors.last, expected: expectedError, @@ -275,40 +345,55 @@ struct ValidatorConformanceTests { } @Test func compositionConstraintsConformance() throws { - let rawYaml = try ConformanceTestHelper.loadYAML(filename: "core/composition_constraints.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYaml) - #expect(!testCases.isEmpty, "Should find test cases in core/composition_constraints.yaml") - - for testCase in testCases { - for testCase in testCases { - let catalogs = buildCompositionCatalogs(for: testCase) - let handler = ValidatorErrorCaptureHandler() - let validationConfig: ValidationConfig - if let flags = testCase.validationConfig { - guard - let config = ConformanceTestHelper.validationConfig( - from: flags, targetVersion: "1.0") - else { - Issue.record("[\(testCase.name)] Missing or unknown validationConfig flags") - continue - } - validationConfig = config - } else { - validationConfig = ValidationConfig(targetVersion: "1.0") + let cases = ConformanceTestHelper.cases("core/composition_constraints.yaml") + #expect(!cases.isEmpty, "Should find test cases in core/composition_constraints.yaml") + + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runSingleCompositionCase(conformanceCase) } - let processor = MessageProcessor( - catalogs: catalogs, - actionHandler: handler, - validationConfig: validationConfig - ) + } + } - _ = try executeValidationSteps( - for: testCase, - processor: processor, - handler: handler + private func runSingleCompositionCase(_ conformanceCase: ConformanceCase) throws { + try validateCaseStructure(conformanceCase) + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse composition case" + ) + guard testCase.action == "process_messages" || testCase.action == "validate" else { + throw UnknownConformanceKeyError( + "[\(conformanceCase.name)] Unknown composition action '\(testCase.action ?? "")'" ) } - } + let catalogs = buildCompositionCatalogs(for: testCase) + let handler = ValidatorErrorCaptureHandler() + let validationConfig: ValidationConfig + if let flags = testCase.validationConfig { + guard + let config = ConformanceTestHelper.validationConfig( + from: flags, + targetVersion: "1.0" + ) + else { + Issue.record("[\(testCase.name)] Missing or unknown validationConfig flags") + return + } + validationConfig = config + } else { + validationConfig = ValidationConfig(targetVersion: "1.0") + } + let processor = MessageProcessor( + catalogs: catalogs, + actionHandler: handler, + validationConfig: validationConfig + ) + + _ = try executeValidationSteps( + for: testCase, + processor: processor, + handler: handler + ) } private func buildCompositionCatalogs(for testCase: ConformanceTestCase) -> [AnyCatalog] { @@ -331,28 +416,42 @@ struct ValidatorConformanceTests { } @Test func reservedKeysConformance() throws { - let rawYAML = try ConformanceTestHelper.loadYAML(filename: "core/reserved_keys.yaml") - let testCases = ConformanceTestHelper.parseTestCases(from: rawYAML) - #expect(!testCases.isEmpty, "Should find test cases in core/reserved_keys.yaml") + let cases = ConformanceTestHelper.cases("core/reserved_keys.yaml") + #expect(!cases.isEmpty, "Should find test cases in core/reserved_keys.yaml") - var executed = 0 - for testCase in testCases { - let catalogs = try buildCatalogsWithAliases(for: testCase) - let handler = ValidatorErrorCaptureHandler() - let processor = MessageProcessor( - catalogs: catalogs, - actionHandler: handler, - validationConfig: ValidationConfig(targetVersion: "1.0") - ) + for conformanceCase in cases { + try ConformanceManifest.run(conformanceCase) { + try runSingleReservedKeysCase(conformanceCase) + } + } + } - executed += try executeValidationSteps( - for: testCase, - processor: processor, - handler: handler, - verifyExpectedSurfaces: true + private func runSingleReservedKeysCase(_ conformanceCase: ConformanceCase) throws { + try validateCaseStructure(conformanceCase) + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse reserved_keys case" + ) + guard testCase.action == "process_messages" || testCase.action == "validate" else { + throw UnknownConformanceKeyError( + "[\(conformanceCase.name)] Unknown reserved_keys action '\(testCase.action ?? "")'" ) } - #expect(executed == testCases.count, "All cases in reserved_keys.yaml should execute") + let catalogs = try buildCatalogsWithAliases(for: testCase) + let handler = ValidatorErrorCaptureHandler() + let processor = MessageProcessor( + catalogs: catalogs, + actionHandler: handler, + validationConfig: ValidationConfig(targetVersion: "1.0") + ) + + let executed = try executeValidationSteps( + for: testCase, + processor: processor, + handler: handler, + verifyExpectedSurfaces: true + ) + #expect(executed > 0, "[\(testCase.name)] Expected at least one step to execute") } private func assertReservedKeysSurfaces( @@ -404,9 +503,9 @@ struct ValidatorConformanceTests { capturedRendererError: RendererError?, expected: ConformanceExpectError, testName: String - ) { + ) throws { if let category = expected.category { - assertErrorCategory(error: error, category: category, testName: testName) + try assertErrorCategory(error: error, category: category, testName: testName) } if let expectedCode = expected.code { assertErrorCode( @@ -451,7 +550,7 @@ struct ValidatorConformanceTests { } } - private func assertErrorCategory(error: Error, category: String, testName: String) { + private func assertErrorCategory(error: Error, category: String, testName: String) throws { switch category { case "ValidationError": #expect( @@ -474,7 +573,8 @@ struct ValidatorConformanceTests { "[\(testName)] Expected A2UICatalogError for '\(category)', got \(type(of: error))" ) default: - break + Issue.record("[\(testName)] Unknown error category '\(category)'") + throw UnknownConformanceKeyError("[\(testName)] Unknown error category '\(category)'") } } From 0548ed4da9a088d836aee338278f27af2de79ce2 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 18:18:16 +0000 Subject: [PATCH 10/48] test(swift-core): run each conformance case as its own test --- .../AccessibilityConformanceTests.swift | 12 ++-- .../ActionsConformanceTests.swift | 24 +++---- .../BasicCatalogConformanceTests.swift | 66 +++++++---------- .../CatalogConformanceTests.swift | 32 ++++----- .../CatalogDeriveConformanceTests.swift | 20 ++---- .../DataModelConformanceTests.swift | 12 ++-- .../DataModelPointerConformanceTests.swift | 49 ++++++------- .../ExpressionsConformanceTests.swift | 70 +++++++++---------- .../MessageProcessorConformanceTests.swift | 50 +++++-------- .../MultiCatalogConformanceTests.swift | 12 ++-- .../NodeResolutionConformanceTests.swift | 18 ++--- .../RPCConformanceTests.swift | 12 ++-- .../SuiteRegistryTests.swift | 25 +++++++ .../ValidatorConformanceTests.swift | 54 ++++++-------- 14 files changed, 195 insertions(+), 261 deletions(-) diff --git a/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift index 9d51b516b2..7ccf7440e7 100644 --- a/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/AccessibilityConformanceTests.swift @@ -60,14 +60,10 @@ struct AccessibilityConformanceTests { "messages", ] - @Test func accessibilityConformance() throws { - let cases = ConformanceTestHelper.cases("core/accessibility.yaml") - #expect(!cases.isEmpty, "Should find test cases in accessibility.yaml") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runSingleAccessibilityCase(conformanceCase) - } + @Test(arguments: ConformanceTestHelper.cases("core/accessibility.yaml")) + func accessibilityConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runSingleAccessibilityCase(conformanceCase) } } diff --git a/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift index 46e06ecc2b..14993fc769 100644 --- a/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift @@ -138,21 +138,17 @@ struct ActionsConformanceTests { "userMessage", ] - @Test func actionsConformance() throws { - let cases = ConformanceTestHelper.cases("core/actions.yaml") - #expect(!cases.isEmpty, "core/actions.yaml should hold test cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - let testCase = conformanceCase.rawDictionary - let action = testCase["action"] as? String ?? "" - guard action == "dispatch_action" else { - throw UnknownConformanceKeyError( - "[\(conformanceCase.name)] Unknown actions.yaml action '\(action)'" - ) - } - try runDispatchActionCase(testCase, name: conformanceCase.name) + @Test(arguments: ConformanceTestHelper.cases("core/actions.yaml")) + func actionsConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + let testCase = conformanceCase.rawDictionary + let action = testCase["action"] as? String ?? "" + guard action == "dispatch_action" else { + throw UnknownConformanceKeyError( + "[\(conformanceCase.name)] Unknown actions.yaml action '\(action)'" + ) } + try runDispatchActionCase(testCase, name: conformanceCase.name) } } diff --git a/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift index fa13c86b79..ea13e778e6 100644 --- a/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift @@ -120,23 +120,19 @@ struct BasicCatalogConformanceTests { #expect(result == .string("Welcome Alice, you are 30 years old.")) } - @Test func evaluateFunctionConformance() throws { - let cases = ConformanceTestHelper.cases("core/functions.yaml", action: "evaluate_function") - #expect(!cases.isEmpty, "core/functions.yaml should hold evaluate_function cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - let testCase = conformanceCase.rawDictionary - try ConformanceTestHelper.assertKnownKeys( - testCase, - Self.knownEvaluateFunctionCaseKeys, - "functions.yaml evaluate_function case '\(conformanceCase.name)'" - ) - if let surfaceDict = testCase["surface"] as? [String: Any] { - try runSurfaceEvaluateFunctionCase(testCase, surfaceDict: surfaceDict) - } else { - try runDirectEvaluateFunctionCase(testCase) - } + @Test(arguments: ConformanceTestHelper.cases("core/functions.yaml", action: "evaluate_function")) + func evaluateFunctionConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + let testCase = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownEvaluateFunctionCaseKeys, + "functions.yaml evaluate_function case '\(conformanceCase.name)'" + ) + if let surfaceDict = testCase["surface"] as? [String: Any] { + try runSurfaceEvaluateFunctionCase(testCase, surfaceDict: surfaceDict) + } else { + try runDirectEvaluateFunctionCase(testCase) } } } @@ -385,25 +381,17 @@ struct BasicCatalogConformanceTests { ) } - @Test func functionsValidateConformance() throws { - let cases = ConformanceTestHelper.cases("core/functions.yaml", action: "validate") - #expect(!cases.isEmpty, "core/functions.yaml should hold validate cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runValidatedCatalogCase(conformanceCase, suiteName: "functions.yaml") - } + @Test(arguments: ConformanceTestHelper.cases("core/functions.yaml", action: "validate")) + func functionsValidateConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runValidatedCatalogCase(conformanceCase, suiteName: "functions.yaml") } } - @Test func indexFunctionConformance() throws { - let cases = ConformanceTestHelper.cases("core/index_function.yaml", action: "validate") - #expect(!cases.isEmpty, "Should load test cases from index_function.yaml") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runValidatedCatalogCase(conformanceCase, suiteName: "index_function.yaml") - } + @Test(arguments: ConformanceTestHelper.cases("core/index_function.yaml", action: "validate")) + func indexFunctionConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runValidatedCatalogCase(conformanceCase, suiteName: "index_function.yaml") } } @@ -598,14 +586,10 @@ struct BasicCatalogConformanceTests { } } - @Test func validationResultConformance() throws { - let cases = ConformanceTestHelper.cases("core/validation_result.yaml", action: "validate") - #expect(!cases.isEmpty, "Should load test cases from validation_result.yaml") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runValidationResultCase(conformanceCase) - } + @Test(arguments: ConformanceTestHelper.cases("core/validation_result.yaml", action: "validate")) + func validationResultConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runValidationResultCase(conformanceCase) } } diff --git a/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift index 57ee80381a..878b229bfa 100644 --- a/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/CatalogConformanceTests.swift @@ -65,24 +65,20 @@ struct CatalogConformanceTests { "message", ] - @Test func catalogConformance() throws { - let cases = ConformanceTestHelper.cases("core/catalog.yaml", actions: Self.supportedActions) - #expect(!cases.isEmpty, "core/catalog.yaml should hold test cases") - - for conformanceCase in cases { - let testCase = conformanceCase.rawDictionary - let version = testCase["protocolVersion"] as? String - let skippedReason = - version == "v0.8" ? "protocol version v0.8 is not supported by A2UICore" : nil - try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { - try ConformanceTestHelper.assertKnownKeys( - testCase, - Self.knownCaseKeys, - "catalog.yaml case '\(conformanceCase.name)'" - ) - let action = testCase["action"] as? String ?? "" - try dispatchCatalogCase(testCase, action: action) - } + @Test(arguments: ConformanceTestHelper.cases("core/catalog.yaml", actions: Self.supportedActions)) + func catalogConformance(_ conformanceCase: ConformanceCase) throws { + let testCase = conformanceCase.rawDictionary + let version = testCase["protocolVersion"] as? String + let skippedReason = + version == "v0.8" ? "protocol version v0.8 is not supported by A2UICore" : nil + try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { + try ConformanceTestHelper.assertKnownKeys( + testCase, + Self.knownCaseKeys, + "catalog.yaml case '\(conformanceCase.name)'" + ) + let action = testCase["action"] as? String ?? "" + try dispatchCatalogCase(testCase, action: action) } } diff --git a/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift index 16b9581cbc..34e9576768 100644 --- a/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/CatalogDeriveConformanceTests.swift @@ -25,7 +25,7 @@ import Testing /// check `absentKeys`. struct CatalogDeriveConformanceTests { /// Cases that check a catalog field the Swift catalog does not have. - private static let skippedCases: [String: String] = [ + static let skippedCases: [String: String] = [ // `derive` takes no instructions, since a Swift catalog has none. "test_v10_catalog_derive_sets_instructions": "Swift Catalog does not store agent instructions" @@ -67,19 +67,11 @@ struct CatalogDeriveConformanceTests { remoteSchema: A2UICommonSchema.allSchemas ) - @Test func catalogDeriveConformance() throws { - let cases = ConformanceTestHelper.cases("core/catalog.yaml", action: "derive") - #expect(!cases.isEmpty, "core/catalog.yaml should hold derive cases") - let caseNames = Set(cases.map(\.name)) - for skippedName in Self.skippedCases.keys { - #expect(caseNames.contains(skippedName), "Unused skip entry '\(skippedName)'") - } - - for conformanceCase in cases { - let skippedReason = Self.skippedCases[conformanceCase.name] - try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { - try runDeriveCase(conformanceCase.json, name: conformanceCase.name) - } + @Test(arguments: ConformanceTestHelper.cases("core/catalog.yaml", action: "derive")) + func catalogDeriveConformance(_ conformanceCase: ConformanceCase) throws { + let skippedReason = Self.skippedCases[conformanceCase.name] + try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { + try runDeriveCase(conformanceCase.json, name: conformanceCase.name) } } diff --git a/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift index bfcd621597..e44f494929 100644 --- a/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/DataModelConformanceTests.swift @@ -68,14 +68,10 @@ struct DataModelConformanceTests { } } - @Test func dataModelConformance() throws { - let cases = ConformanceTestHelper.cases("core/data_model.yaml") - #expect(!cases.isEmpty, "core/data_model.yaml should hold test cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runDataModelCase(conformanceCase.rawDictionary, name: conformanceCase.name) - } + @Test(arguments: ConformanceTestHelper.cases("core/data_model.yaml")) + func dataModelConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runDataModelCase(conformanceCase.rawDictionary, name: conformanceCase.name) } } diff --git a/swift/core/Tests/A2UIConformanceTests/DataModelPointerConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/DataModelPointerConformanceTests.swift index 421d857b2c..3662f40ad9 100644 --- a/swift/core/Tests/A2UIConformanceTests/DataModelPointerConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/DataModelPointerConformanceTests.swift @@ -50,22 +50,32 @@ struct DataModelPointerConformanceTests { "path", ] - @Test func dataModelPointersConformance() throws { - try runPayloadDataModelSuite(filename: "core/data_model_pointers.yaml") + @Test(arguments: ConformanceTestHelper.cases("core/data_model_pointers.yaml")) + func dataModelPointersConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runPayloadDataModelCase( + conformanceCase.rawDictionary, + name: conformanceCase.name, + suite: "core/data_model_pointers.yaml" + ) + } } - @Test func dataDeletionConformance() throws { - try runPayloadDataModelSuite(filename: "core/data_deletion.yaml") + @Test(arguments: ConformanceTestHelper.cases("core/data_deletion.yaml")) + func dataDeletionConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runPayloadDataModelCase( + conformanceCase.rawDictionary, + name: conformanceCase.name, + suite: "core/data_deletion.yaml" + ) + } } - @Test func dataContextPathResolutionConformance() throws { - let cases = ConformanceTestHelper.cases("core/data_context.yaml") - #expect(!cases.isEmpty, "core/data_context.yaml should hold test cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runDataContextCase(conformanceCase.rawDictionary, name: conformanceCase.name) - } + @Test(arguments: ConformanceTestHelper.cases("core/data_context.yaml")) + func dataContextPathResolutionConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runDataContextCase(conformanceCase.rawDictionary, name: conformanceCase.name) } } @@ -93,21 +103,6 @@ struct DataModelPointerConformanceTests { #expect(actual == expected, "\(name): got \(actual), expected \(expected)") } - private func runPayloadDataModelSuite(filename: String) throws { - let cases = ConformanceTestHelper.cases(filename) - #expect(!cases.isEmpty, "\(filename) should hold test cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runPayloadDataModelCase( - conformanceCase.rawDictionary, - name: conformanceCase.name, - suite: filename - ) - } - } - } - private func runPayloadDataModelCase( _ testCase: [String: Any], name: String, diff --git a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift index 6dc0583494..9989ce5750 100644 --- a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift @@ -46,17 +46,15 @@ struct ExpressionsConformanceTests { private let parser = ExpressionParser() - @Test func expressionsConformance() throws { - let cases = ConformanceTestHelper.cases( + @Test( + arguments: ConformanceTestHelper.cases( "core/expressions.yaml", action: "parse_expression_template" ) - #expect(!cases.isEmpty, "core/expressions.yaml should hold parse_expression_template cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runParseExpressionCase(conformanceCase.rawDictionary, name: conformanceCase.name) - } + ) + func expressionsConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runParseExpressionCase(conformanceCase.rawDictionary, name: conformanceCase.name) } } @@ -129,42 +127,38 @@ struct ExpressionsConformanceTests { /// Runs the `validate` cases of `core/expressions.yaml`, which render `formatString` templates /// on a v1.0 surface and compare the resolved component properties. @MainActor - @Test func expressionsValidateConformance() throws { - let cases = ConformanceTestHelper.cases("core/expressions.yaml", action: "validate") - #expect(!cases.isEmpty, "core/expressions.yaml should hold validate cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - let rawDict = conformanceCase.rawDictionary - try ConformanceTestHelper.assertKnownKeys( - rawDict, - Self.knownValidateCaseKeys, - "expressions.yaml validate case '\(conformanceCase.name)'" - ) - if let steps = rawDict["steps"] as? [[String: Any]] { - for step in steps { + @Test(arguments: ConformanceTestHelper.cases("core/expressions.yaml", action: "validate")) + func expressionsValidateConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + let rawDict = conformanceCase.rawDictionary + try ConformanceTestHelper.assertKnownKeys( + rawDict, + Self.knownValidateCaseKeys, + "expressions.yaml validate case '\(conformanceCase.name)'" + ) + if let steps = rawDict["steps"] as? [[String: Any]] { + for step in steps { + try ConformanceTestHelper.assertKnownKeys( + step, + Self.knownValidateStepKeys, + "expressions.yaml step in '\(conformanceCase.name)'", + allowMetadata: false + ) + if let errDict = step["expectError"] as? [String: Any] { try ConformanceTestHelper.assertKnownKeys( - step, - Self.knownValidateStepKeys, - "expressions.yaml step in '\(conformanceCase.name)'", + errDict, + Self.knownExpectErrorKeys, + "expressions.yaml step.expectError in '\(conformanceCase.name)'", allowMetadata: false ) - if let errDict = step["expectError"] as? [String: Any] { - try ConformanceTestHelper.assertKnownKeys( - errDict, - Self.knownExpectErrorKeys, - "expressions.yaml step.expectError in '\(conformanceCase.name)'", - allowMetadata: false - ) - } } } - let testCase = try #require( - conformanceCase.parsedTestCase(), - "[\(conformanceCase.name)] Could not parse validate case" - ) - try runValidateCase(testCase) } + let testCase = try #require( + conformanceCase.parsedTestCase(), + "[\(conformanceCase.name)] Could not parse validate case" + ) + try runValidateCase(testCase) } } diff --git a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift index 4b3d599187..99bc5d9108 100644 --- a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift @@ -37,7 +37,7 @@ private final class ConformanceActionCaptureHandler: ActionHandling, @MainActor struct MessageProcessorConformanceTests { /// `process_messages` cases whose payload is accepted but reports an error when it renders. - private static let expectedRuntimeErrors: [String: ExpectedRuntimeError] = [ + static let expectedRuntimeErrors: [String: ExpectedRuntimeError] = [ "test_v10_call_to_undeclared_function_without_strict_mode_accepted": ExpectedRuntimeError( code: "EXPRESSION_ERROR", surfaceID: "s", @@ -48,7 +48,7 @@ struct MessageProcessorConformanceTests { /// Cases of the optional agent_renderer_app feature /// (blueprints/features/agent_renderer_app.blueprint.md), which passes a validation config to /// one call. This SDK has no agent, so it does not implement the feature. - private static let skippedCases: [String: String] = [ + static let skippedCases: [String: String] = [ "test_v10_call_validation_config_relaxes_strict_processor": "per-call validationConfig is part of the optional agent_renderer_app feature", "test_v10_call_validation_config_tightens_permissive_processor": @@ -113,47 +113,33 @@ struct MessageProcessorConformanceTests { ] /// A runtime error a case expects rendering to report through the action handler. - private struct ExpectedRuntimeError { + struct ExpectedRuntimeError { let code: String let surfaceID: String let messageContains: String } - @Test func messageProcessorV09Conformance() throws { - try runMessageProcessorSuite( - filename: "core/message_processor_v0_9.yaml", - defaultVersion: "v0.9" - ) + @Test(arguments: ConformanceTestHelper.cases("core/message_processor_v0_9.yaml")) + func messageProcessorV09Conformance(_ conformanceCase: ConformanceCase) throws { + try runSingleMessageProcessorCase(conformanceCase, defaultVersion: "v0.9") } - @Test func messageProcessorV10Conformance() throws { - let cases = ConformanceTestHelper.cases("core/message_processor_v1_0.yaml") - let caseNames = Set(cases.map(\.name)) - for skippedName in Self.skippedCases.keys { - #expect(caseNames.contains(skippedName), "Unused skip entry '\(skippedName)'") - } - try runMessageProcessorSuite( - filename: "core/message_processor_v1_0.yaml", - defaultVersion: "1.0" - ) + @Test(arguments: ConformanceTestHelper.cases("core/message_processor_v1_0.yaml")) + func messageProcessorV10Conformance(_ conformanceCase: ConformanceCase) throws { + try runSingleMessageProcessorCase(conformanceCase, defaultVersion: "1.0") } - private func runMessageProcessorSuite( - filename: String, + private func runSingleMessageProcessorCase( + _ conformanceCase: ConformanceCase, defaultVersion: String ) throws { - let cases = ConformanceTestHelper.cases(filename) - #expect(!cases.isEmpty, "\(filename) should hold test cases") - - for conformanceCase in cases { - let skippedReason = Self.skippedCases[conformanceCase.name] - try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { - try dispatchMessageProcessorCase( - conformanceCase.rawDictionary, - name: conformanceCase.name, - defaultVersion: defaultVersion - ) - } + let skippedReason = Self.skippedCases[conformanceCase.name] + try ConformanceManifest.run(conformanceCase, skippedReason: skippedReason) { + try dispatchMessageProcessorCase( + conformanceCase.rawDictionary, + name: conformanceCase.name, + defaultVersion: defaultVersion + ) } } diff --git a/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift index fc11f4c34d..2b15a49f2a 100644 --- a/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift @@ -55,14 +55,10 @@ struct MultiCatalogConformanceTests { "message", ] - @Test func multiCatalogConformance() throws { - let cases = ConformanceTestHelper.cases("core/multi_catalog.yaml") - #expect(!cases.isEmpty, "core/multi_catalog.yaml should hold test cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try dispatchMultiCatalogCase(conformanceCase.rawDictionary, name: conformanceCase.name) - } + @Test(arguments: ConformanceTestHelper.cases("core/multi_catalog.yaml")) + func multiCatalogConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try dispatchMultiCatalogCase(conformanceCase.rawDictionary, name: conformanceCase.name) } } diff --git a/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift index fe30574afb..dc134eaef8 100644 --- a/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/NodeResolutionConformanceTests.swift @@ -117,19 +117,11 @@ struct NodeResolutionConformanceTests { "validationResults", ] - @Test func nodeResolutionConformance() throws { - let cases = ConformanceTestHelper.cases("core/node_resolution.yaml") - #expect(!cases.isEmpty, "core/node_resolution.yaml should hold test cases") - let caseNames = Set(cases.map(\.name)) - for knownIssueName in Self.placeholderKnownIssues.keys { - #expect(caseNames.contains(knownIssueName), "Unused known issue '\(knownIssueName)'") - } - - for conformanceCase in cases { - let knownIssueReason = Self.placeholderKnownIssues[conformanceCase.name] - try ConformanceManifest.run(conformanceCase, knownIssueReason: knownIssueReason) { - try runResolveNodesCase(conformanceCase.rawDictionary) - } + @Test(arguments: ConformanceTestHelper.cases("core/node_resolution.yaml")) + func nodeResolutionConformance(_ conformanceCase: ConformanceCase) throws { + let knownIssueReason = Self.placeholderKnownIssues[conformanceCase.name] + try ConformanceManifest.run(conformanceCase, knownIssueReason: knownIssueReason) { + try runResolveNodesCase(conformanceCase.rawDictionary) } } diff --git a/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift index 9f36e57695..323e47e383 100644 --- a/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/RPCConformanceTests.swift @@ -110,14 +110,10 @@ struct RPCConformanceTests { "rendererError", ] - @Test func rpcFunctionsConformance() async throws { - let cases = ConformanceTestHelper.cases("core/rpc_functions.yaml") - #expect(!cases.isEmpty, "Should load test cases from rpc_functions.yaml") - - for conformanceCase in cases { - try await ConformanceManifest.runAsync(conformanceCase) { - try await runSingleRPCCase(conformanceCase) - } + @Test(arguments: ConformanceTestHelper.cases("core/rpc_functions.yaml")) + func rpcFunctionsConformance(_ conformanceCase: ConformanceCase) async throws { + try await ConformanceManifest.runAsync(conformanceCase) { + try await runSingleRPCCase(conformanceCase) } } diff --git a/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift b/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift index eff428455c..f2c5fa78c3 100644 --- a/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift @@ -17,6 +17,7 @@ import Testing /// Verifies that every `conformance/core/*.yaml` suite and action is registered in /// `ConformanceTestHelper` and records skipped manifest entries for unsupported suites. +@MainActor struct SuiteRegistryTests { @Test func allCoreSuitesAndActionsAreRegistered() throws { let coreDir = ConformanceTestHelper.repoRoot @@ -45,9 +46,33 @@ struct SuiteRegistryTests { ) try verifySupportedSuiteActions() + verifyOverrideTablesAreUsed() recordUnsupportedSuiteSkips() } + @MainActor + private func verifyOverrideTablesAreUsed() { + let deriveNames = Set( + ConformanceTestHelper.cases("core/catalog.yaml", action: "derive").map(\.name)) + for skippedName in CatalogDeriveConformanceTests.skippedCases.keys { + #expect(deriveNames.contains(skippedName), "Unused skip entry '\(skippedName)'") + } + + let mpV10Names = Set( + ConformanceTestHelper.cases("core/message_processor_v1_0.yaml").map(\.name)) + for skippedName in MessageProcessorConformanceTests.skippedCases.keys { + #expect(mpV10Names.contains(skippedName), "Unused skip entry '\(skippedName)'") + } + for errorName in MessageProcessorConformanceTests.expectedRuntimeErrors.keys { + #expect(mpV10Names.contains(errorName), "Unused expectedRuntimeErrors entry '\(errorName)'") + } + + let nodeResNames = Set(ConformanceTestHelper.cases("core/node_resolution.yaml").map(\.name)) + for knownIssueName in NodeResolutionConformanceTests.placeholderKnownIssues.keys { + #expect(nodeResNames.contains(knownIssueName), "Unused known issue '\(knownIssueName)'") + } + } + private func verifySupportedSuiteActions() throws { for (suite, actionMap) in ConformanceTestHelper.coreSuiteActions.sorted(by: { $0.key < $1.key }) { diff --git a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift index 83365a8c72..252c10d8a0 100644 --- a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift @@ -65,26 +65,24 @@ struct ValidatorConformanceTests { "surfaceId", ] - @Test func validatorConformance() throws { - try runValidatorSuite(filename: "core/validator_v0_9.yaml", versionPrefix: "v0.9") - } - - @Test func validatorV10Conformance() throws { - try runValidatorSuite(filename: "core/validator_v1_0.yaml", versionPrefix: "1.0") + @Test(arguments: ConformanceTestHelper.cases("core/validator_v0_9.yaml")) + func validatorConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runSingleValidatorCase(conformanceCase, versionPrefix: "v0.9") + } } - @Test func errorResponsesConformance() throws { - try runValidatorSuite(filename: "core/error_responses.yaml", versionPrefix: nil) + @Test(arguments: ConformanceTestHelper.cases("core/validator_v1_0.yaml")) + func validatorV10Conformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runSingleValidatorCase(conformanceCase, versionPrefix: "1.0") + } } - private func runValidatorSuite(filename: String, versionPrefix: String?) throws { - let cases = ConformanceTestHelper.cases(filename) - #expect(!cases.isEmpty, "\(filename) should hold test cases") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runSingleValidatorCase(conformanceCase, versionPrefix: versionPrefix) - } + @Test(arguments: ConformanceTestHelper.cases("core/error_responses.yaml")) + func errorResponsesConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runSingleValidatorCase(conformanceCase, versionPrefix: nil) } } @@ -344,14 +342,10 @@ struct ValidatorConformanceTests { } } - @Test func compositionConstraintsConformance() throws { - let cases = ConformanceTestHelper.cases("core/composition_constraints.yaml") - #expect(!cases.isEmpty, "Should find test cases in core/composition_constraints.yaml") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runSingleCompositionCase(conformanceCase) - } + @Test(arguments: ConformanceTestHelper.cases("core/composition_constraints.yaml")) + func compositionConstraintsConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runSingleCompositionCase(conformanceCase) } } @@ -415,14 +409,10 @@ struct ValidatorConformanceTests { return catalogs } - @Test func reservedKeysConformance() throws { - let cases = ConformanceTestHelper.cases("core/reserved_keys.yaml") - #expect(!cases.isEmpty, "Should find test cases in core/reserved_keys.yaml") - - for conformanceCase in cases { - try ConformanceManifest.run(conformanceCase) { - try runSingleReservedKeysCase(conformanceCase) - } + @Test(arguments: ConformanceTestHelper.cases("core/reserved_keys.yaml")) + func reservedKeysConformance(_ conformanceCase: ConformanceCase) throws { + try ConformanceManifest.run(conformanceCase) { + try runSingleReservedKeysCase(conformanceCase) } } From 7691b275cd95a4edcaa7ff0a1be5c3b1c1d15ef4 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Sat, 31 Oct 2026 18:23:59 +0000 Subject: [PATCH 11/48] test(py-core): record executed conformance cases and enforce strict dispatch Have the Python core conformance harness emit executed-case manifest records when A2UI_CONFORMANCE_MANIFEST_DIR is set, validate every case and expectation block against known-key tables, verify that UNRUNNABLE_SUITES, UNSUPPORTED, and EXPECTED_FAILURES entries match loaded cases, and check the manifest in Python CI. --- .github/workflows/python_ci.yml | 8 + python/a2ui_core/tests/conftest.py | 118 +- .../a2ui_core/tests/node_resolution_runner.py | 133 +- python/a2ui_core/tests/test_conformance.py | 2770 +++++++++++------ 4 files changed, 2029 insertions(+), 1000 deletions(-) diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml index 100c68ab40..5a7ddb576f 100644 --- a/.github/workflows/python_ci.yml +++ b/.github/workflows/python_ci.yml @@ -110,7 +110,15 @@ jobs: run: | uv run --all-packages mypy . - name: Run Unit Tests + env: + A2UI_CONFORMANCE_MANIFEST_DIR: ${{ runner.temp }}/a2ui-manifest run: uv run pytest + - name: Verify Core Conformance Manifest + run: | + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --harness python-core \ + --corpus core - name: Set Up Dart uses: ./.github/actions/setup-dart - name: Run CLI Python Runtime Tests diff --git a/python/a2ui_core/tests/conftest.py b/python/a2ui_core/tests/conftest.py index e9b07ea757..4766c4986f 100644 --- a/python/a2ui_core/tests/conftest.py +++ b/python/a2ui_core/tests/conftest.py @@ -21,14 +21,24 @@ from __future__ import annotations +from collections.abc import Generator, Sequence import importlib.util -import sys +import json +import os from pathlib import Path +import sys from types import ModuleType +from typing import Any + +import pytest # The name the harness module is registered under in `sys.modules`. Other test # suites that load the harness use the same name, so it is loaded once. _HARNESS_MODULE = "a2ui_core_tests_timing_harness" +_CONFORMANCE_TEST_NAMES = frozenset({ + "test_conformance_suite", + "test_conformance_timing", +}) def _load_timing_harness() -> ModuleType: @@ -45,3 +55,109 @@ def _load_timing_harness() -> ModuleType: timing = _load_timing_harness().timing + + +def _extract_case_identity(item: pytest.Item) -> tuple[str, str] | None: + """Returns `(suite, case_name)` for a parameterized conformance item.""" + if getattr(item, "originalname", None) not in _CONFORMANCE_TEST_NAMES: + return None + callspec = getattr(item, "callspec", None) + params = getattr(callspec, "params", None) + if not isinstance(params, dict): + return None + rel_path = params.get("rel_path") + case = params.get("case") + if not isinstance(rel_path, str) or not isinstance(case, dict): + return None + case_name = case.get("name") + if not isinstance(case_name, str) or not case_name: + return None + return rel_path.replace("\\", "/"), case_name + + +def _append_manifest_record( + suite: str, case_name: str, status: str, reason: str | None = None +) -> None: + """Appends one executed-case record when `A2UI_CONFORMANCE_MANIFEST_DIR` is set.""" + manifest_dir = os.environ.get("A2UI_CONFORMANCE_MANIFEST_DIR") + if not manifest_dir: + return + os.makedirs(manifest_dir, exist_ok=True) + manifest_path = Path(manifest_dir) / f"python-core-{os.getpid()}.jsonl" + record: dict[str, str] = { + "harness": "python-core", + "suite": suite, + "case": case_name, + "status": status, + } + if reason: + record["reason"] = reason + with manifest_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record) + "\n") + + +def _extract_skip_reason(report: pytest.TestReport) -> str: + """Extracts a human-readable skip or xfail reason from a pytest TestReport.""" + wasxfail = getattr(report, "wasxfail", None) + if isinstance(wasxfail, str) and wasxfail.strip(): + return wasxfail.strip() + if wasxfail is not None: + return "expected failure" + longrepr = getattr(report, "longrepr", None) + if isinstance(longrepr, tuple) and len(longrepr) >= 3: + text = str(longrepr[2]).removeprefix("Skipped: ").strip() + return text or "skipped" + if longrepr: + return str(longrepr).removeprefix("Skipped: ").strip() or "skipped" + return "skipped" + + +def _report_status_and_reason( + report: pytest.TestReport, +) -> tuple[str, str | None]: + """Maps a pytest TestReport to `(status, reason)` for the manifest.""" + wasxfail = getattr(report, "wasxfail", None) + if wasxfail is not None: + if report.outcome == "skipped": + return "xfailed", _extract_skip_reason(report) + return "failed", None + if report.outcome == "skipped": + return "skipped", _extract_skip_reason(report) + if report.outcome == "passed": + return "passed", None + return "failed", None + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[Any] +) -> Generator[None, Any, None]: + """Records executed conformance test outcomes to the manifest JSONL file.""" + del call + outcome = yield + report: pytest.TestReport = outcome.get_result() + if report.when != "call" and not ( + report.when == "setup" and report.outcome != "passed" + ): + return + identity = _extract_case_identity(item) + if identity is None: + return + status, reason = _report_status_and_reason(report) + _append_manifest_record(identity[0], identity[1], status, reason) + + +def pytest_deselected(items: Sequence[pytest.Item]) -> None: + """Records deselected performance timing cases as skipped in the manifest.""" + for item in items: + if getattr(item, "originalname", None) != "test_conformance_timing": + continue + identity = _extract_case_identity(item) + if identity is None: + continue + _append_manifest_record( + identity[0], + identity[1], + "skipped", + "performance: runs in the performance job", + ) diff --git a/python/a2ui_core/tests/node_resolution_runner.py b/python/a2ui_core/tests/node_resolution_runner.py index 09f9b0e2c9..7c1fb3a84c 100644 --- a/python/a2ui_core/tests/node_resolution_runner.py +++ b/python/a2ui_core/tests/node_resolution_runner.py @@ -16,12 +16,14 @@ from __future__ import annotations +from collections.abc import Mapping import copy from dataclasses import dataclass import inspect import json from pathlib import Path from typing import Any, Callable + import yaml from a2ui.core.catalog import Catalog, FunctionImplementation @@ -35,6 +37,64 @@ ) from a2ui.core.state import ComponentModel, SurfaceModel +_CASE_KEYS = frozenset({ + "name", + "description", + "features", + "action", + "fixture", + "expect", + "steps", +}) +_FIXTURE_KEYS = frozenset({ + "catalog", + "components", + "data", +}) +_EXPECT_KEYS = frozenset({ + "nodes", + "emissions", + "metadata_emissions", + "destroyed", + "same_nodes", + "replaced_nodes", + "data", + "events", + "functions", + "errors", +}) +_NODE_FIELD_KEYS = frozenset({ + "component_id", + "type", + "state", + "data_path", + "metadata", + "isValid", + "validationErrors", + "validationResults", + "props", +}) +_STEP_KEYS_BY_OP: dict[str, frozenset[str]] = { + "set_data": frozenset({"op", "path", "value", "expect"}), + "update_components": frozenset({"op", "components", "expect"}), + "remove_component": frozenset({"op", "component_id", "expect"}), + "process_messages": frozenset({"op", "messages", "expect"}), + "write": frozenset({"op", "node", "property", "value", "expect"}), + "invoke": frozenset({"op", "node", "property", "metadata", "expect"}), + "dispose": frozenset({"op", "expect"}), +} + + +def _assert_known_keys( + obj: Mapping[str, Any], + allowed: frozenset[str], + where: str, + context: str, +) -> None: + for key in obj: + if key not in allowed: + raise AssertionError(f"Unknown {where} '{key}' in {context}") + @dataclass class _Watch: @@ -110,6 +170,7 @@ def _check_nodes( expected_nodes.keys() ), f"{reason}: node paths {sorted(nodes.keys())} != {sorted(expected_nodes.keys())}" for path, fields in expected_nodes.items(): + _assert_known_keys(fields, _NODE_FIELD_KEYS, f"node '{path}' key", reason) node = _required_node(nodes, path) actual = { "component_id": node.component_id, @@ -291,6 +352,22 @@ def _check_errors( ), f"{reason}: error.{k} expected {v!r}, got {actual_err.get(k)!r}" +def _check_node_identity( + before: dict[str, ComponentNode], + after: dict[str, ComponentNode], + expected: dict[str, Any], + reason: str, +) -> None: + for path in expected.get("same_nodes", ()): + assert _required_node(after, path) is _required_node( + before, path + ), f"{reason}: {path} identity" + for path in expected.get("replaced_nodes", ()): + assert _required_node(after, path) is not _required_node( + before, path + ), f"{reason}: {path} replacement" + + def _check_step_state( resolver: NodeResolver, surface: SurfaceModel, @@ -302,43 +379,36 @@ def _check_step_state( before: dict[str, ComponentNode], reason: str, ) -> None: + _assert_known_keys(expected, _EXPECT_KEYS, "expect key", reason) after = _mounted_nodes(resolver) _check_nodes(after, expected, reason) emissions, metadata_emissions, destroyed = _collect_watch_counts( watches, set(after.values()), reason ) - assert emissions == (expected.get("emissions") or {}), ( - f"{reason}: emissions expected {expected.get('emissions') or {}}, got" - f" {emissions}" - ) + exp_emissions = expected.get("emissions") or {} + assert ( + emissions == exp_emissions + ), f"{reason}: emissions expected {exp_emissions}, got {emissions}" if "metadata_emissions" in expected: - assert metadata_emissions == (expected.get("metadata_emissions") or {}), ( - f"{reason}: metadata_emissions expected" - f" {expected.get('metadata_emissions') or {}}, got {metadata_emissions}" - ) - assert destroyed == (expected.get("destroyed") or {}), ( - f"{reason}: destroyed expected {expected.get('destroyed') or {}}, got" - f" {destroyed}" - ) - for path in expected.get("same_nodes") or []: - assert _required_node(after, path) is _required_node( - before, path - ), f"{reason}: {path} identity" - for path in expected.get("replaced_nodes") or []: - assert _required_node(after, path) is not _required_node( - before, path - ), f"{reason}: {path} replacement" + exp_meta = expected.get("metadata_emissions") or {} + assert ( + metadata_emissions == exp_meta + ), f"{reason}: metadata_emissions expected {exp_meta}, got {metadata_emissions}" + exp_destroyed = expected.get("destroyed") or {} + assert ( + destroyed == exp_destroyed + ), f"{reason}: destroyed expected {exp_destroyed}, got {destroyed}" + _check_node_identity(before, after, expected, reason) for path, val in (expected.get("data") or {}).items(): assert ( _normalize(surface.data_model.get(path), path) == val ), f"{reason}: data {path}" - assert events == ( - expected.get("events") or [] - ), f"{reason}: events expected {expected.get('events') or []}, got {events}" - assert functions == (expected.get("functions") or []), ( - f"{reason}: functions expected {expected.get('functions') or []}, got" - f" {functions}" - ) + exp_events = expected.get("events") or [] + assert events == exp_events, f"{reason}: events expected {exp_events}, got {events}" + exp_functions = expected.get("functions") or [] + assert ( + functions == exp_functions + ), f"{reason}: functions expected {exp_functions}, got {functions}" _check_errors(errors, expected, reason) @@ -352,6 +422,10 @@ def _execute_step_op( reason: str, ) -> None: op = step["op"] + allowed_step_keys = _STEP_KEYS_BY_OP.get(op) + if allowed_step_keys is None: + raise AssertionError(f"Unknown node operation: {op}") + _assert_known_keys(step, allowed_step_keys, f"step ({op}) key", reason) if op == "set_data": surface.data_model.set(step["path"], step["value"]) elif op == "update_components": @@ -377,15 +451,16 @@ def _execute_step_op( asyncio.run(res) elif op == "dispose": resolver.dispose() - else: - raise AssertionError(f"Unknown node operation: {op}") def run_node_resolution_case(test_case: dict[str, Any], conformance_root: Path) -> None: """Executes a single resolve_nodes conformance test case.""" + case_context = f"core/node_resolution.yaml#{test_case.get('name', '')}" + _assert_known_keys(test_case, _CASE_KEYS, "case key", case_context) assert test_case.get("action") == "resolve_nodes" fixture_path = conformance_root / test_case["fixture"] fixture = yaml.safe_load(fixture_path.read_text(encoding="utf-8")) + _assert_known_keys(fixture, _FIXTURE_KEYS, "fixture key", case_context) functions: list[dict[str, Any]] = [] events: list[dict[str, Any]] = [] errors: list[dict[str, Any]] = [] diff --git a/python/a2ui_core/tests/test_conformance.py b/python/a2ui_core/tests/test_conformance.py index d0dbff2496..935bd140ae 100644 --- a/python/a2ui_core/tests/test_conformance.py +++ b/python/a2ui_core/tests/test_conformance.py @@ -12,51 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections.abc import Callable, Mapping import contextlib import glob import json import os +from pathlib import Path import re import sys -from pathlib import Path -from collections.abc import Callable from typing import Any + from pydantic import TypeAdapter import pytest import yaml -try: - from .node_resolution_runner import run_node_resolution_case -except ImportError: - import importlib.util as _importlib_util - - _nr_spec = _importlib_util.spec_from_file_location( - "node_resolution_runner", - os.path.join(os.path.dirname(__file__), "node_resolution_runner.py"), - ) - assert _nr_spec and _nr_spec.loader - _nr_mod = _importlib_util.module_from_spec(_nr_spec) - _nr_spec.loader.exec_module(_nr_mod) - run_node_resolution_case = _nr_mod.run_node_resolution_case -from a2ui.core.catalog import Catalog, CatalogApi from a2ui.core.basic_catalog import v0_8, v0_9, v1_0 +from a2ui.core.catalog import Catalog, CatalogApi from a2ui.core.common import to_canonical_version -from a2ui.core.schema import ProtocolVersion -from a2ui.core.schema.v0_9 import ClientToServerMessage -from a2ui.core.schema.v1_0 import RendererToAgentMessage -from a2ui.core.state import DataModel, SurfaceModel -from a2ui.core.resolution import DataContext -from a2ui.core.processing import ( - CapabilitiesOptions, - MessageProcessor, - MessageProcessorOptions, -) -from a2ui.core.validation import ( - RELAXED_VALIDATION, - STRICT_VALIDATION, - PayloadValidator, - ValidationConfig, -) from a2ui.core.exceptions import ( A2uiCatalogError, A2uiDataError, @@ -68,6 +40,36 @@ A2uiStateError, A2uiValidationError, ) +from a2ui.core.processing import ( + CapabilitiesOptions, + MessageProcessor, + MessageProcessorOptions, +) +from a2ui.core.resolution import DataContext +from a2ui.core.schema import ProtocolVersion +from a2ui.core.schema.v0_9 import ClientToServerMessage +from a2ui.core.schema.v1_0 import RendererToAgentMessage +from a2ui.core.state import DataModel, SurfaceModel +from a2ui.core.validation import ( + RELAXED_VALIDATION, + STRICT_VALIDATION, + PayloadValidator, + ValidationConfig, +) + +try: + from .node_resolution_runner import run_node_resolution_case +except ImportError: + import importlib.util as _importlib_util + + _nr_spec = _importlib_util.spec_from_file_location( + "node_resolution_runner", + os.path.join(os.path.dirname(__file__), "node_resolution_runner.py"), + ) + assert _nr_spec and _nr_spec.loader + _nr_mod = _importlib_util.module_from_spec(_nr_spec) + _nr_spec.loader.exec_module(_nr_mod) + run_node_resolution_case = _nr_mod.run_node_resolution_case CATEGORY_TO_EXCEPTION = { "ParseError": (A2uiParseError, A2uiExpressionError), @@ -91,28 +93,7 @@ "1.0", } -SKIP_TEST_NAMES: set[str] = { - # A known difference of the Python core: its topology checks read a v0.8 - # string property as a child reference when it names a component the - # surface holds, which the TypeScript and Kotlin cores do not. - "test_v08_topology_plain_string_property_not_treated_as_child_ref", - "test_v08_topology_plain_string_property_does_not_close_cycle", - # Cases of the optional agent_renderer_app feature - # (blueprints/features/agent_renderer_app.blueprint.md), which takes the - # validation rules for one call. This SDK has no renderer, so it does not - # implement the feature. - "test_v10_call_validation_config_relaxes_strict_processor", - "test_v10_call_validation_config_tightens_permissive_processor", - "test_v10_call_validation_config_leaves_out_rules_as_strict", -} - -# Transition skip list containing specific test suite files or basenames to skip entirely. -SKIP_TEST_SUITES: set[str] = set() - # Suites the core library cannot meaningfully execute, with the reason for each. -# The core library has no access to the UI frameworks that apply accessibility -# attributes, so running these here would only exercise mocks. The framework -# renderers will run them once the v1.0 catalogs land for those renderers. UNRUNNABLE_SUITES: dict[str, str] = { "core/accessibility.yaml": ( "Accessibility attributes are applied by the UI framework renderers" @@ -122,6 +103,67 @@ "core/adapter_conformance.yaml": "UI framework adapter suite", } +# Individual `#` entries that use an optional feature or key the +# Python core SDK does not implement. Checked before `_assert_known_keys` and +# recorded as `skipped` with their reason. +UNSUPPORTED: dict[str, str] = { + "core/message_processor_v1_0.yaml#test_v10_call_validation_config_relaxes_strict_processor": ( + "Per-call validationConfig on process_messages is an optional" + " renderer-app feature not implemented by the Python core SDK." + ), + "core/message_processor_v1_0.yaml#test_v10_call_validation_config_tightens_permissive_processor": ( + "Per-call validationConfig on process_messages is an optional" + " renderer-app feature not implemented by the Python core SDK." + ), + "core/message_processor_v1_0.yaml#test_v10_call_validation_config_leaves_out_rules_as_strict": ( + "Per-call validationConfig on process_messages is an optional" + " renderer-app feature not implemented by the Python core SDK." + ), +} + +# Individual `#` entries that exercise a real behavioural gap in +# the Python core SDK. Executed strictly and recorded as `xfailed`. +EXPECTED_FAILURES: dict[str, str] = { + "core/validator_v0_8.yaml#test_v08_topology_plain_string_property_not_treated_as_child_ref": ( + "Python core v0.8 topology checks read a string property as a child" + " reference when it names a component on the surface." + ), + "core/validator_v0_8.yaml#test_v08_topology_plain_string_property_does_not_close_cycle": ( + "Python core v0.8 topology checks read a string property as a child" + " reference when it names a component on the surface." + ), + "core/rpc_functions.yaml#test_rpc_ui_binding_triggers_call_agent_function_async_resolution": ( + "Python DataContext does not route undeclared component binding" + " function calls through MessageProcessor.call_agent_function." + ), + "core/rpc_functions.yaml#test_rpc_ui_action_missing_local_function_fallback_to_agent": ( + "Python DataContext.resolve_action does not fall back to" + " MessageProcessor.call_agent_function when a function is not" + " registered locally." + ), +} + +_CASE_METADATA_KEYS: frozenset[str] = frozenset({ + "name", + "description", + "features", +}) + +_CURRENT_CASE_KEY: str = "" + + +def _assert_known_keys( + obj: Mapping[str, Any], + allowed: frozenset[str], + where: str, + case_key: str | None = None, +) -> None: + target = case_key or _CURRENT_CASE_KEY or str(obj.get("name", "")) + for key in obj: + if key not in allowed: + pytest.fail(f"Unknown {where} '{key}' in {target}") + + # Actions that time an operation rather than check what it returns. Their cases # run in `test_conformance_timing`, which is marked `performance` and so runs # only under `uv run pytest -m performance` (see timing_harness.py). @@ -162,34 +204,46 @@ def find_yaml_files(dir_path: str) -> list[str]: return sorted(results) +def _protocol_version_from_catalogs_list(catalogs: Any) -> str | None: + if not isinstance(catalogs, list): + return None + for cat in catalogs: + if isinstance(cat, dict) and cat.get("protocolVersion"): + return str(cat["protocolVersion"]) + return None + + +def _protocol_version_from_catalog_id(cat_id: str) -> str: + if "v08" in cat_id or "v0_8" in cat_id: + return "v0.8" + if "v09" in cat_id or "v0_9" in cat_id: + return "v0.9" + if "v10" in cat_id or "v1_0" in cat_id or "v1.0" in cat_id: + return "1.0" + return "v0.9" + + def resolve_protocol_version(case: dict[str, Any]) -> str | None: - if "protocolVersion" in case and case["protocolVersion"]: - return case["protocolVersion"] + if case.get("protocolVersion"): + return str(case["protocolVersion"]) cat_spec = case.get("catalog") if isinstance(case.get("catalog"), dict) else {} - if "protocolVersion" in cat_spec and cat_spec["protocolVersion"]: - return cat_spec["protocolVersion"] + if cat_spec.get("protocolVersion"): + return str(cat_spec["protocolVersion"]) c_schema = ( case.get("catalogSchema") if isinstance(case.get("catalogSchema"), dict) else {} ) - if "protocolVersion" in c_schema and c_schema["protocolVersion"]: - return c_schema["protocolVersion"] - if "catalogs" in case and isinstance(case["catalogs"], list): - for cat in case["catalogs"]: - if isinstance(cat, dict) and cat.get("protocolVersion"): - return cat["protocolVersion"] + if c_schema.get("protocolVersion"): + return str(c_schema["protocolVersion"]) + from_list = _protocol_version_from_catalogs_list(case.get("catalogs")) + if from_list: + return from_list cat_id = str( case.get("catalogId") or cat_spec.get("catalogId") or c_schema.get("catalogId") or "" ) - if "v08" in cat_id or "v0_8" in cat_id: - return "v0.8" - if "v09" in cat_id or "v0_9" in cat_id: - return "v0.9" - if "v10" in cat_id or "v1_0" in cat_id or "v1.0" in cat_id: - return "1.0" - return "v0.9" + return _protocol_version_from_catalog_id(cat_id) def resolve_catalog_id(case: dict[str, Any]) -> str | None: @@ -218,10 +272,7 @@ def load_conformance_cases() -> list[tuple[str, str, dict[str, Any]]]: if not yaml_files: SUITE_LOAD_ERRORS.append(f"No conformance suites found under {CORE_DIR}") for file_path in yaml_files: - rel_path = os.path.relpath(file_path, CONFORMANCE_ROOT) - base_name = os.path.basename(file_path) - if rel_path in SKIP_TEST_SUITES or base_name in SKIP_TEST_SUITES: - continue + rel_path = os.path.relpath(file_path, CONFORMANCE_ROOT).replace(os.sep, "/") try: with open(file_path, "r", encoding="utf-8") as f: data = yaml.safe_load(f) @@ -243,264 +294,259 @@ def load_conformance_cases() -> list[tuple[str, str, dict[str, Any]]]: if not name: SUITE_LOAD_ERRORS.append(f"{rel_path}: entry {index} has no 'name'") continue - if name in SKIP_TEST_NAMES: - continue test_id = f"{rel_path}::{name}" cases.append((test_id, rel_path, case)) return cases -def get_catalogs_for_test_case(case: dict[str, Any]) -> list[Any]: - catalogs_map: dict[str, Any] = {} - - for cat in ALL_CATALOGS: - if hasattr(cat, "catalog_id"): - catalogs_map[cat.catalog_id] = cat - standard_catalog = Catalog( - catalog_id="standard", - protocol_version="v0.9", - components=list(basic_catalog.components.values()), - ) - catalogs_map["basic"] = basic_catalog - catalogs_map["standard"] = standard_catalog - catalogs_map["v0.8:basic"] = v08_catalog - catalogs_map["v0.9:basic"] = v09_catalog - catalogs_map["v1.0:basic"] = v10_catalog - - version = resolve_protocol_version(case) or "v0.9" - cur_basic = ( - v10_catalog - if version in ("1.0", "v1.0") - else (v08_catalog if version == "v0.8" else v09_catalog) - ) - v10_basic_alias = Catalog( - catalog_id="basic", - protocol_version="1.0", - components=list(v10_catalog.components.values()), - functions=list(v10_catalog.functions.values()), - ) - - if version in ("1.0", "v1.0"): - catalogs_map["basic"] = v10_basic_alias - - def add_catalog_id(cat_id: str, ver: str | None = None): - if cat_id and ( - cat_id not in catalogs_map - or not any( - getattr(c, "catalog_id", None) == cat_id for c in catalogs_map.values() - ) - ): - catalogs_map[cat_id] = Catalog( - catalog_id=cat_id, - protocol_version=ver or version, - components=list(cur_basic.components.values()), +def _build_inline_catalog( + spec: dict[str, Any], + c_id: str, + p_ver: str, + action: str | None, + forbid_unlisted_functions: bool = False, +) -> Any: + c_comps = spec.get("components") + c_theme = spec.get("theme") + c_funcs = spec.get("functions") + if c_comps or c_theme or c_funcs: + c_schema: dict[str, Any] = {"catalogId": c_id} + if forbid_unlisted_functions and not c_funcs: + c_schema["$defs"] = {"anyFunction": {"not": {}}} + if c_comps: + c_schema["components"] = c_comps + if c_theme: + c_schema["theme"] = c_theme + if c_funcs: + c_schema["functions"] = c_funcs + return Catalog.from_json(c_schema, catalog_id=c_id, protocol_version=p_ver) + is_cap = action in ("get_client_capabilities", "get_renderer_capabilities") + default_comps = [] if is_cap else list(basic_catalog.components.values()) + return Catalog(catalog_id=c_id, protocol_version=p_ver, components=default_comps) + + +def _load_single_catalog_spec( + case: dict[str, Any], + catalogs_map: dict[str, Any], + specified_catalogs: list[Any], +) -> None: + cat_spec = case.get("catalog") + if not isinstance(cat_spec, dict): + return + p_ver = resolve_protocol_version(case) or "v0.9" + if "catalogSchema" in cat_spec: + c_id = resolve_catalog_id(case) or f"catalog-{case.get('name')}" + if c_id: + cat = Catalog.from_json( + cat_spec["catalogSchema"], catalog_id=c_id, protocol_version=p_ver ) + catalogs_map[c_id] = cat + specified_catalogs.append(cat) + elif "components" in cat_spec or "catalogId" in cat_spec: + c_id = ( + cat_spec.get("catalogId") + or resolve_catalog_id(case) + or f"catalog-{case.get('name')}" + ) + cat = _build_inline_catalog( + cat_spec, + c_id, + p_ver, + case.get("action"), + forbid_unlisted_functions=True, + ) + catalogs_map[c_id] = cat + specified_catalogs.append(cat) - specified_catalogs: list[Any] = [] - if "catalog" in case and isinstance(case["catalog"], dict): - cat_spec = case["catalog"] - if "catalogSchema" in cat_spec: - c_schema = cat_spec["catalogSchema"] - c_id = resolve_catalog_id(case) or f"catalog-{case.get('name')}" - p_ver = resolve_protocol_version(case) or "v0.9" +def _load_catalogs_list_spec( + case: dict[str, Any], + version: str, + catalogs_map: dict[str, Any], + specified_catalogs: list[Any], +) -> None: + catalogs_list = case.get("catalogs") + if not isinstance(catalogs_list, list): + return + for item in catalogs_list: + if not isinstance(item, dict): + continue + if "catalogSchema" in item: + c_schema = item["catalogSchema"] + c_id = c_schema.get("catalogId") or item.get("catalogId") + p_ver = ( + item.get("protocolVersion") or c_schema.get("protocolVersion") or "v0.9" + ) if c_id: cat = Catalog.from_json( c_schema, catalog_id=c_id, protocol_version=p_ver ) catalogs_map[c_id] = cat specified_catalogs.append(cat) - elif "components" in cat_spec or "catalogId" in cat_spec: - c_id = ( - cat_spec.get("catalogId") - or resolve_catalog_id(case) - or f"catalog-{case.get('name')}" - ) - p_ver = resolve_protocol_version(case) or "v0.9" - c_comps = cat_spec.get("components") - c_theme = cat_spec.get("theme") - c_funcs = cat_spec.get("functions") - if c_comps or c_theme or c_funcs: - c_schema = {"catalogId": c_id} - if not c_funcs: - c_schema["$defs"] = {"anyFunction": {"not": {}}} - if c_comps: - c_schema["components"] = c_comps - if c_theme: - c_schema["theme"] = c_theme - if c_funcs: - c_schema["functions"] = c_funcs - cat = Catalog.from_json( - c_schema, - catalog_id=c_id, - protocol_version=p_ver, - ) - else: - default_comps = ( - [] - if case.get("action") - in ("get_client_capabilities", "get_renderer_capabilities") - else list(basic_catalog.components.values()) - ) - cat = Catalog( - catalog_id=c_id, - protocol_version=p_ver, - components=default_comps, - ) + elif "catalogId" in item: + c_id = item["catalogId"] + p_ver = item.get("protocolVersion") or version + cat = _build_inline_catalog(item, c_id, p_ver, case.get("action")) catalogs_map[c_id] = cat specified_catalogs.append(cat) - if "catalogs" in case and isinstance(case["catalogs"], list): - for item in case["catalogs"]: - if isinstance(item, dict): - if "catalogSchema" in item: - c_schema = item["catalogSchema"] - c_id = c_schema.get("catalogId") or item.get("catalogId") - p_ver = ( - item.get("protocolVersion") - or c_schema.get("protocolVersion") - or "v0.9" - ) - if c_id: - cat = Catalog.from_json( - c_schema, catalog_id=c_id, protocol_version=p_ver - ) - catalogs_map[c_id] = cat - specified_catalogs.append(cat) - elif "catalogId" in item: - c_id = item["catalogId"] - p_ver = item.get("protocolVersion") or version - c_comps = item.get("components") - c_theme = item.get("theme") - c_funcs = item.get("functions") - if c_comps or c_theme or c_funcs: - c_schema = {"catalogId": c_id} - if c_comps: - c_schema["components"] = c_comps - if c_theme: - c_schema["theme"] = c_theme - if c_funcs: - c_schema["functions"] = c_funcs - cat = Catalog.from_json( - c_schema, catalog_id=c_id, protocol_version=p_ver - ) - else: - default_comps = ( - [] - if case.get("action") - in ("get_client_capabilities", "get_renderer_capabilities") - else list(basic_catalog.components.values()) - ) - cat = Catalog( - catalog_id=c_id, - protocol_version=p_ver, - components=default_comps, - ) - catalogs_map[c_id] = cat - specified_catalogs.append(cat) - if "catalogPaths" in case and isinstance(case["catalogPaths"], list): - for p in case["catalogPaths"]: - full_p = os.path.abspath(os.path.join(CONFORMANCE_ROOT, "../", p)) - if not os.path.exists(full_p): - raise FileNotFoundError( - f"catalogPaths entry '{p}' does not exist (resolved to {full_p})" - ) - if os.path.exists(full_p): - with open(full_p, "r", encoding="utf-8") as f: - c_json = json.load(f) - c_id = c_json.get("catalogId") or c_json.get("id") or "test-catalog" - p_ver = resolve_protocol_version(case) or "v0.9" - if ( - c_id - in ( - v10_catalog.catalog_id, - v09_catalog.catalog_id, - v08_catalog.catalog_id, - ) - or "basic/catalog.json" in p - ): - matching_basic = ( - v10_catalog - if "1.0" in p_ver - else (v08_catalog if "0.8" in p_ver else v09_catalog) - ) - specified_catalogs.append(matching_basic) - catalogs_map[c_id] = matching_basic - else: - cat = Catalog.from_json( - c_json, catalog_id=c_id, protocol_version=p_ver - ) - specified_catalogs.append(cat) - if c_id != "test-catalog": - test_cat = Catalog.from_json( - c_json, - catalog_id="test-catalog", - protocol_version=p_ver, - ) - catalogs_map["test-catalog"] = test_cat - specified_catalogs.append(test_cat) - - messages: list[Any] = case.get("messages") or ( - [case["payload"]] if "payload" in case else [] - ) - if "steps" in case: - for step in case["steps"]: - msgs = step.get("messages") or step.get("payload") - if msgs: - if isinstance(msgs, list): - messages.extend(msgs) - elif isinstance(msgs, dict): - messages.append(msgs) - expect_err = case.get("expectError") - is_catalog_error_case = isinstance(expect_err, dict) and expect_err.get( - "category" - ) in ("CatalogError", "A2uiCatalogError") +def _basic_catalog_for_version(p_ver: str) -> Any: + if "1.0" in p_ver: + return v10_catalog + if "0.8" in p_ver: + return v08_catalog + return v09_catalog + + +def _load_catalog_paths_spec( + case: dict[str, Any], + catalogs_map: dict[str, Any], + specified_catalogs: list[Any], +) -> None: + paths = case.get("catalogPaths") + if not isinstance(paths, list): + return + basic_ids = { + v10_catalog.catalog_id, + v09_catalog.catalog_id, + v08_catalog.catalog_id, + } + p_ver = resolve_protocol_version(case) or "v0.9" + for p in paths: + full_p = os.path.abspath(os.path.join(CONFORMANCE_ROOT, "../", p)) + if not os.path.exists(full_p): + raise FileNotFoundError( + f"catalogPaths entry '{p}' does not exist (resolved to {full_p})" + ) + with open(full_p, "r", encoding="utf-8") as f: + c_json = json.load(f) + c_id = c_json.get("catalogId") or c_json.get("id") or "test-catalog" + if c_id in basic_ids or "basic/catalog.json" in p: + matching_basic = _basic_catalog_for_version(p_ver) + specified_catalogs.append(matching_basic) + catalogs_map[c_id] = matching_basic + continue + cat = Catalog.from_json(c_json, catalog_id=c_id, protocol_version=p_ver) + specified_catalogs.append(cat) + if c_id != "test-catalog": + test_cat = Catalog.from_json( + c_json, catalog_id="test-catalog", protocol_version=p_ver + ) + catalogs_map["test-catalog"] = test_cat + specified_catalogs.append(test_cat) + + +def _collect_case_messages(case: dict[str, Any]) -> list[Any]: + raw_msgs = case.get("messages") or ([case["payload"]] if "payload" in case else []) + messages: list[Any] = list(raw_msgs) if isinstance(raw_msgs, list) else [raw_msgs] + for step in case.get("steps") or []: + msgs = step.get("messages") or step.get("payload") + if isinstance(msgs, list): + messages.extend(msgs) + elif isinstance(msgs, dict): + messages.append(msgs) + return messages + + +def _record_message_catalog_ids( + item: dict[str, Any], + scan_version: str | None, + add_catalog_id: Callable[[str, str | None], None], +) -> None: + for key in ("createSurface", "beginRendering"): + sub = item.get(key) + if isinstance(sub, dict) and "catalogId" in sub: + add_catalog_id(sub["catalogId"], scan_version) + +def _scan_messages_for_catalog_ids( + messages: list[Any], + is_catalog_error_case: bool, + add_catalog_id: Callable[[str, str | None], None], +) -> None: scan_version: str | None = None + stack: list[Any] = list(reversed(messages)) + while stack: + item = stack.pop() + if isinstance(item, list): + stack.extend(reversed(item)) + continue + if not isinstance(item, dict): + continue + if isinstance(item.get("version"), str): + scan_version = item["version"] + if "messages" in item: + stack.append(item["messages"]) + if not is_catalog_error_case: + _record_message_catalog_ids(item, scan_version, add_catalog_id) + + +def _base_catalogs_map(version: str) -> dict[str, Any]: + catalogs_map: dict[str, Any] = { + cat.catalog_id: cat for cat in ALL_CATALOGS if hasattr(cat, "catalog_id") + } + catalogs_map["basic"] = basic_catalog + catalogs_map["standard"] = Catalog( + catalog_id="standard", + protocol_version="v0.9", + components=list(basic_catalog.components.values()), + ) + catalogs_map["v0.8:basic"] = v08_catalog + catalogs_map["v0.9:basic"] = v09_catalog + catalogs_map["v1.0:basic"] = v10_catalog + if version in ("1.0", "v1.0"): + catalogs_map["basic"] = Catalog( + catalog_id="basic", + protocol_version="1.0", + components=list(v10_catalog.components.values()), + functions=list(v10_catalog.functions.values()), + ) + return catalogs_map + + +def get_catalogs_for_test_case(case: dict[str, Any]) -> list[Any]: + version = resolve_protocol_version(case) or "v0.9" + cur_basic = _basic_catalog_for_version(version) + catalogs_map = _base_catalogs_map(version) - def scan(item: Any): - nonlocal scan_version - if not item: + def add_catalog_id(cat_id: str, ver: str | None = None) -> None: + has_exact_id = any( + getattr(c, "catalog_id", None) == cat_id for c in catalogs_map.values() + ) + if not cat_id or has_exact_id: return - if isinstance(item, list): - for sub in item: - scan(sub) - elif isinstance(item, dict): - if "version" in item and isinstance(item["version"], str): - scan_version = item["version"] - if "messages" in item: - scan(item["messages"]) - if not is_catalog_error_case: - if ( - "createSurface" in item - and isinstance(item["createSurface"], dict) - and "catalogId" in item["createSurface"] - ): - add_catalog_id(item["createSurface"]["catalogId"], scan_version) - if ( - "beginRendering" in item - and isinstance(item["beginRendering"], dict) - and "catalogId" in item["beginRendering"] - ): - add_catalog_id(item["beginRendering"]["catalogId"], scan_version) - - scan(messages) + catalogs_map[cat_id] = Catalog( + catalog_id=cat_id, + protocol_version=ver or version, + components=list(cur_basic.components.values()), + ) + + specified_catalogs: list[Any] = [] + _load_single_catalog_spec(case, catalogs_map, specified_catalogs) + _load_catalogs_list_spec(case, version, catalogs_map, specified_catalogs) + _load_catalog_paths_spec(case, catalogs_map, specified_catalogs) + + expect_err = case.get("expectError") + is_cat_err = isinstance(expect_err, dict) and expect_err.get("category") in ( + "CatalogError", + "A2uiCatalogError", + ) + _scan_messages_for_catalog_ids( + _collect_case_messages(case), is_cat_err, add_catalog_id + ) if ( case.get("action") in ("get_client_capabilities", "get_renderer_capabilities") and specified_catalogs ): return specified_catalogs - return ( - specified_catalogs - + [cur_basic] - + [ - c - for c in catalogs_map.values() - if c not in specified_catalogs and c != cur_basic - ] - ) + extra = [ + c + for c in catalogs_map.values() + if c not in specified_catalogs and c != cur_basic + ] + return specified_catalogs + [cur_basic] + extra def _error_message_matches(expected: str, actual: str) -> bool: @@ -563,11 +609,47 @@ def _renderer_error_path_matches( return False +_EXPECT_ERROR_KEYS: frozenset[str] = frozenset({ + "category", + "code", + "details", + "functionCallId", + "message", + "path", + "rendererError", + "surfaceId", +}) +_EXPECT_ERROR_DETAIL_KEYS: frozenset[str] = frozenset({ + "code", + "message", + "path", + "surfaceId", +}) +_RENDERER_ERROR_ENVELOPE_KEYS: frozenset[str] = frozenset({ + "version", + "error", +}) +_RENDERER_ERROR_BODY_KEYS: frozenset[str] = frozenset({ + "code", + "functionCallId", + "message", + "path", + "surfaceId", +}) + + def _assert_renderer_error( exp_re: dict[str, Any], captured_renderer_errors: list[dict[str, Any]] | None, err_details: list[Any] | None = None, ) -> None: + if "error" in exp_re: + _assert_known_keys(exp_re, _RENDERER_ERROR_ENVELOPE_KEYS, "rendererError key") + _assert_known_keys( + exp_re["error"], _RENDERER_ERROR_BODY_KEYS, "rendererError.error key" + ) + else: + _assert_known_keys(exp_re, _RENDERER_ERROR_BODY_KEYS, "rendererError key") assert ( captured_renderer_errors ), f"Expected rendererError {exp_re!r} to be emitted, but none was captured." @@ -654,6 +736,9 @@ def _assert_error_details_list( ) -> None: for exp_d in expected_details: if isinstance(exp_d, dict): + _assert_known_keys( + exp_d, _EXPECT_ERROR_DETAIL_KEYS, "expectError.details[] key" + ) assert any( _detail_matches_expected(d, exp_d) for d in err_details ), f"Expected detail {exp_d!r} not found in {err_details!r}" @@ -707,12 +792,56 @@ def _assert_error_dict_expectations( ) +def _assert_exception_code(exc: Any, expected_code: str) -> None: + err_details = getattr(exc, "details", []) + err_code = getattr(exc, "code", None) + in_details = bool(err_details) and any( + getattr(d, "code", None) == expected_code for d in err_details + ) + assert ( + err_code == expected_code or expected_code in str(exc) or in_details + ), f"Expected error code '{expected_code}' not found in exception '{exc}'" + + +def _exception_matches_expected_message(exc: Any, message: str) -> bool: + msg_norm = message.lower() + err_details = getattr(exc, "details", []) + detail_msgs = ( + " ".join(d.message for d in err_details).lower() if err_details else "" + ) + err_str = f"{str(exc).lower()} {detail_msgs}" + if _error_message_matches(message, err_str): + return True + if err_details and any(message in d.message for d in err_details): + return True + if re.search(message, str(exc)): + return True + return _fuzzy_error_message_matches(msg_norm, err_str) + + +def _fuzzy_error_message_matches(msg_norm: str, err_str: str) -> bool: + if "additional properties are not allowed" in msg_norm: + return "extra" in err_str + if "was unexpected" in msg_norm: + return "unrecognized" in err_str or "unexpected" in err_str + if "cannot contain child" in msg_norm: + return "cannot contain child" in err_str + if "cannot be placed under parent" in msg_norm: + return "cannot be placed under parent" in err_str + if "protocol version mismatch" in msg_norm or "mismatched protocol" in msg_norm: + return ( + "mismatched protocol" in err_str or "protocol version mismatch" in err_str + ) + return False + + @contextlib.contextmanager def assert_raises( expect_error: Any, captured_renderer_errors: list[dict[str, Any]] | None = None, ): if isinstance(expect_error, dict): + _assert_known_keys(expect_error, _EXPECT_ERROR_KEYS, "expectError key") category = expect_error.get("category") message = expect_error.get("message", "") expected_class = CATEGORY_TO_EXCEPTION.get(category, (A2uiError, ValueError)) @@ -724,60 +853,11 @@ def assert_raises( yield if isinstance(expect_error, dict) and expect_error.get("code"): - expected_code = expect_error["code"] - err_details = getattr(excinfo.value, "details", []) - err_code = getattr(excinfo.value, "code", None) - assert ( - err_code == expected_code - or expected_code in str(excinfo.value) - or ( - err_details - and any(getattr(d, "code", None) == expected_code for d in err_details) - ) - ), ( - f"Expected error code '{expected_code}' not found in exception" - f" '{excinfo.value}'" - ) + _assert_exception_code(excinfo.value, expect_error["code"]) if message: - msg_norm = message.lower() - err_details = getattr(excinfo.value, "details", []) - detail_msgs = ( - " ".join([d.message for d in err_details]).lower() if err_details else "" - ) - err_str = f"{str(excinfo.value).lower()} {detail_msgs}" - match = ( - _error_message_matches(message, err_str) - or (err_details and any(message in d.message for d in err_details)) - or re.search(message, str(excinfo.value)) - or ( - "additional properties are not allowed" in msg_norm - and ("extra inputs are not permitted" in err_str or "extra" in err_str) - ) - or ( - "was unexpected" in msg_norm - and ("unrecognized" in err_str or "unexpected" in err_str) - ) - or ( - "cannot contain child" in msg_norm and "cannot contain child" in err_str - ) - or ( - "cannot be placed under parent" in msg_norm - and "cannot be placed under parent" in err_str - ) - or ( - ( - "protocol version mismatch" in msg_norm - or "mismatched protocol" in msg_norm - ) - and ( - "mismatched protocol" in err_str - or "protocol version mismatch" in err_str - ) - ) - ) - assert ( - match + assert _exception_matches_expected_message( + excinfo.value, message ), f"Expected error '{message}' not found in exception '{excinfo.value}'" if isinstance(expect_error, dict): @@ -802,12 +882,50 @@ def test_all_suites_load() -> None: ) +def test_unsupported_and_expected_failures_are_used() -> None: + """Every UNRUNNABLE_SUITES, UNSUPPORTED, and EXPECTED_FAILURES entry must match a loaded case.""" + loaded_suites = {rel_path for _, rel_path, _ in ALL_CONFORMANCE_CASES} + loaded_keys = { + f"{rel_path}#{case.get('name')}" for _, rel_path, case in ALL_CONFORMANCE_CASES + } + unused_suites = sorted(set(UNRUNNABLE_SUITES) - loaded_suites) + assert not unused_suites, f"Unused UNRUNNABLE_SUITES entries: {unused_suites}" + unused_unsupported = sorted(set(UNSUPPORTED) - loaded_keys) + assert not unused_unsupported, f"Unused UNSUPPORTED entries: {unused_unsupported}" + unused_xfails = sorted(set(EXPECTED_FAILURES) - loaded_keys) + assert not unused_xfails, f"Unused EXPECTED_FAILURES entries: {unused_xfails}" + + +def _validate_resolve_nodes_case(case: dict[str, Any]) -> None: + run_node_resolution_case(case, Path(CONFORMANCE_ROOT)) + + +def _dispatch_conformance_action( + rel_path: str, action: str, case: dict[str, Any] +) -> None: + if action == "resolve_nodes": + assert ( + rel_path == "core/node_resolution.yaml" + ), "resolve_nodes is only supported in core/node_resolution.yaml" + handler = _ACTION_HANDLERS.get(action) + if handler is None: + pytest.fail( + f"Action '{action}' has no handler in the core Python harness." + " Add one, or add the suite to UNRUNNABLE_SUITES with a reason." + ) + handler(case) + + @pytest.mark.parametrize( "test_id, rel_path, case", CONFORMANCE_CASES, ids=[c[0] for c in CONFORMANCE_CASES], ) def test_conformance_suite(test_id: str, rel_path: str, case: dict[str, Any]) -> None: + global _CURRENT_CASE_KEY + case_key = f"{rel_path}#{case.get('name', '')}" + _CURRENT_CASE_KEY = case_key + ver = resolve_protocol_version(case) if ver and ver not in SUPPORTED_PROTOCOL_VERSIONS: pytest.fail( @@ -817,63 +935,36 @@ def test_conformance_suite(test_id: str, rel_path: str, case: dict[str, Any]) -> if rel_path in UNRUNNABLE_SUITES: pytest.skip(UNRUNNABLE_SUITES[rel_path]) + if case_key in UNSUPPORTED: + pytest.skip(UNSUPPORTED[case_key]) + action = case.get("action") - if not action: + if not action or not isinstance(action, str): pytest.fail(f"Test case '{test_id}' missing required 'action' field.") - if action == "from_json": - validate_from_json_case(case) - elif action == "derive": - validate_derive_case(case) - elif action == "validation_schema": - validate_validation_schema_case(case) - elif action == "round_trip": - validate_round_trip_case(case) - elif action == "to_json": - validate_to_json_case(case) - elif action == "common_types_schema": - validate_common_types_schema_case(case) - elif action == "agent_to_renderer_schema": - validate_agent_to_renderer_schema_case(case) - elif action == "validate_common_type": - validate_common_type_case(case) - elif action == "validate": - validate_pure_validation_case(case) - elif action == "process_messages": - validate_process_messages_case(case) - elif action in ( - "get_client_capabilities", - "get_renderer_capabilities", - ): - validate_capabilities_case(case) - elif action in ("get_renderer_data_model", "get_client_data_model"): - validate_get_renderer_data_model_case(case) - elif action == "resolve_path": - validate_resolve_path_case(case) - elif action == "data_model": - validate_data_model_case(case) - elif action == "handle_rpc": - validate_handle_rpc_case(case) - elif action == "select_catalog": - validate_select_catalog_case(case) - elif action == "accessibility_check": - validate_accessibility_check_case(case) - elif action == "parse_expression_template": - validate_parse_expression_template_case(case) - elif action == "resolve_nodes": - assert ( - rel_path == "core/node_resolution.yaml" - ), "resolve_nodes is only supported in core/node_resolution.yaml" - run_node_resolution_case(case, Path(CONFORMANCE_ROOT)) - elif action == "evaluate_function": - validate_evaluate_function_case(case) - elif action == "dispatch_action": - validate_dispatch_action_case(case) + if case_key in EXPECTED_FAILURES: + try: + _dispatch_conformance_action(rel_path, action, case) + except (Exception, pytest.fail.Exception): + pytest.xfail(EXPECTED_FAILURES[case_key]) + pytest.fail(f"Expected failure '{case_key}' passed unexpectedly.") else: - pytest.fail( - f"Action '{action}' has no handler in the core Python harness." - " Add one, or add the suite to UNRUNNABLE_SUITES with a reason." - ) + _dispatch_conformance_action(rel_path, action, case) + + +_MEASURE_GROWTH_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalogPaths", + "max_ratio", + "operation", + "payload", + "protocolVersion", + "sizes", +}) +_TIMING_ARGS_KEYS: frozenset[str] = frozenset({ + "validation_config", +}) def _timed_processing( @@ -885,6 +976,8 @@ def _timed_processing( Each timed run creates a processor and hands it every message in one `process_messages` call. """ + _assert_known_keys(case, _MEASURE_GROWTH_CASE_KEYS, "case key") + _assert_known_keys(case["args"], _TIMING_ARGS_KEYS, "args key") catalogs = [_load_catalog_path(p) for p in case["catalogPaths"]] config = {"none": None, "relaxed": RELAXED_VALIDATION}[ case["args"]["validation_config"] @@ -925,6 +1018,8 @@ def test_conformance_timing( test_id: str, rel_path: str, case: dict[str, Any], timing: Any ) -> None: """Runs a `measure_growth` case of `core/message_processor_scaling.yaml`.""" + global _CURRENT_CASE_KEY + _CURRENT_CASE_KEY = f"{rel_path}#{case.get('name', '')}" assert case["action"] == "measure_growth", test_id assert case["operation"] == "process_messages", test_id timing.measure_growth(case, _timed_processing(case)) @@ -1015,6 +1110,28 @@ def _extract_template_binding(p_val: Any) -> tuple[str, str] | None: return p_val["componentId"], raw_tpl_path +def _resolve_template_children( + surface: Any, + ctx: Any, + props: dict[str, Any], + resolve_one: Callable[[str, str, str], None], +) -> None: + for p_val in list(props.values()): + tpl_info = _extract_template_binding(p_val) + if tpl_info is None: + continue + tpl_comp_id, raw_tpl_path = tpl_info + tpl_path = ctx.data_context.resolve_path(raw_tpl_path) + arr = surface.data_model.get(tpl_path) + if not isinstance(arr, list): + continue + base_tpl_path = tpl_path.rstrip("/") + for i in range(len(arr)): + scoped_path = f"{base_tpl_path}/{i}" + resolve_one(tpl_comp_id, scoped_path, f"{tpl_comp_id}_{i}") + resolve_one(tpl_comp_id, scoped_path, f"{tpl_comp_id}-[{scoped_path}]") + + def _resolve_surface_components(surface: Any) -> dict[str, dict[str, Any]]: from a2ui.core.resolution import ComponentContext, GenericBinder @@ -1033,22 +1150,7 @@ def _resolve_one(comp_id: str, data_path: str, instance_key: str) -> None: props = dict(binder.current_props) binder.dispose() - for _, p_val in list(props.items()): - tpl_info = _extract_template_binding(p_val) - if tpl_info is not None: - tpl_comp_id, raw_tpl_path = tpl_info - tpl_path = ctx.data_context.resolve_path(raw_tpl_path) - arr = surface.data_model.get(tpl_path) - if isinstance(arr, list): - base_tpl_path = tpl_path.rstrip("/") - for i in range(len(arr)): - scoped_path = f"{base_tpl_path}/{i}" - _resolve_one(tpl_comp_id, scoped_path, f"{tpl_comp_id}_{i}") - _resolve_one( - tpl_comp_id, - scoped_path, - f"{tpl_comp_id}-[{scoped_path}]", - ) + _resolve_template_children(surface, ctx, props, _resolve_one) idx_suffix = _indexed_suffix_for_path(data_path) if idx_suffix: @@ -1069,80 +1171,119 @@ def _resolve_one(comp_id: str, data_path: str, instance_key: str) -> None: return resolved +_SURFACE_STATE_EXPECT_KEYS: frozenset[str] = frozenset({ + "surfaces", +}) +_SURFACE_EXPECT_KEYS: frozenset[str] = frozenset({ + "components", + "dataModel", + "exists", + "isValid", + "metadata", + "sendDataModel", + "theme", + "validationErrors", + "validationResult", + "validationResults", +}) + + +def _expected_component_items(comps_expected: Any) -> list[tuple[Any, Any]]: + if isinstance(comps_expected, dict): + return list(comps_expected.items()) + if isinstance(comps_expected, list): + return [(c.get("id"), c) for c in comps_expected if isinstance(c, dict)] + return [] + + +def _prop_value_matches(norm_val: Any, p_val: Any) -> bool: + if norm_val == p_val: + return True + return ( + isinstance(norm_val, int) + and not isinstance(norm_val, bool) + and isinstance(p_val, str) + and str(norm_val) == p_val + ) + + +def _assert_single_component_state( + s_id: str, + c_id: Any, + c_exp: Any, + surface: Any, + resolved_nodes: dict[str, dict[str, Any]], +) -> None: + comp = surface.components_model.get(c_id) + node_info = resolved_nodes.get(c_id) + assert ( + comp is not None or node_info is not None + ), f"Component '{c_id}' missing from surface '{s_id}'" + if not isinstance(c_exp, dict): + return + c_type = comp.type if comp else (node_info["type"] if node_info else "") + if "component" in c_exp: + assert c_type == c_exp["component"] + if "metadata" in c_exp: + assert comp is not None + assert comp.metadata == c_exp["metadata"] + if not node_info: + return + node_props = node_info["props"] + for p_key, p_val in c_exp.items(): + if p_key in ("id", "component", "metadata"): + continue + norm_val = _normalize_prop_value(node_props.get(p_key)) + assert _prop_value_matches(norm_val, p_val), ( + f"Property '{p_key}' mismatch on component '{c_id}':" + f" got {norm_val!r}, expected {p_val!r}" + ) + + +def _assert_surface_components_state( + s_id: str, comps_expected: Any, surface: Any +) -> None: + comp_items = _expected_component_items(comps_expected) + resolved_nodes = _resolve_surface_components(surface) + for c_id, c_exp in comp_items: + _assert_single_component_state(s_id, c_id, c_exp, surface, resolved_nodes) + if isinstance(comps_expected, list) and all( + surface.components_model.get(c_id) is not None for c_id, _ in comp_items + ): + assert len(surface.components_model.keys) == len(comp_items), ( + f"Surface '{s_id}' component count mismatch: expected" + f" {len(comp_items)}, got {len(surface.components_model.keys)}" + ) + + +def _assert_single_surface_state( + processor: MessageProcessor, s_id: str, s_exp: dict[str, Any] +) -> None: + _assert_known_keys(s_exp, _SURFACE_EXPECT_KEYS, f"expect.surfaces.{s_id} key") + surface = processor.model.get_surface(s_id) + if s_exp.get("exists") is False: + assert surface is None + return + assert surface is not None + if "theme" in s_exp: + assert surface.theme == s_exp["theme"] + if "metadata" in s_exp: + assert surface.metadata == s_exp["metadata"] + if "sendDataModel" in s_exp: + assert surface.send_data_model == s_exp["sendDataModel"] + if "dataModel" in s_exp: + assert surface.data_model.get("/") == s_exp["dataModel"] + if "components" in s_exp: + _assert_surface_components_state(s_id, s_exp["components"], surface) + _assert_surface_validation_fields(s_id, s_exp, surface) + + def _assert_expected_surface_state( processor: MessageProcessor, expected: dict[str, Any] ) -> None: - if "surfaces" in expected: - for s_id, s_exp in expected["surfaces"].items(): - surface = processor.model.get_surface(s_id) - if s_exp.get("exists") is False: - assert surface is None - continue - assert surface is not None - if "theme" in s_exp: - assert surface.theme == s_exp["theme"] - if "metadata" in s_exp: - assert surface.metadata == s_exp["metadata"] - if "sendDataModel" in s_exp: - assert surface.send_data_model == s_exp["sendDataModel"] - if "dataModel" in s_exp: - assert surface.data_model.get("/") == s_exp["dataModel"] - if "components" in s_exp: - comps_expected = s_exp["components"] - if isinstance(comps_expected, dict): - comp_items = list(comps_expected.items()) - elif isinstance(comps_expected, list): - comp_items = [ - (c.get("id"), c) for c in comps_expected if isinstance(c, dict) - ] - else: - comp_items = [] - resolved_nodes = _resolve_surface_components(surface) - for c_id, c_exp in comp_items: - comp = surface.components_model.get(c_id) - node_info = resolved_nodes.get(c_id) - assert ( - comp is not None or node_info is not None - ), f"Component '{c_id}' missing from surface '{s_id}'" - if isinstance(c_exp, dict): - c_type = ( - comp.type - if comp - else (node_info["type"] if node_info else "") - ) - if "component" in c_exp: - assert c_type == c_exp["component"] - if "metadata" in c_exp: - assert comp is not None - assert comp.metadata == c_exp["metadata"] - if node_info: - node_props = node_info["props"] - for p_key, p_val in c_exp.items(): - if p_key in ("id", "component", "metadata"): - continue - norm_val = _normalize_prop_value(node_props.get(p_key)) - assert norm_val == p_val or ( - isinstance(norm_val, int) - and not isinstance(norm_val, bool) - and isinstance(p_val, str) - and str(norm_val) == p_val - ), ( - f"Property '{p_key}' mismatch on component" - f" '{c_id}': got {norm_val!r}, expected" - f" {p_val!r}" - ) - - if isinstance(comps_expected, list) and all( - surface.components_model.get(c_id) is not None - for c_id, _ in comp_items - ): - assert len(surface.components_model.keys) == len(comp_items), ( - f"Surface '{s_id}' component count mismatch: expected" - f" {len(comp_items)}, got" - f" {len(surface.components_model.keys)}" - ) - - _assert_surface_validation_fields(s_id, s_exp, surface) + _assert_known_keys(expected, _SURFACE_STATE_EXPECT_KEYS, "expect key") + for s_id, s_exp in (expected.get("surfaces") or {}).items(): + _assert_single_surface_state(processor, s_id, s_exp) def _run_pure_validation_step_with_error( @@ -1165,7 +1306,49 @@ def _run_pure_validation_step_with_error( ) +_VALIDATE_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "catalog", + "catalogPaths", + "catalogs", + "expect", + "expect_error", + "expectError", + "messages", + "payload", + "protocolVersion", + "steps", + "strictMode", +}) +_VALIDATE_STEP_KEYS: frozenset[str] = frozenset({ + "expect", + "expect_error", + "expectError", + "message", + "messages", + "payload", +}) + + +def _step_messages(step: dict[str, Any], idx: int) -> Any: + messages = step.get("messages") or step.get("payload") + if not messages and "message" in step: + messages = [step["message"]] + assert messages, f"Step {idx} has no messages or payload" + return messages + + +def _step_expect_error( + step: dict[str, Any], case: dict[str, Any], is_last: bool +) -> Any: + err = step.get("expectError") or step.get("expect_error") + if err or not is_last: + return err + return case.get("expectError") or case.get("expect_error") + + def validate_pure_validation_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _VALIDATE_CASE_KEYS, "case key") catalogs = get_catalogs_for_test_case(case) val_config = ( RELAXED_VALIDATION if case.get("strictMode") is False else STRICT_VALIDATION @@ -1176,28 +1359,23 @@ def validate_pure_validation_case(case: dict[str, Any]) -> None: captured_renderer_errors: list[dict[str, Any]] = [] processor.model.on_error.subscribe(lambda msg: captured_renderer_errors.append(msg)) - steps = case.get("steps") or [case] + has_steps = "steps" in case + steps = case["steps"] if has_steps else [case] + assert steps, "validate case must have at least one step" for idx, step in enumerate(steps): - messages = step.get("messages") or step.get("payload") - if not messages and "message" in step: - messages = [step["message"]] - if not messages: - continue - - expect_error = step.get("expectError") or ( - case.get("expectError") if idx == len(steps) - 1 else None - ) - + if has_steps: + _assert_known_keys(step, _VALIDATE_STEP_KEYS, f"steps[{idx}] key") + messages = _step_messages(step, idx) + is_last = idx == len(steps) - 1 + expect_error = _step_expect_error(step, case, is_last) if expect_error: captured_renderer_errors.clear() with assert_raises(expect_error, captured_renderer_errors): _run_pure_validation_step_with_error(processor, messages) else: processor.process_messages(messages) - expected = step.get("expect") - if not expected and idx == len(steps) - 1: - expected = case.get("expect") + expected = step.get("expect") or (case.get("expect") if is_last else None) if expected: _assert_expected_surface_state(processor, expected) @@ -1215,27 +1393,71 @@ def validate_pure_validation_case(case: dict[str, Any]) -> None: def _case_validation_config(flags: dict[str, bool]) -> ValidationConfig: """Builds the ValidationConfig a case's `validationConfig` flags describe.""" - unknown = set(flags) - set(_CASE_VALIDATION_FLAGS) - assert not unknown, f"Unknown validationConfig flags: {sorted(unknown)}" + _assert_known_keys(flags, frozenset(_CASE_VALIDATION_FLAGS), "validationConfig key") return ValidationConfig( **{_CASE_VALIDATION_FLAGS[name]: value for name, value in flags.items()} ) -def validate_process_messages_case(case: dict[str, Any]) -> None: - catalogs = get_catalogs_for_test_case(case) +_PROCESS_MESSAGES_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "catalog", + "catalogPaths", + "catalogs", + "expect", + "expectError", + "messages", + "options", + "payload", + "protocolVersion", + "steps", + "strict_mode", + "strictMode", + "validationConfig", +}) +_PROCESS_MESSAGES_STEP_KEYS: frozenset[str] = frozenset({ + "expect", + "expectError", + "message", + "messages", + "payload", +}) + + +def _process_messages_validation_config( + case: dict[str, Any], +) -> ValidationConfig | None: if "validationConfig" in case: # The case sets its checks, so they don't depend on this harness. - val_config: ValidationConfig | None = _case_validation_config( - case["validationConfig"] - ) + return _case_validation_config(case["validationConfig"]) + is_strict = bool( + case.get("strictMode") + or case.get("strict_mode") + or case.get("options", {}).get("strict_mode") + ) + return STRICT_VALIDATION if is_strict else None + + +def _run_process_messages_step( + processor: MessageProcessor, + messages: Any, + expect_error: Any, + expected: dict[str, Any], + captured_renderer_errors: list[dict[str, Any]], +) -> None: + if expect_error: + captured_renderer_errors.clear() + with assert_raises(expect_error, captured_renderer_errors): + processor.process_messages(messages) else: - is_strict = bool( - case.get("strictMode") - or case.get("strict_mode") - or case.get("options", {}).get("strict_mode") - ) - val_config = STRICT_VALIDATION if is_strict else None + processor.process_messages(messages) + _assert_expected_surface_state(processor, expected) + + +def validate_process_messages_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _PROCESS_MESSAGES_CASE_KEYS, "case key") + catalogs = get_catalogs_for_test_case(case) + val_config = _process_messages_validation_config(case) processor = MessageProcessor( catalogs, options=MessageProcessorOptions(validation_config=val_config) ) @@ -1245,47 +1467,58 @@ def validate_process_messages_case(case: dict[str, Any]) -> None: messages = case.get("messages") or ( [case["payload"]] if "payload" in case else None ) - expect_error = case.get("expectError") - if messages: - if expect_error: - captured_renderer_errors.clear() - with assert_raises(expect_error, captured_renderer_errors): - processor.process_messages(messages) - else: - processor.process_messages(messages) - expected = case.get("expect", {}) - _assert_expected_surface_state(processor, expected) + _run_process_messages_step( + processor, + messages, + case.get("expectError"), + case.get("expect", {}), + captured_renderer_errors, + ) return steps = case.get("steps", []) - for step in steps: - messages = step.get("messages") or step.get("payload") - if not messages and "message" in step: - messages = [step["message"]] - if not messages: - continue - + assert steps, "process_messages case must have messages, payload, or steps" + for idx, step in enumerate(steps): + _assert_known_keys(step, _PROCESS_MESSAGES_STEP_KEYS, f"steps[{idx}] key") + step_msgs = _step_messages(step, idx) + is_last = step is steps[-1] expect_error = step.get("expectError") or ( - case.get("expectError") if step is steps[-1] else None + case.get("expectError") if is_last else None + ) + expected = step.get("expect") or (case.get("expect", {}) if is_last else {}) + _run_process_messages_step( + processor, + step_msgs, + expect_error, + expected, + captured_renderer_errors, ) - if expect_error: - captured_renderer_errors.clear() - with assert_raises(expect_error, captured_renderer_errors): - processor.process_messages(messages) - else: - processor.process_messages(messages) - expected = step.get("expect") or ( - case.get("expect") if step is steps[-1] else {} - ) - _assert_expected_surface_state(processor, expected) + +_CAPABILITIES_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalog", + "catalogs", + "expect", + "protocolVersion", +}) +_CAPABILITIES_ARGS_KEYS: frozenset[str] = frozenset({ + "componentEnvelopeRef", + "includeInlineCatalogs", + "version", + "versions", +}) def validate_capabilities_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _CAPABILITIES_CASE_KEYS, "case key") + assert "expect" in case, "capabilities case requires 'expect'" catalogs = get_catalogs_for_test_case(case) processor = MessageProcessor(catalogs) args = case.get("args") or {} + _assert_known_keys(args, _CAPABILITIES_ARGS_KEYS, "args key") raw_versions = args.get("versions") or ( [args["version"]] if "version" in args @@ -1337,6 +1570,18 @@ def _assert_self_contained(schema: dict[str, Any]) -> None: # The `expect` keys a from_json case may use (FromJsonExpect in # conformance/conformance_schema.json). An unknown key fails the case rather # than being silently ignored. +_FROM_JSON_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "catalog", + "catalogFile", + "catalogId", + "catalogPath", + "catalogSchema", + "expect", + "expectError", + "protocolVersion", + "schema", +}) _FROM_JSON_EXPECT_KEYS = frozenset({ "catalogId", "componentMetadata", @@ -1442,7 +1687,45 @@ def _assert_matches_builtin_catalog(loaded: CatalogApi) -> None: ), "Function mismatch" +def _assert_from_json_expectations(cat: CatalogApi, expected: dict[str, Any]) -> None: + _assert_known_keys(expected, _FROM_JSON_EXPECT_KEYS, "expect key") + if "catalogId" in expected: + assert cat.catalog_id == expected["catalogId"] + if "protocolVersion" in expected: + assert cat.protocol_version == expected["protocolVersion"] + if isinstance(expected.get("components"), (list, dict)): + for comp_name in expected["components"]: + assert cat.get_component(comp_name) is not None + if isinstance(expected.get("functions"), list): + for fn_name in expected["functions"]: + assert cat.get_function(fn_name) is not None + if isinstance(expected.get("componentMetadata"), dict): + for comp_name, exp_meta in expected["componentMetadata"].items(): + comp_api = cat.get_component(comp_name) + assert comp_api is not None + assert comp_api.metadata == exp_meta + if expected.get("selfContained"): + _assert_self_contained(cat.validation_schema) + if expected.get("matchesBuiltinCatalog"): + _assert_matches_builtin_catalog(cat) + _assert_from_json_component_samples(cat, expected) + + +def _assert_from_json_component_samples( + cat: CatalogApi, expected: dict[str, Any] +) -> None: + if "validComponents" not in expected and "invalidComponents" not in expected: + return + validator = PayloadValidator(cat) + for component in expected.get("validComponents", []): + validator.validate_component(component) + for component in expected.get("invalidComponents", []): + with pytest.raises(A2uiValidationError): + validator.validate_component(component) + + def validate_from_json_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _FROM_JSON_CASE_KEYS, "case key") c_path = case.get("catalogPath") if c_path: full_p = os.path.abspath(os.path.join(CONFORMANCE_ROOT, "../", c_path)) @@ -1464,48 +1747,30 @@ def validate_from_json_case(case: dict[str, Any]) -> None: Catalog.from_json(c_schema, catalog_id=c_id, protocol_version=p_ver) else: cat = Catalog.from_json(c_schema, catalog_id=c_id, protocol_version=p_ver) - expected = case.get("expect", {}) - unknown_keys = set(expected) - _FROM_JSON_EXPECT_KEYS - assert ( - not unknown_keys - ), f"Unknown from_json expect keys: {sorted(unknown_keys)}" - if "catalogId" in expected: - assert cat.catalog_id == expected["catalogId"] - if "protocolVersion" in expected: - assert cat.protocol_version == expected["protocolVersion"] - if "components" in expected: - if isinstance(expected["components"], list): - for comp_name in expected["components"]: - assert cat.get_component(comp_name) is not None - elif isinstance(expected["components"], dict): - for comp_name in expected["components"]: - assert cat.get_component(comp_name) is not None - if "functions" in expected: - if isinstance(expected["functions"], list): - for fn_name in expected["functions"]: - assert cat.get_function(fn_name) is not None - if "componentMetadata" in expected and isinstance( - expected["componentMetadata"], dict - ): - for comp_name, exp_meta in expected["componentMetadata"].items(): - comp_api = cat.get_component(comp_name) - assert comp_api is not None - assert comp_api.metadata == exp_meta - if expected.get("selfContained"): - _assert_self_contained(cat.validation_schema) - if expected.get("matchesBuiltinCatalog"): - _assert_matches_builtin_catalog(cat) - if "validComponents" in expected or "invalidComponents" in expected: - validator = PayloadValidator(cat) - for component in expected.get("validComponents", []): - validator.validate_component(component) - for component in expected.get("invalidComponents", []): - with pytest.raises(A2uiValidationError): - validator.validate_component(component) + _assert_from_json_expectations(cat, case.get("expect", {})) # The `expect` keys a `derive` case may use (`DeriveExpect` in # conformance_schema.json). +_DERIVE_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalog", + "catalogFile", + "catalogId", + "catalogPath", + "catalogSchema", + "expect", + "expectError", + "protocolVersion", + "schema", +}) +_DERIVE_ARGS_KEYS: frozenset[str] = frozenset({ + "catalogId", + "components", + "functions", + "instructions", +}) _DERIVE_EXPECT_KEYS = frozenset({ "absentKeys", "catalogId", @@ -1517,53 +1782,31 @@ def validate_from_json_case(case: dict[str, Any]) -> None: }) -def validate_derive_case(case: dict[str, Any]) -> None: - """Runs a `derive` case. - - Loads the source catalog with `from_json`, derives a catalog from it, and - checks the derived catalog and its `to_json()` document. - """ - document, _ = _load_case_catalog(case) - source = Catalog.from_json(document) - args = case["args"] - - def entries(kind: str) -> list[Any] | None: - # A map with one key is a new entry, built by loading it into a - # catalog of the source's version. - listed = args.get(kind) - if listed is None: - return None - picked: list[Any] = [] - for item in listed: - if isinstance(item, str): - picked.append(item) - continue - [(name, definition)] = item.items() - holder = Catalog.from_json({ - "catalogId": "conformance:derive-entries", - "protocolVersion": document.get("protocolVersion"), - kind: {name: definition}, - }) - picked.append(getattr(holder, kind)[name]) - return picked - - def derive() -> CatalogApi: - return source.derive( - catalog_id=args["catalogId"], - components=entries("components"), - functions=entries("functions"), - instructions=args.get("instructions"), - ) - - if "expectError" in case: - with assert_raises(case["expectError"]): - derive() - return - - expected = case["expect"] - unknown_keys = set(expected) - _DERIVE_EXPECT_KEYS - assert not unknown_keys, f"Unknown derive expect keys: {sorted(unknown_keys)}" - derived = derive() +def _derive_entries( + args: dict[str, Any], document: dict[str, Any], kind: str +) -> list[Any] | None: + # A map with one key is a new entry, built by loading it into a + # catalog of the source's version. + listed = args.get(kind) + if listed is None: + return None + picked: list[Any] = [] + for item in listed: + if isinstance(item, str): + picked.append(item) + continue + [(name, definition)] = item.items() + holder = Catalog.from_json({ + "catalogId": "conformance:derive-entries", + "protocolVersion": document.get("protocolVersion"), + kind: {name: definition}, + }) + picked.append(getattr(holder, kind)[name]) + return picked + + +def _assert_derived_catalog(derived: CatalogApi, expected: dict[str, Any]) -> None: + _assert_known_keys(expected, _DERIVE_EXPECT_KEYS, "expect key") out = derived.to_json() if "catalogId" in expected: assert derived.catalog_id == expected["catalogId"] @@ -1591,6 +1834,37 @@ def derive() -> CatalogApi: assert key not in out, f"Expected to_json() to carry no '{key}'" +def validate_derive_case(case: dict[str, Any]) -> None: + """Runs a `derive` case. + + Loads the source catalog with `from_json`, derives a catalog from it, and + checks the derived catalog and its `to_json()` document. + """ + _assert_known_keys(case, _DERIVE_CASE_KEYS, "case key") + assert ( + "expect" in case or "expectError" in case + ), "derive case requires 'expect' or 'expectError'" + document, _ = _load_case_catalog(case) + source = Catalog.from_json(document) + args = case["args"] + _assert_known_keys(args, _DERIVE_ARGS_KEYS, "args key") + + def derive() -> CatalogApi: + return source.derive( + catalog_id=args["catalogId"], + components=_derive_entries(args, document, "components"), + functions=_derive_entries(args, document, "functions"), + instructions=args.get("instructions"), + ) + + if "expectError" in case: + with assert_raises(case["expectError"]): + derive() + return + + _assert_derived_catalog(derive(), case["expect"]) + + def normalize_set_keywords(node: Any) -> Any: """Sorts the values of keywords whose order has no meaning. @@ -1638,6 +1912,20 @@ def _load_case_catalog(case: dict[str, Any]) -> tuple[Any, str | None]: ), None +_VALIDATION_SCHEMA_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "catalog", + "catalogFile", + "catalogId", + "catalogPath", + "catalogSchema", + "expectError", + "expectFile", + "protocolVersion", + "schema", +}) + + def validate_validation_schema_case(case: dict[str, Any]) -> None: """Runs a `validation_schema` case against its golden artifact. @@ -1645,7 +1933,7 @@ def validate_validation_schema_case(case: dict[str, Any]) -> None: names, ignoring object key order, which Python dicts ignore already, and the order of `enum` values and `required` names. """ - assert "expect" not in case, "validation_schema cases take no expect" + _assert_known_keys(case, _VALIDATION_SCHEMA_CASE_KEYS, "case key") p_ver = resolve_protocol_version(case) c_schema, _ = _load_case_catalog(case) c_id = ( @@ -1674,6 +1962,18 @@ def validate_validation_schema_case(case: dict[str, Any]) -> None: ), f"validation_schema differs from {expect_file}" +_ROUND_TRIP_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "catalog", + "catalogFile", + "catalogId", + "catalogPath", + "catalogSchema", + "protocolVersion", + "schema", +}) + + def validate_round_trip_case(case: dict[str, Any]) -> None: """Runs a `round_trip` case: `from_json(c).to_json()` equals `c`. @@ -1683,8 +1983,7 @@ def validate_round_trip_case(case: dict[str, Any]) -> None: comparison ignores key order, which Python dicts ignore already, and the order of `enum` and `required`. """ - unexpected = {"expect", "expectFile"} & set(case) - assert not unexpected, f"round_trip cases take no {sorted(unexpected)}" + _assert_known_keys(case, _ROUND_TRIP_CASE_KEYS, "case key") document, _ = _load_case_catalog(case) p_ver = resolve_protocol_version(case) c_id = resolve_catalog_id(case) @@ -1708,6 +2007,20 @@ def builtin_catalog(name: str, protocol_version: str | None) -> CatalogApi | Non return module.BasicCatalog() if module is not None else None +_TO_JSON_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "builtinCatalog", + "catalog", + "catalogFile", + "catalogId", + "catalogPath", + "catalogSchema", + "expectFile", + "protocolVersion", + "schema", +}) + + def validate_to_json_case(case: dict[str, Any]) -> None: """Runs a `to_json` case: `catalog.to_json()` equals `expectFile`. @@ -1716,7 +2029,7 @@ def validate_to_json_case(case: dict[str, Any]) -> None: load `catalogPath` or `catalog` with `from_json`. The comparison ignores key order and the order of `enum` and `required`. """ - assert "expect" not in case, "to_json cases take expectFile, not expect" + _assert_known_keys(case, _TO_JSON_CASE_KEYS, "case key") p_ver = resolve_protocol_version(case) builtin_name = case.get("builtinCatalog") if builtin_name: @@ -1741,10 +2054,18 @@ def validate_to_json_case(case: dict[str, Any]) -> None: ), f"to_json differs from {expect_file}" +_SCHEMA_EXPORT_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "expectFile", + "protocolVersion", +}) + + def validate_common_types_schema_case(case: dict[str, Any]) -> None: from a2ui.core.catalog import get_common_types_schema_json from a2ui.core.common import to_protocol_version + _assert_known_keys(case, _SCHEMA_EXPORT_CASE_KEYS, "case key") generated = json.loads( get_common_types_schema_json( to_protocol_version(resolve_protocol_version(case) or "") @@ -1762,6 +2083,7 @@ def validate_agent_to_renderer_schema_case(case: dict[str, Any]) -> None: from a2ui.core.common import to_protocol_version from a2ui.core.schema import get_agent_to_renderer_schema_json + _assert_known_keys(case, _SCHEMA_EXPORT_CASE_KEYS, "case key") generated = json.loads( get_agent_to_renderer_schema_json( to_protocol_version(resolve_protocol_version(case) or "") @@ -1775,11 +2097,24 @@ def validate_agent_to_renderer_schema_case(case: dict[str, Any]) -> None: ) +_VALIDATE_COMMON_TYPE_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "definition", + "protocolVersion", + "steps", +}) +_VALIDATE_COMMON_TYPE_STEP_KEYS: frozenset[str] = frozenset({ + "expectError", + "value", +}) + + def validate_common_type_case(case: dict[str, Any]) -> None: from pydantic import TypeAdapter, ValidationError from a2ui.core.schema import v0_9 as schema_v0_9, v1_0 as schema_v1_0 + _assert_known_keys(case, _VALIDATE_COMMON_TYPE_CASE_KEYS, "case key") schema_packages = { "v0.9": schema_v0_9, "v0.9.1": schema_v0_9, @@ -1797,7 +2132,9 @@ def validate_common_type_case(case: dict[str, Any]) -> None: ), f"'{definition}' is not a common types definition in {p_ver}" adapter: TypeAdapter[Any] = TypeAdapter(schema_pkg.COMMON_TYPES_DEFS[definition]) - for step in case["steps"]: + assert case.get("steps"), "validate_common_type case requires non-empty 'steps'" + for idx, step in enumerate(case["steps"]): + _assert_known_keys(step, _VALIDATE_COMMON_TYPE_STEP_KEYS, f"steps[{idx}] key") value = step["value"] expect_error = step.get("expectError") if expect_error: @@ -1817,11 +2154,27 @@ def validate_common_type_case(case: dict[str, Any]) -> None: ) +_RESOLVE_PATH_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalog", + "expect", +}) +_RESOLVE_PATH_ARGS_KEYS: frozenset[str] = frozenset({ + "context_path", + "contextPath", + "path", +}) + + def validate_resolve_path_case(case: dict[str, Any]) -> None: from a2ui.core.resolution.data_context import DataContext from a2ui.core.state.surface_model import SurfaceModel + _assert_known_keys(case, _RESOLVE_PATH_CASE_KEYS, "case key") + assert "expect" in case, "resolve_path case requires 'expect'" args = case.get("args", {}) + _assert_known_keys(args, _RESOLVE_PATH_ARGS_KEYS, "args key") path = args.get("path", "") context_path = args.get("contextPath") or args.get("context_path") surface = SurfaceModel(surface_id="dummy", default_catalog=basic_catalog) @@ -1834,11 +2187,81 @@ def validate_resolve_path_case(case: dict[str, Any]) -> None: assert res == expected +_DATA_MODEL_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "catalog", + "expect", + "initial", + "steps", + "watch", +}) +_DATA_MODEL_STEP_KEYS_BY_OP: dict[str, frozenset[str]] = { + "get": frozenset({ + "op", + "path", + "expect", + "expect_absent", + "expect_type", + "expect_error", + "expectError", + "expect_notified", + "expect_values", + }), + "set": frozenset({ + "op", + "path", + "value", + "expect_error", + "expectError", + "expect_notified", + "expect_values", + }), + "delete": frozenset({ + "op", + "path", + "expect_error", + "expectError", + "expect_notified", + "expect_values", + }), + "dispose": frozenset({ + "op", + "expect_error", + "expectError", + "expect_notified", + "expect_values", + }), +} + + +def _assert_data_model_observer_expectations( + idx: int, step: dict[str, Any], observers: list[Any] +) -> None: + if "expect_notified" in step: + expected_notified = step["expect_notified"] + actual_notified = [ + obs.path for obs in observers for _ in range(obs.change_count) + ] + assert sorted(actual_notified) == sorted(expected_notified), ( + f"Step {idx} ({step.get('op')}) expect_notified mismatch:" + f" got {actual_notified}, expected {expected_notified}" + ) + + if "expect_values" in step: + for v_path, exp_val in step["expect_values"].items(): + obs = next((o for o in observers if o.path == v_path), None) + assert obs is not None, f"Path '{v_path}' in expect_values is not watched" + assert obs.current_value == exp_val, ( + f"Step {idx} ({step.get('op')}) expect_values mismatch for" + f" '{v_path}': got {obs.current_value}, expected {exp_val}" + ) + + def validate_data_model_case(case: dict[str, Any]) -> None: from a2ui.core.state.data_model import DataModel - initial = case.get("initial") - model = DataModel(initial_data=initial) + _assert_known_keys(case, _DATA_MODEL_CASE_KEYS, "case key") + model = DataModel(initial_data=case.get("initial")) class _Observer: @@ -1852,14 +2275,19 @@ def on_change(self, val: Any) -> None: self.current_value = val observers: list[_Observer] = [] - watch_paths = case.get("watch") or [] - for p in watch_paths: + for p in case.get("watch") or []: obs = _Observer(p) model.subscribe(p, obs.on_change) observers.append(obs) steps = case.get("steps") or [] + assert steps or "expect" in case, "data_model case requires 'steps' or 'expect'" for idx, step in enumerate(steps): + op = step.get("op") + allowed_step_keys = _DATA_MODEL_STEP_KEYS_BY_OP.get(str(op)) + if allowed_step_keys is None: + pytest.fail(f"Unknown data_model op '{op}' in {_CURRENT_CASE_KEY}") + _assert_known_keys(step, allowed_step_keys, f"steps[{idx}] ({op}) key") for obs in observers: obs.change_count = 0 @@ -1870,59 +2298,53 @@ def on_change(self, val: Any) -> None: continue _apply_data_model_op(model, step) + _assert_data_model_observer_expectations(idx, step, observers) - if "expect_notified" in step: - expected_notified = step["expect_notified"] - actual_notified: list[str] = [] - for obs in observers: - for _ in range(obs.change_count): - actual_notified.append(obs.path) - assert sorted(actual_notified) == sorted(expected_notified), ( - f"Step {idx} ({step.get('op')}) expect_notified mismatch: " - f"got {actual_notified}, expected {expected_notified}" - ) + if "expect" in case: + assert model.get("/") == case["expect"] - if "expect_values" in step: - expected_values = step["expect_values"] - for v_path, exp_val in expected_values.items(): - obs = next((o for o in observers if o.path == v_path), None) - assert ( - obs is not None - ), f"Path '{v_path}' in expect_values is not watched" - assert obs.current_value == exp_val, ( - f"Step {idx} ({step.get('op')}) expect_values mismatch for" - f" '{v_path}': got {obs.current_value}, expected {exp_val}" - ) - if "expect" in case: - expected = case["expect"] - assert model.get("/") == expected +_DATA_MODEL_GET_EXPECT_KEYS = ( + "expect", + "expect_absent", + "expect_type", + "expect_notified", + "expect_values", + "expect_error", +) + + +def _apply_data_model_get(model: Any, step: dict[str, Any]) -> None: + step_path = step.get("path", "") + assert any( + k in step for k in _DATA_MODEL_GET_EXPECT_KEYS + ), "data_model 'get' step requires an expectation" + actual = model.get(step_path) + if step.get("expect_absent") is True: + assert actual is None, f"Expected path '{step_path}' to be absent, got {actual}" + if "expect_type" in step: + exp_type = step["expect_type"] + expected_cls = ( + list if exp_type == "list" else (dict if exp_type == "object" else None) + ) + if expected_cls is None: + pytest.fail(f"Unknown expect_type '{exp_type}' in {_CURRENT_CASE_KEY}") + assert isinstance(actual, expected_cls), ( + f"Expected path '{step_path}' to be {expected_cls.__name__}," + f" got {type(actual)}" + ) + if "expect" in step: + assert actual == step["expect"], ( + f"Get at path '{step_path}' mismatch: got {actual}, expected" + f" {step['expect']}" + ) def _apply_data_model_op(model: Any, step: dict[str, Any]) -> None: op = step.get("op") step_path = step.get("path", "") if op == "get": - actual = model.get(step_path) - if step.get("expect_absent") is True: - assert ( - actual is None - ), f"Expected path '{step_path}' to be absent, got {actual}" - if "expect_type" in step: - exp_type = step["expect_type"] - if exp_type == "list": - assert isinstance( - actual, list - ), f"Expected path '{step_path}' to be list, got {type(actual)}" - elif exp_type == "object": - assert isinstance( - actual, dict - ), f"Expected path '{step_path}' to be dict, got {type(actual)}" - if "expect" in step: - assert actual == step["expect"], ( - f"Get at path '{step_path}' mismatch: got {actual}, expected" - f" {step['expect']}" - ) + _apply_data_model_get(model, step) elif op == "set": model.set(step_path, step.get("value")) elif op == "delete": @@ -1933,213 +2355,479 @@ def _apply_data_model_op(model: Any, step: dict[str, Any]) -> None: raise ValueError(f"Unknown data_model op: {op}") -def validate_get_renderer_data_model_case(case: dict[str, Any]) -> None: - catalogs = get_catalogs_for_test_case(case) - processor = MessageProcessor(catalogs) +_GET_DATA_MODEL_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalogPaths", + "expect", + "messages", + "payload", + "protocolVersion", + "steps", +}) +_GET_DATA_MODEL_ARGS_KEYS: frozenset[str] = frozenset({ + "version", +}) +_GET_DATA_MODEL_STEP_KEYS: frozenset[str] = frozenset({ + "messages", + "payload", +}) + + +def _feed_renderer_data_model_messages( + processor: MessageProcessor, case: dict[str, Any] +) -> None: steps = case.get("steps") if steps: - for step in steps: + for idx, step in enumerate(steps): + _assert_known_keys(step, _GET_DATA_MODEL_STEP_KEYS, f"steps[{idx}] key") msgs = step.get("messages") or step.get("payload") - if msgs: - processor.process_messages(msgs) - else: - msgs = case.get("messages") or ([case["payload"]] if "payload" in case else []) - if msgs: + assert msgs, f"Step {idx} has no messages or payload" processor.process_messages(msgs) + return + msgs = case.get("messages") or ([case["payload"]] if "payload" in case else []) + assert msgs, "get_renderer_data_model case has no messages or payload" + processor.process_messages(msgs) + + +def validate_get_renderer_data_model_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _GET_DATA_MODEL_CASE_KEYS, "case key") + assert "expect" in case, "get_renderer_data_model case requires 'expect'" + processor = MessageProcessor(get_catalogs_for_test_case(case)) + _feed_renderer_data_model_messages(processor, case) args = case.get("args") or {} + _assert_known_keys(args, _GET_DATA_MODEL_ARGS_KEYS, "args key") ver = args.get("version") or resolve_protocol_version(case) res = processor.get_renderer_data_model(ver) expected = case.get("expect") - if expected is None and "expect" in case: + if expected is None: assert res is None elif isinstance(expected, dict): - if "data" in expected: - assert res == expected["data"] - else: - assert res == expected + assert res == (expected["data"] if "data" in expected else expected) + else: + pytest.fail(f"Unexpected expect value {expected!r} in {_CURRENT_CASE_KEY}") -def validate_handle_rpc_case(case: dict[str, Any]) -> None: - args = case.get("args", {}) - message = args.get("message") - outbound_call = args.get("outboundCall") - inbound_response = args.get("inboundResponse") - fn_metadata = args.get("functionMetadata", {}) - user_activation = bool(args.get("userActivationPresent", False)) +_HANDLE_RPC_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalog", + "expect", + "expectError", + "protocolVersion", +}) +_HANDLE_RPC_ARGS_KEYS: frozenset[str] = frozenset({ + "catalogId", + "catalogVersion", + "dataModel", + "disposeBeforeResponse", + "functionMetadata", + "inboundResponse", + "message", + "outboundCall", + "secondOutboundCall", + "surfaceId", + "uiActionFallbackScenario", + "uiBindingScenario", + "userActivationPresent", +}) +_HANDLE_RPC_EXPECT_KEYS: frozenset[str] = frozenset({ + "correlatedCallId", + "error", + "initialValue", + "outboundCall", + "resolvedValue", + "response", + "result", +}) +_UI_BINDING_SCENARIO_KEYS: frozenset[str] = frozenset({ + "catalogId", + "component", + "inboundResponseValue", + "surfaceId", +}) +_UI_ACTION_FALLBACK_SCENARIO_KEYS: frozenset[str] = frozenset({ + "actionPayload", + "catalogId", + "inboundResponseValue", + "surfaceId", +}) + + +def _make_mock_rpc_executor(name: str) -> Callable[..., Any]: + def execute(fn_args: dict[str, Any], *_args: Any, **_kwargs: Any) -> Any: + if name == "playMedia": + return {"playing": True, "timestamp": 0} + if name == "openExternalUrl": + return {"opened": True} + if name == "syncState": + return None + if name == "failingFunction": + raise RuntimeError("An error occurred during function execution.") + if name == "calculateTax": + amount = (fn_args or {}).get("amount", 0) + return amount * 0.1 + if name == "echoArgs": + return dict(fn_args or {}) + return None + + return execute + - from a2ui.core.catalog import Catalog, FunctionImplementation +def _build_rpc_mock_functions(fn_metadata: dict[str, Any]) -> list[Any]: + from a2ui.core.catalog import FunctionImplementation funcs: list[FunctionImplementation] = [] for fn_name, meta in fn_metadata.items(): allowed = meta.get("allowedCallers", "rendererOrAgent") requires_activation = meta.get("requiresUserActivation", False) - - def make_exec(name: str): - def execute(fn_args: dict[str, Any], *args: Any, **kwargs: Any) -> Any: - if name == "playMedia": - return {"playing": True, "timestamp": 0} - elif name == "openExternalUrl": - return {"opened": True} - elif name == "syncState": - return None - elif name == "failingFunction": - raise Exception("An error occurred during function execution.") - elif name == "calculateTax": - amount = (fn_args or {}).get("amount", 0) - return amount * 0.1 - elif name == "echoArgs": - return dict(fn_args or {}) - return None - - return execute - fn_schema = meta.get("schema") or meta.get("parameters") or {"type": "object"} funcs.append( FunctionImplementation( name=fn_name, return_type=meta.get("returnType", "any"), schema=fn_schema, - execute=make_exec(fn_name), + execute=_make_mock_rpc_executor(fn_name), allowed_callers=allowed, requires_user_activation=requires_activation, ) ) + return funcs + + +def _start_rpc_outbound_call(processor: MessageProcessor, spec: dict[str, Any]) -> Any: + from a2ui.core.rpc import CallOptions + from a2ui.core.schema.v1_0.common_types import FunctionCall + + call_fn = spec["callFunction"] + call_name = call_fn.get("@call") or call_fn.get("call") + return processor.call_agent_function( + surface_id=spec["surfaceId"], + call=FunctionCall( + call=call_name, + catalogId=call_fn.get("catalogId") + or "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + args=call_fn.get("args"), + ), + options=CallOptions( + function_call_id=spec["functionCallId"], + version="1.0", + timeout_ms=spec.get("timeoutMs"), + ), + ) + + +def _run_rpc_outbound_success( + outbound_call: dict[str, Any], + inbound_response: dict[str, Any], + expect_dict: dict[str, Any], + processor: MessageProcessor, +) -> None: + correlated_id = expect_dict.get("correlatedCallId") + if correlated_id is not None: + assert ( + inbound_response["agentFunctionResponse"]["functionCallId"] == correlated_id + ) + fut = _start_rpc_outbound_call(processor, outbound_call) + processor.process_messages(inbound_response) + assert fut.done() + assert fut.result() == expect_dict.get("result") + + +def _run_rpc_outbound_error( + args: dict[str, Any], + outbound_call: dict[str, Any], + inbound_response: Any, + expected_err: dict[str, Any], + processor: MessageProcessor, +) -> None: + import asyncio + + from a2ui.core.exceptions import A2uiRpcError + + if "secondOutboundCall" in args: + + async def _expect_duplicate() -> None: + pending = _start_rpc_outbound_call(processor, outbound_call) + with pytest.raises(A2uiRpcError) as exc_info: + _start_rpc_outbound_call(processor, args["secondOutboundCall"]) + assert exc_info.value.code == expected_err.get("code", "DUPLICATE") + pending.cancel() + + asyncio.run(_expect_duplicate()) + elif inbound_response: + + async def _expect_inbound_error() -> None: + pending = _start_rpc_outbound_call(processor, outbound_call) + processor.process_messages(inbound_response) + with pytest.raises(A2uiRpcError) as exc_info: + await pending + if "code" in expected_err: + assert exc_info.value.code == expected_err["code"] + if "message" in expected_err: + assert expected_err["message"] in str(exc_info.value) + + asyncio.run(_expect_inbound_error()) + elif args.get("disposeBeforeResponse"): + + async def _expect_cancelled() -> None: + pending = _start_rpc_outbound_call(processor, outbound_call) + processor.rpc.dispose() + with pytest.raises((A2uiRpcError, asyncio.CancelledError)) as exc_info: + await pending + if isinstance(exc_info.value, A2uiRpcError) and "code" in expected_err: + assert exc_info.value.code == expected_err["code"] + + asyncio.run(_expect_cancelled()) + elif "timeoutMs" in outbound_call: + + async def _expect_timeout() -> None: + with pytest.raises(A2uiRpcError) as exc_info: + await _start_rpc_outbound_call(processor, outbound_call) + assert exc_info.value.code == expected_err.get("code", "TIMEOUT") + + asyncio.run(_expect_timeout()) + else: + pytest.fail(f"Unhandled outboundCall error scenario in {_CURRENT_CASE_KEY}") + + +def _assert_outbound_call_agent_message( + outbound_messages: list[dict[str, Any]], expected_outbound: dict[str, Any] +) -> str: + call_msgs = [m for m in outbound_messages if "callAgentFunction" in m] + assert ( + len(call_msgs) == 1 + ), f"Expected 1 outbound callAgentFunction message, got {call_msgs}" + outbound = call_msgs[0] + assert outbound.get("version") == expected_outbound.get("version") + actual_call = outbound["callAgentFunction"] + expected_call = expected_outbound["callAgentFunction"] + assert actual_call.get("surfaceId") == expected_call.get("surfaceId") + assert actual_call.get("callFunction") == expected_call.get("callFunction") + call_id = actual_call.get("functionCallId") + assert isinstance(call_id, str) and call_id + return call_id + + +def _run_ui_binding_scenario( + scenario: dict[str, Any], + expected: dict[str, Any], + processor: MessageProcessor, + outbound_messages: list[dict[str, Any]], +) -> None: + from a2ui.core.resolution import NodeResolver, ResolvedBinding + from a2ui.core.state import ComponentModel + + _assert_known_keys( + scenario, _UI_BINDING_SCENARIO_KEYS, "args.uiBindingScenario key" + ) + surface_id = scenario["surfaceId"] + scenario_cat_id = scenario["catalogId"] + scenario_cat = Catalog( + catalog_id=scenario_cat_id, + protocol_version="1.0", + components=list(v10_catalog.components.values()), + functions=[], + ) + surface = SurfaceModel( + surface_id=surface_id, + default_catalog=scenario_cat, + available_catalogs={scenario_cat_id: scenario_cat}, + protocol_version="1.0", + ) + processor.model.add_surface(surface) + comp = dict(scenario["component"]) + comp_id = comp.pop("id") + comp_type = comp.pop("component") + surface.components_model.add_component( + ComponentModel(comp_id, comp_type, scenario_cat, comp) + ) + resolver = NodeResolver(surface, scenario_cat) + try: + root = resolver.root + assert root is not None, "Expected root ComponentNode to be mounted" + initial_binding = root.props.get("text") + initial_val = ( + initial_binding.value + if isinstance(initial_binding, ResolvedBinding) + else initial_binding + ) + assert initial_val == expected.get("initialValue") + + call_id = _assert_outbound_call_agent_message( + outbound_messages, expected["outboundCall"] + ) + processor.process_messages({ + "version": "1.0", + "agentFunctionResponse": { + "functionCallId": call_id, + "value": scenario["inboundResponseValue"], + }, + }) + resolved_binding = root.props.get("text") + resolved_val = ( + resolved_binding.value + if isinstance(resolved_binding, ResolvedBinding) + else resolved_binding + ) + assert resolved_val == expected.get("resolvedValue") + finally: + resolver.dispose() + surface.dispose() + + +def _run_ui_action_fallback_scenario( + scenario: dict[str, Any], + expected: dict[str, Any], + processor: MessageProcessor, + outbound_messages: list[dict[str, Any]], +) -> None: + import asyncio + import inspect + + _assert_known_keys( + scenario, + _UI_ACTION_FALLBACK_SCENARIO_KEYS, + "args.uiActionFallbackScenario key", + ) + surface_id = scenario["surfaceId"] + scenario_cat_id = scenario["catalogId"] + scenario_cat = Catalog( + catalog_id=scenario_cat_id, + protocol_version="1.0", + components=[], + functions=[], + ) + surface = SurfaceModel( + surface_id=surface_id, + default_catalog=scenario_cat, + available_catalogs={scenario_cat_id: scenario_cat}, + protocol_version="1.0", + ) + processor.model.add_surface(surface) + context = DataContext(surface=surface, path="/") + + async def _run() -> None: + action_res = context.resolve_action(scenario["actionPayload"]) + call_id = _assert_outbound_call_agent_message( + outbound_messages, expected["outboundCall"] + ) + processor.process_messages({ + "version": "1.0", + "agentFunctionResponse": { + "functionCallId": call_id, + "value": scenario["inboundResponseValue"], + }, + }) + result = await action_res if inspect.isawaitable(action_res) else action_res + assert result == expected.get("result") + + try: + asyncio.run(_run()) + finally: + surface.dispose() + + +def validate_handle_rpc_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _HANDLE_RPC_CASE_KEYS, "case key") + args = case.get("args", {}) + _assert_known_keys(args, _HANDLE_RPC_ARGS_KEYS, "args key") + expect_dict = case.get("expect") or {} + _assert_known_keys(expect_dict, _HANDLE_RPC_EXPECT_KEYS, "expect key") + + message = args.get("message") + outbound_call = args.get("outboundCall") + inbound_response = args.get("inboundResponse") + fn_metadata = args.get("functionMetadata", {}) + user_activation = bool(args.get("userActivationPresent", False)) + funcs = _build_rpc_mock_functions(fn_metadata) cat_id, cat_version = _resolve_rpc_catalog_info(args, message, outbound_call, case) catalogs = _build_rpc_catalogs(cat_id, cat_version, funcs) + outbound_messages: list[dict[str, Any]] = [] captured_renderer_errors: list[dict[str, Any]] = [] processor = MessageProcessor( catalogs=catalogs, options=MessageProcessorOptions( - outbound_listener=lambda msg: None, + outbound_listener=lambda msg: outbound_messages.append(msg), error_handler=lambda msg: captured_renderer_errors.append(msg), ), ) _setup_rpc_surface_if_needed(args, catalogs[0], catalogs, cat_version, processor) + handled = False if message: _verify_rpc_inbound_message( message, case, processor, user_activation, captured_renderer_errors ) + handled = True - if ( - outbound_call - and inbound_response - and not (case.get("expectError") or (case.get("expect") or {}).get("error")) - ): - correlated_id = case.get("expect", {}).get("correlatedCallId") - if correlated_id is not None: - assert ( - inbound_response["agentFunctionResponse"]["functionCallId"] - == correlated_id - ) - - from a2ui.core.rpc import CallOptions - from a2ui.core.schema.v1_0.common_types import FunctionCall - - call_fn = outbound_call["callFunction"] - call_name = call_fn.get("@call") or call_fn.get("call") - fut = processor.call_agent_function( - surface_id=outbound_call["surfaceId"], - call=FunctionCall( - call=call_name, - catalogId=call_fn.get("catalogId") - or "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", - args=call_fn.get("args"), - ), - options=CallOptions( - function_call_id=outbound_call["functionCallId"], - version="1.0", - ), - ) - processor.process_messages(inbound_response) - assert fut.done() - assert fut.result() == case.get("expect", {}).get("result") - elif outbound_call and ( - case.get("expectError") or (case.get("expect") or {}).get("error") - ): - import asyncio - - from a2ui.core.exceptions import A2uiRpcError - from a2ui.core.rpc import CallOptions - from a2ui.core.schema.v1_0.common_types import FunctionCall - - expected_err = case.get("expectError") or case.get("expect", {}).get("error") - - def _start_call(spec: dict[str, Any]) -> Any: - call_fn = spec["callFunction"] - call_name = call_fn.get("@call") or call_fn.get("call") - return processor.call_agent_function( - surface_id=spec["surfaceId"], - call=FunctionCall( - call=call_name, - catalogId=call_fn.get("catalogId") - or "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", - args=call_fn.get("args"), - ), - options=CallOptions( - function_call_id=spec["functionCallId"], - version="1.0", - timeout_ms=spec.get("timeoutMs"), - ), - ) - - if "secondOutboundCall" in args: - - async def _expect_duplicate() -> None: - # The first call stays pending, so reusing its id must be refused. - pending = _start_call(outbound_call) - with pytest.raises(A2uiRpcError) as exc_info: - _start_call(args["secondOutboundCall"]) - assert exc_info.value.code == expected_err.get("code", "DUPLICATE") - pending.cancel() - - asyncio.run(_expect_duplicate()) - elif inbound_response: - - async def _expect_inbound_error() -> None: - pending = _start_call(outbound_call) - processor.process_messages(inbound_response) - with pytest.raises(A2uiRpcError) as exc_info: - await pending - if "code" in expected_err: - assert exc_info.value.code == expected_err["code"] - if "message" in expected_err: - assert expected_err["message"] in str(exc_info.value) - - asyncio.run(_expect_inbound_error()) - elif args.get("disposeBeforeResponse"): - - async def _expect_cancelled() -> None: - pending = _start_call(outbound_call) - processor.rpc.dispose() - with pytest.raises((A2uiRpcError, asyncio.CancelledError)) as exc_info: - await pending - if isinstance(exc_info.value, A2uiRpcError) and "code" in expected_err: - assert exc_info.value.code == expected_err["code"] - - asyncio.run(_expect_cancelled()) - elif "timeoutMs" in outbound_call: + has_err = bool(case.get("expectError") or expect_dict.get("error")) + if outbound_call and inbound_response and not has_err: + _run_rpc_outbound_success( + outbound_call, inbound_response, expect_dict, processor + ) + handled = True + elif outbound_call and has_err: + expected_err = case.get("expectError") or expect_dict.get("error") or {} + _run_rpc_outbound_error( + args, outbound_call, inbound_response, expected_err, processor + ) + handled = True - async def _expect_timeout() -> None: - # No response arrives, so the handler's timer must reject the future. - with pytest.raises(A2uiRpcError) as exc_info: - await _start_call(outbound_call) - assert exc_info.value.code == expected_err.get("code", "TIMEOUT") + if "uiBindingScenario" in args: + _run_ui_binding_scenario( + args["uiBindingScenario"], expect_dict, processor, outbound_messages + ) + handled = True + + if "uiActionFallbackScenario" in args: + _run_ui_action_fallback_scenario( + args["uiActionFallbackScenario"], + expect_dict, + processor, + outbound_messages, + ) + handled = True - asyncio.run(_expect_timeout()) + if not handled: + pytest.fail(f"No handle_rpc scenario branch matched in {_CURRENT_CASE_KEY}") def validate_accessibility_check_case(case: dict[str, Any]) -> None: + del case pytest.skip(UNRUNNABLE_SUITES["core/accessibility.yaml"]) -def validate_select_catalog_case(case: dict[str, Any]) -> None: - from a2ui.core.resolution import DataContext - from a2ui.core.state import ComponentModel, SurfaceModel +_SELECT_CATALOG_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalog", + "expectError", + "expectSelected", +}) +_SELECT_CATALOG_ARGS_KEYS: frozenset[str] = frozenset({ + "catalogs", + "components", + "functionCall", + "surface", +}) +_SELECT_CATALOG_SURFACE_KEYS: frozenset[str] = frozenset({ + "defaultCatalogId", + "id", + "supportedCatalogIds", +}) +_SELECT_CATALOG_FUNCTION_CALL_KEYS: frozenset[str] = frozenset({ + "@call", + "args", + "call", + "catalogId", +}) - args = case.get("args", {}) - surface_args = args.get("surface", {}) + +def _build_select_catalog_surface( + args: dict[str, Any], surface_args: dict[str, Any] +) -> tuple[str, CatalogApi, dict[str, CatalogApi], SurfaceModel]: s_id = surface_args.get("id", "main_surface") default_cat_id = surface_args.get("defaultCatalogId", "basic") @@ -2167,99 +2855,156 @@ def validate_select_catalog_case(case: dict[str, Any]) -> None: default_catalog=default_cat, available_catalogs=catalogs_dict, ) - - expect_err = case.get("expectError") - - if expect_err: - with assert_raises(expect_err): - for cat_id, cat in catalogs_dict.items(): + return s_id, default_cat, catalogs_dict, surface + + +def _select_catalog_for_components( + components: dict[str, Any], + s_id: str, + default_cat: CatalogApi, + catalogs_dict: dict[str, CatalogApi], + surface: SurfaceModel, + check_version_mismatch: bool, +) -> str | None: + from a2ui.core.state import ComponentModel + + last_selected: str | None = None + for c_id, c_data in components.items(): + comp_cat_id = c_data.get("catalogId") + if comp_cat_id: + if comp_cat_id not in catalogs_dict: + raise A2uiCatalogError( + f"Catalog '{comp_cat_id}' is not supported by surface '{s_id}'." + ) + comp_cat = catalogs_dict[comp_cat_id] + if check_version_mismatch: def_ver = getattr(default_cat, "protocol_version", None) - cat_ver = getattr(cat, "protocol_version", None) + cat_ver = getattr(comp_cat, "protocol_version", None) if def_ver and cat_ver and def_ver != cat_ver: raise A2uiCatalogError( - f"Protocol version mismatch: cannot mix catalog '{cat_id}'" - f" ({cat_ver}) with surface version {def_ver}." + f"Component '{c_id}' catalog protocol version {cat_ver}" + f" mismatches default catalog protocol version {def_ver}." ) + else: + comp_cat = default_cat + last_selected = comp_cat.catalog_id + comp_model = ComponentModel( + c_id, c_data.get("component", "Box"), catalog=comp_cat + ) + surface.components_model.add_component(comp_model) + return last_selected - if "components" in args: - for c_id, c_data in args["components"].items(): - comp_cat_id = c_data.get("catalogId") - if comp_cat_id: - if comp_cat_id not in catalogs_dict: - raise A2uiCatalogError( - f"Catalog '{comp_cat_id}' is not supported by surface" - f" '{s_id}'." - ) - comp_cat = catalogs_dict[comp_cat_id] - def_ver = getattr(default_cat, "protocol_version", None) - cat_ver = getattr(comp_cat, "protocol_version", None) - if def_ver and cat_ver and def_ver != cat_ver: - raise A2uiCatalogError( - f"Component '{c_id}' catalog protocol version {cat_ver}" - " mismatches default catalog protocol version" - f" {def_ver}." - ) - else: - comp_cat = default_cat - - comp_model = ComponentModel( - c_id, c_data.get("component", "Box"), catalog=comp_cat - ) - surface.components_model.add_component(comp_model) - elif "functionCall" in args: - fn_call = args["functionCall"] - fn_cat_id = fn_call.get("catalogId") - ctx = DataContext(surface=surface, path="/") - ctx._execute_function( - fn_call["call"], fn_call.get("args", {}), catalog_id=fn_cat_id - ) - else: - if "components" in args: - last_selected = None - for c_id, c_data in args["components"].items(): - comp_cat_id = c_data.get("catalogId") - if comp_cat_id: - if comp_cat_id not in catalogs_dict: - raise A2uiCatalogError( - f"Catalog '{comp_cat_id}' is not supported by surface" - f" '{s_id}'." - ) - comp_cat = catalogs_dict[comp_cat_id] - else: - comp_cat = default_cat - last_selected = comp_cat.catalog_id - - comp_model = ComponentModel( - c_id, c_data.get("component", "Box"), catalog=comp_cat - ) - surface.components_model.add_component(comp_model) - if "expectSelected" in case: - assert last_selected == case["expectSelected"] +def _run_select_catalog_error_case( + args: dict[str, Any], + s_id: str, + default_cat: Any, + catalogs_dict: dict[str, Any], + surface: SurfaceModel, +) -> None: + def_ver = getattr(default_cat, "protocol_version", None) + for cat_id, cat in catalogs_dict.items(): + cat_ver = getattr(cat, "protocol_version", None) + if def_ver and cat_ver and def_ver != cat_ver: + raise A2uiCatalogError( + f"Protocol version mismatch: cannot mix catalog '{cat_id}'" + f" ({cat_ver}) with surface version {def_ver}." + ) + + if "components" in args: + _select_catalog_for_components( + args["components"], + s_id, + default_cat, + catalogs_dict, + surface, + check_version_mismatch=True, + ) + elif "functionCall" in args: + fn_call = args["functionCall"] + ctx = DataContext(surface=surface, path="/") + ctx._execute_function( + fn_call.get("@call") or fn_call["call"], + fn_call.get("args", {}), + catalog_id=fn_call.get("catalogId"), + ) + + +def validate_select_catalog_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _SELECT_CATALOG_CASE_KEYS, "case key") + assert ( + "expectError" in case or "expectSelected" in case + ), "select_catalog case requires 'expectError' or 'expectSelected'" + args = case.get("args", {}) + _assert_known_keys(args, _SELECT_CATALOG_ARGS_KEYS, "args key") + surface_args = args.get("surface", {}) + _assert_known_keys(surface_args, _SELECT_CATALOG_SURFACE_KEYS, "args.surface key") + if isinstance(args.get("functionCall"), dict): + _assert_known_keys( + args["functionCall"], + _SELECT_CATALOG_FUNCTION_CALL_KEYS, + "args.functionCall key", + ) + + s_id, default_cat, catalogs_dict, surface = _build_select_catalog_surface( + args, surface_args + ) + expect_err = case.get("expectError") + + if expect_err: + with assert_raises(expect_err): + _run_select_catalog_error_case( + args, s_id, default_cat, catalogs_dict, surface + ) + elif "components" in args: + last_selected = _select_catalog_for_components( + args["components"], + s_id, + default_cat, + catalogs_dict, + surface, + check_version_mismatch=False, + ) + assert last_selected == case["expectSelected"] + elif "functionCall" in args: + fn_cat_id = args["functionCall"].get("catalogId") + if fn_cat_id: + if fn_cat_id not in catalogs_dict: + raise A2uiCatalogError(f"Catalog not found: {fn_cat_id}") + selected = catalogs_dict[fn_cat_id].catalog_id + else: + assert surface.default_catalog is not None + selected = surface.default_catalog.catalog_id + assert selected == case["expectSelected"] + else: + pytest.fail( + "select_catalog case requires 'components' or 'functionCall' in" + f" {_CURRENT_CASE_KEY}" + ) - elif "functionCall" in args: - fn_call = args["functionCall"] - fn_cat_id = fn_call.get("catalogId") - ctx = DataContext(surface=surface, path="/") - if fn_cat_id: - if fn_cat_id not in catalogs_dict: - raise A2uiCatalogError(f"Catalog not found: {fn_cat_id}") - selected = catalogs_dict[fn_cat_id].catalog_id - else: - selected = surface.default_catalog.catalog_id - if "expectSelected" in case: - assert selected == case["expectSelected"] +_PARSE_EXPRESSION_TEMPLATE_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "expect", + "expect_error", + "expectError", + "input", +}) def validate_parse_expression_template_case(case: dict[str, Any]) -> None: from a2ui.core.expressions.expression_parser import ExpressionParser + _assert_known_keys(case, _PARSE_EXPRESSION_TEMPLATE_CASE_KEYS, "case key") + assert ( + "expect" in case or "expect_error" in case or "expectError" in case + ), "parse_expression_template case requires 'expect' or 'expect_error'" input_str = case.get("input", "") expect_error = case.get("expect_error") or case.get("expectError") parser = ExpressionParser() if expect_error: + _assert_known_keys(expect_error, _EXPECT_ERROR_KEYS, "expect_error key") cat = expect_error.get("category", "ParseError") msg = expect_error.get("message") expected_types = CATEGORY_TO_EXCEPTION.get(cat, (A2uiError, ValueError)) @@ -2412,6 +3157,42 @@ def _verify_rpc_inbound_message( ) ) _assert_rpc_response_payload(responses, expect_dict["response"]) + else: + pytest.fail( + "Inbound handle_rpc case requires expectError, expect.error, or" + f" expect.response in {_CURRENT_CASE_KEY}" + ) + + +_DISPATCH_ACTION_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "actionPayload", + "catalog", + "catalogPaths", + "catalogs", + "dataModel", + "expectDataModel", + "expectDispatched", + "expectError", + "metadata", + "protocolVersion", + "scope", + "sourceComponentId", + "surfaceId", + "useBasicCatalog", + "userActivation", +}) +_EXPECT_DISPATCHED_KEYS: frozenset[str] = frozenset({ + "catalogId", + "context", + "hasTimestamp", + "metadata", + "name", + "sourceComponentId", + "surfaceId", + "timestampIso8601", + "userMessage", +}) def _assert_dispatched_action_event( @@ -2424,6 +3205,9 @@ def _assert_dispatched_action_event( return if expect_dispatched is None: return + _assert_known_keys( + expect_dispatched, _EXPECT_DISPATCHED_KEYS, "expectDispatched key" + ) assert len(dispatched) >= 1, "Expected action to be dispatched, but none was" actual = dispatched[0] for key in ( @@ -2444,20 +3228,10 @@ def _assert_dispatched_action_event( ) -def validate_dispatch_action_case(case: dict[str, Any]) -> None: +def _build_dispatch_action_catalog(case: dict[str, Any]) -> Any: import copy from a2ui.core.catalog import FunctionImplementation - action_payload = case["actionPayload"] - data_model_dict = case.get("dataModel") or {} - surface_id = case.get("surfaceId", "main") - source_comp_id = case.get("sourceComponentId", "test_comp") - scope = case.get("scope") - user_activation = bool(case.get("userActivation", True)) - expect_dispatched = case.get("expectDispatched") - expect_data_model = case.get("expectDataModel") - expect_error = case.get("expectError") - catalogs = get_catalogs_for_test_case(case) base_cat = catalogs[0] if catalogs else v09_catalog @@ -2487,7 +3261,22 @@ def _fail_action(_args: dict[str, Any], _ctx: Any = None, _sig: Any = None) -> N schema={"type": "object", "properties": {}}, execute=_fail_action, ) - data_model = DataModel(data_model_dict) + return default_cat + + +def validate_dispatch_action_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _DISPATCH_ACTION_CASE_KEYS, "case key") + assert ( + "expectError" in case or "expectDispatched" in case or "expectDataModel" in case + ), "dispatch_action case requires an expectation" + action_payload = case["actionPayload"] + surface_id = case.get("surfaceId", "main") + source_comp_id = case.get("sourceComponentId", "test_comp") + user_activation = bool(case.get("userActivation", True)) + expect_error = case.get("expectError") + + default_cat = _build_dispatch_action_catalog(case) + data_model = DataModel(case.get("dataModel") or {}) surface = SurfaceModel( surface_id=surface_id, default_catalog=default_cat, @@ -2507,8 +3296,7 @@ def _on_surf_err(err: dict[str, Any]) -> None: captured_renderer_errors.append({"version": ver, "error": dict(err)}) surface.on_error.subscribe(_on_surf_err) - - context = DataContext(surface=surface, path=scope or "/") + context = DataContext(surface=surface, path=case.get("scope") or "/") if expect_error: with assert_raises(expect_error, captured_renderer_errors): @@ -2538,9 +3326,9 @@ def _on_surf_err(err: dict[str, Any]) -> None: source_component_id=source_comp_id, metadata=case.get("metadata"), ) - _assert_dispatched_action_event(case, expect_dispatched, dispatched) - if expect_data_model is not None: - assert data_model.get("/") == expect_data_model + _assert_dispatched_action_event(case, case.get("expectDispatched"), dispatched) + if case.get("expectDataModel") is not None: + assert data_model.get("/") == case["expectDataModel"] def _normalize_version(version: Any) -> str: @@ -2585,6 +3373,26 @@ def dispatch_error(self, error: dict[str, Any]) -> None: raise exc +_EVALUATE_FUNCTION_CASE_KEYS: frozenset[str] = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "catalogId", + "catalogPaths", + "dataModel", + "expect", + "expect_error", + "expectError", + "function", + "locale", + "protocolVersion", + "surface", + "useBasicCatalog", +}) +_EVALUATE_FUNCTION_SURFACE_KEYS: frozenset[str] = frozenset({ + "catalogId", +}) + + def _evaluate_function_on_surface(case: dict[str, Any]) -> Any: """Evaluates the case's function as a call on a surface (the `surface` block). @@ -2593,6 +3401,7 @@ def _evaluate_function_on_surface(case: dict[str, Any]) -> Any: catalog is the one `surface.catalogId` names, if any. The call goes through the SDK's catalog resolution, naming the case's call-level `catalogId`. """ + _assert_known_keys(case["surface"], _EVALUATE_FUNCTION_SURFACE_KEYS, "surface key") version = case["protocolVersion"] available = { cat.catalog_id: cat @@ -2617,46 +3426,67 @@ def _evaluate_function_on_surface(case: dict[str, Any]) -> Any: return context.resolve_dynamic_value(call) -def validate_evaluate_function_case(case: dict[str, Any]) -> None: +def _evaluate_function_standalone(case: dict[str, Any]) -> Any: func_name = case["function"] args = case["args"] - data_model_dict = case.get("dataModel") or {} - locale = case.get("locale", "en-US") - expect_error = case.get("expectError") or case.get("expect_error") - - if "surface" in case: - - def _invoke() -> Any: - return _evaluate_function_on_surface(case) + catalogs = get_catalogs_for_test_case(case) + default_cat = catalogs[0] if catalogs else v09_catalog + surface = SurfaceModel( + surface_id="main", + default_catalog=default_cat, + data_model=DataModel(case.get("dataModel") or {}), + ) + surface.locale = case.get("locale", "en-US") + context = DataContext(surface=surface, path="/") + functions = getattr(default_cat, "functions", None) + if functions and func_name in functions: + fn = functions[func_name] + if getattr(fn, "execute_func", None) is not None: + return fn.execute_func(args, context) + return fn.execute(args, context) + return context.resolve_dynamic_value({"call": func_name, "args": args}) - else: - catalogs = get_catalogs_for_test_case(case) - default_cat = catalogs[0] if catalogs else v09_catalog - data_model = DataModel(data_model_dict) - surface = SurfaceModel( - surface_id="main", - default_catalog=default_cat, - data_model=data_model, - ) - surface.locale = locale - context = DataContext(surface=surface, path="/") - - def _invoke() -> Any: - if ( - default_cat - and hasattr(default_cat, "functions") - and func_name in default_cat.functions - ): - fn = default_cat.functions[func_name] - if getattr(fn, "execute_func", None) is not None: - return fn.execute_func(args, context) - return fn.execute(args, context) - return context.resolve_dynamic_value({"call": func_name, "args": args}) +def validate_evaluate_function_case(case: dict[str, Any]) -> None: + _assert_known_keys(case, _EVALUATE_FUNCTION_CASE_KEYS, "case key") + assert ( + "expect" in case or "expectError" in case or "expect_error" in case + ), "evaluate_function case requires 'expect' or 'expectError'" + expect_error = case.get("expectError") or case.get("expect_error") + runner = ( + _evaluate_function_on_surface + if "surface" in case + else _evaluate_function_standalone + ) if expect_error: with assert_raises(expect_error): - _invoke() + runner(case) else: - result = _invoke() - expected = case.get("expect") - assert result == expected + assert runner(case) == case.get("expect") + + +_ACTION_HANDLERS: dict[str, Callable[[dict[str, Any]], None]] = { + "accessibility_check": validate_accessibility_check_case, + "agent_to_renderer_schema": validate_agent_to_renderer_schema_case, + "common_types_schema": validate_common_types_schema_case, + "data_model": validate_data_model_case, + "derive": validate_derive_case, + "dispatch_action": validate_dispatch_action_case, + "evaluate_function": validate_evaluate_function_case, + "from_json": validate_from_json_case, + "get_client_capabilities": validate_capabilities_case, + "get_client_data_model": validate_get_renderer_data_model_case, + "get_renderer_capabilities": validate_capabilities_case, + "get_renderer_data_model": validate_get_renderer_data_model_case, + "handle_rpc": validate_handle_rpc_case, + "parse_expression_template": validate_parse_expression_template_case, + "process_messages": validate_process_messages_case, + "resolve_nodes": _validate_resolve_nodes_case, + "resolve_path": validate_resolve_path_case, + "round_trip": validate_round_trip_case, + "select_catalog": validate_select_catalog_case, + "to_json": validate_to_json_case, + "validate": validate_pure_validation_case, + "validate_common_type": validate_common_type_case, + "validation_schema": validate_validation_schema_case, +} From 2084dd477b65cafe5ae7d30396e14651051f2f75 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 05:54:52 +0000 Subject: [PATCH 12/48] test(core): address review feedback and fix Swift conformance test compilation --- python/a2ui_core/tests/node_resolution_runner.py | 4 ++-- .../core/Tests/A2UIConformanceTests/ConformanceManifest.swift | 3 +-- .../A2UIConformanceTests/ExpressionsConformanceTests.swift | 1 + .../MessageProcessorConformanceTests.swift | 2 +- .../core/web_core/tests/conformance/conformance_test.mjs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/a2ui_core/tests/node_resolution_runner.py b/python/a2ui_core/tests/node_resolution_runner.py index 7c1fb3a84c..1156409bcc 100644 --- a/python/a2ui_core/tests/node_resolution_runner.py +++ b/python/a2ui_core/tests/node_resolution_runner.py @@ -358,11 +358,11 @@ def _check_node_identity( expected: dict[str, Any], reason: str, ) -> None: - for path in expected.get("same_nodes", ()): + for path in expected.get("same_nodes") or (): assert _required_node(after, path) is _required_node( before, path ), f"{reason}: {path} identity" - for path in expected.get("replaced_nodes", ()): + for path in expected.get("replaced_nodes") or (): assert _required_node(after, path) is not _required_node( before, path ), f"{reason}: {path} replacement" diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift index d0991c93ef..40088745b0 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift @@ -63,8 +63,7 @@ public enum ConformanceManifest { } guard - let jsonData = try? JSONSerialization.data(withJSONObject: entry, options: [.sortedKeys]), - var lineData = String(data: jsonData, encoding: .utf8)?.data(using: .utf8) + var lineData = try? JSONSerialization.data(withJSONObject: entry, options: [.sortedKeys]) else { return } diff --git a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift index 9989ce5750..a12f1478f6 100644 --- a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift @@ -15,6 +15,7 @@ import A2UICore import BasicCatalog import Foundation +import OrderedCollections import OrderedJSON import Testing diff --git a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift index 99bc5d9108..f3365a5c7a 100644 --- a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift @@ -488,7 +488,7 @@ struct MessageProcessorConformanceTests { _ step: [String: Any], processor: MessageProcessor, handler: ConformanceActionCaptureHandler, - expectedRuntimeError: ConformanceExpectRendererError?, + expectedRuntimeError: ExpectedRuntimeError?, name: String ) throws { let payload = ConformanceTestHelper.toJSONValue(step["messages"] ?? [:]) diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index 716ba7b100..5ece67c066 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -309,7 +309,7 @@ function assertKnownKeysRecord(record, allowed, where) { function hasNestedKeyPath(obj, dottedPath) { let cur = obj; for (const segment of dottedPath.split('.')) { - if (!cur || typeof cur !== 'object' || !(segment in cur)) return false; + if (!cur || typeof cur !== 'object' || !Object.hasOwn(cur, segment)) return false; cur = cur[segment]; } return cur !== undefined; From 4e3482a31ba0caca87f3d85c21af9fba10e649c4 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:52:49 +0000 Subject: [PATCH 13/48] test(conformance): propagate unknown-key errors through xfail and audit scaling and skip tables Re-raise UnknownConformanceKeyError from expected-failure wrappers in Python, Dart, and Swift so YAML schema drift is not recorded as xfailed. Verify SKIP_TEST_NAMES in TypeScript verifyHarnessInvariants, add key validation and manifest recording to Swift MessageProcessorScalingConformanceTests, and update the Swift CI exclude reason. --- .github/workflows/swift_build_and_test.yml | 2 +- .../test/conformance/conformance_harness.dart | 17 +++- .../conformance/conformance_harness_test.dart | 23 ++++- .../a2ui_core/tests/node_resolution_runner.py | 6 +- python/a2ui_core/tests/test_conformance.py | 28 ++++++- .../ConformanceManifest.swift | 14 ++++ .../ConformanceTestHelper.swift | 6 +- ...sageProcessorScalingConformanceTests.swift | 83 ++++++++++++++++--- .../SuiteRegistryTests.swift | 6 ++ .../tests/conformance/conformance_test.mjs | 12 +++ 10 files changed, 176 insertions(+), 21 deletions(-) diff --git a/.github/workflows/swift_build_and_test.yml b/.github/workflows/swift_build_and_test.yml index 0b949407d1..d7ec4efceb 100644 --- a/.github/workflows/swift_build_and_test.yml +++ b/.github/workflows/swift_build_and_test.yml @@ -65,7 +65,7 @@ jobs: --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ --harness swift-core \ --corpus core \ - --exclude core/message_processor_scaling.yaml="no Swift timing harness" + --exclude "core/message_processor_scaling.yaml=performance: runs in the test (performance) job with A2UI_RUN_TIMING=1" ios-build: # `swift test` above builds for macOS only, so `#if os(iOS)` and UIKit code diff --git a/dart/a2ui_core/test/conformance/conformance_harness.dart b/dart/a2ui_core/test/conformance/conformance_harness.dart index 676defbc3c..52908470dc 100644 --- a/dart/a2ui_core/test/conformance/conformance_harness.dart +++ b/dart/a2ui_core/test/conformance/conformance_harness.dart @@ -46,6 +46,16 @@ const Set knownExpectErrorKeys = { 'surfaceId', }; +/// Thrown by [assertKnownKeys] when a conformance YAML object contains an +/// unrecognized key. +/// +/// Extends [TestFailure] so `throwsA(isA())` matchers continue to +/// work, while [_runExpectedFailureCase] can rethrow unknown-key errors instead +/// of treating them as expected SDK failures. +class UnknownConformanceKeyError extends TestFailure { + UnknownConformanceKeyError(super.message); +} + /// Fails the current test if [obj] contains any keys not in [allowed]. void assertKnownKeys( Map obj, @@ -55,7 +65,7 @@ void assertKnownKeys( final List unknown = obj.keys.where((k) => !allowed.contains(k)).toList()..sort(); if (unknown.isNotEmpty) { - fail('Unhandled keys at $where: $unknown'); + throw UnknownConformanceKeyError('Unhandled keys at $where: $unknown'); } } @@ -277,6 +287,11 @@ Future _runExpectedFailureCase( caughtError = error; } + if (caughtError is UnknownConformanceKeyError) { + _recordIfSuite(suite, name, 'failed', '$caughtError', manifestDir); + throw caughtError; + } + if (caughtError != null) { _recordIfSuite(suite, name, 'xfailed', reason, manifestDir); onSkip('expected failure: $reason ($caughtError)'); diff --git a/dart/a2ui_core/test/conformance/conformance_harness_test.dart b/dart/a2ui_core/test/conformance/conformance_harness_test.dart index 7126b8c2ef..d18ab00d2a 100644 --- a/dart/a2ui_core/test/conformance/conformance_harness_test.dart +++ b/dart/a2ui_core/test/conformance/conformance_harness_test.dart @@ -317,7 +317,7 @@ void main() { 'case', ), throwsA( - isA().having( + isA().having( (e) => e.message, 'message', contains('Unhandled keys at case: [bogus]'), @@ -325,6 +325,27 @@ void main() { ), ); }); + + test('expectedFailures does not swallow assertKnownKeys failures', + () async { + final skippedMessages = []; + await expectLater( + () => runConformanceCase( + const {'name': 'xfailing_case', 'bogus': true}, + () { + assertKnownKeys( + const {'name': 'xfailing_case', 'bogus': true}, + const {'name'}, + 'case', + ); + }, + expectedFailures: const {'xfailing_case': 'known gap'}, + onSkip: skippedMessages.add, + ), + throwsA(isA()), + ); + expect(skippedMessages, isEmpty); + }); }); group('recordManifest', () { diff --git a/python/a2ui_core/tests/node_resolution_runner.py b/python/a2ui_core/tests/node_resolution_runner.py index 1156409bcc..9506810a6d 100644 --- a/python/a2ui_core/tests/node_resolution_runner.py +++ b/python/a2ui_core/tests/node_resolution_runner.py @@ -85,6 +85,10 @@ } +class UnknownConformanceKeyError(AssertionError): + """Raised when a conformance YAML dictionary contains an unrecognized key.""" + + def _assert_known_keys( obj: Mapping[str, Any], allowed: frozenset[str], @@ -93,7 +97,7 @@ def _assert_known_keys( ) -> None: for key in obj: if key not in allowed: - raise AssertionError(f"Unknown {where} '{key}' in {context}") + raise UnknownConformanceKeyError(f"Unknown {where} '{key}' in {context}") @dataclass diff --git a/python/a2ui_core/tests/test_conformance.py b/python/a2ui_core/tests/test_conformance.py index 935bd140ae..7b8ebb52ff 100644 --- a/python/a2ui_core/tests/test_conformance.py +++ b/python/a2ui_core/tests/test_conformance.py @@ -58,7 +58,10 @@ ) try: - from .node_resolution_runner import run_node_resolution_case + from .node_resolution_runner import ( + UnknownConformanceKeyError, + run_node_resolution_case, + ) except ImportError: import importlib.util as _importlib_util @@ -69,6 +72,7 @@ assert _nr_spec and _nr_spec.loader _nr_mod = _importlib_util.module_from_spec(_nr_spec) _nr_spec.loader.exec_module(_nr_mod) + UnknownConformanceKeyError = _nr_mod.UnknownConformanceKeyError run_node_resolution_case = _nr_mod.run_node_resolution_case CATEGORY_TO_EXCEPTION = { @@ -161,7 +165,7 @@ def _assert_known_keys( target = case_key or _CURRENT_CASE_KEY or str(obj.get("name", "")) for key in obj: if key not in allowed: - pytest.fail(f"Unknown {where} '{key}' in {target}") + raise UnknownConformanceKeyError(f"Unknown {where} '{key}' in {target}") # Actions that time an operation rather than check what it returns. Their cases @@ -909,7 +913,7 @@ def _dispatch_conformance_action( ), "resolve_nodes is only supported in core/node_resolution.yaml" handler = _ACTION_HANDLERS.get(action) if handler is None: - pytest.fail( + raise UnknownConformanceKeyError( f"Action '{action}' has no handler in the core Python harness." " Add one, or add the suite to UNRUNNABLE_SUITES with a reason." ) @@ -945,6 +949,8 @@ def test_conformance_suite(test_id: str, rel_path: str, case: dict[str, Any]) -> if case_key in EXPECTED_FAILURES: try: _dispatch_conformance_action(rel_path, action, case) + except UnknownConformanceKeyError: + raise except (Exception, pytest.fail.Exception): pytest.xfail(EXPECTED_FAILURES[case_key]) pytest.fail(f"Expected failure '{case_key}' passed unexpectedly.") @@ -1033,6 +1039,22 @@ def test_resolve_nodes_outside_its_suite_is_unsupported() -> None: ) +def test_expected_failure_does_not_swallow_unknown_keys() -> None: + expected_key = next(iter(EXPECTED_FAILURES)) + rel_path, case_name = expected_key.split("#", 1) + bad_case = { + "name": case_name, + "action": "handle_rpc", + "unexpectedBogusKey": True, + } + with pytest.raises(UnknownConformanceKeyError, match="unexpectedBogusKey"): + test_conformance_suite( + f"{rel_path}::{case_name}", + rel_path, + bad_case, + ) + + def test_catalog_conformance_covers_published_catalogs() -> None: """Every published catalog has a from_json selfContained case in catalog.yaml.""" repo_root = os.path.abspath(os.path.join(CONFORMANCE_ROOT, "..")) diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift index 40088745b0..ad88dcf1e8 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift @@ -100,13 +100,20 @@ public enum ConformanceManifest { return } if let knownIssueReason { + var unknownKeyError: UnknownConformanceKeyError? withKnownIssue(Comment(rawValue: knownIssueReason)) { do { try body() + } catch let error as UnknownConformanceKeyError { + unknownKeyError = error } catch { Issue.record(error) } } + if let unknownKeyError { + record(testCase, status: .failed, reason: unknownKeyError.description) + throw unknownKeyError + } record(testCase, status: .xfailed, reason: knownIssueReason) return } @@ -131,13 +138,20 @@ public enum ConformanceManifest { return } if let knownIssueReason { + var unknownKeyError: UnknownConformanceKeyError? await withKnownIssue(Comment(rawValue: knownIssueReason)) { do { try await body() + } catch let error as UnknownConformanceKeyError { + unknownKeyError = error } catch { Issue.record(error) } } + if let unknownKeyError { + record(testCase, status: .failed, reason: unknownKeyError.description) + throw unknownKeyError + } record(testCase, status: .xfailed, reason: knownIssueReason) return } diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift index 0dc4544455..2163f1f2ad 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift @@ -1317,7 +1317,7 @@ extension ConformanceTestHelper { } /// Verifies that every key in `dict` belongs to `allowed` (or `metadataCaseKeys` when - /// `allowMetadata` is true), recording an issue and throwing on the first unknown key. + /// `allowMetadata` is true), throwing `UnknownConformanceKeyError` on the first unknown key. public static func assertKnownKeys( _ dict: [String: Any], _ allowed: Set, @@ -1327,13 +1327,12 @@ extension ConformanceTestHelper { let permitted = allowMetadata ? allowed.union(metadataCaseKeys) : allowed for key in dict.keys.sorted() where !permitted.contains(key) { let msg = "Unknown \(location) key '\(key)' (allowed: \(permitted.sorted()))" - Issue.record(Comment(rawValue: msg)) throw UnknownConformanceKeyError(msg) } } /// Verifies that every key in `dict` belongs to `allowed` (or `metadataCaseKeys` when - /// `allowMetadata` is true), recording an issue and throwing on the first unknown key. + /// `allowMetadata` is true), throwing `UnknownConformanceKeyError` on the first unknown key. public static func assertKnownKeys( _ dict: OrderedDictionary, _ allowed: Set, @@ -1343,7 +1342,6 @@ extension ConformanceTestHelper { let permitted = allowMetadata ? allowed.union(metadataCaseKeys) : allowed for key in dict.keys.sorted() where !permitted.contains(key) { let msg = "Unknown \(location) key '\(key)' (allowed: \(permitted.sorted()))" - Issue.record(Comment(rawValue: msg)) throw UnknownConformanceKeyError(msg) } } diff --git a/swift/core/Tests/A2UIConformanceTests/MessageProcessorScalingConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MessageProcessorScalingConformanceTests.swift index 7822f19959..358d976c8c 100644 --- a/swift/core/Tests/A2UIConformanceTests/MessageProcessorScalingConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MessageProcessorScalingConformanceTests.swift @@ -32,11 +32,26 @@ struct MessageProcessorScalingConformanceTests { /// time to resolve a surface arriving in n messages grows with n squared. At the sizes of the /// composition case, 1000 and 4000, that dominates; making resolution incremental (#3231) is /// expected to remove it. - private static let knownGaps: [String: String] = [ + static let knownGaps: [String: String] = [ "test_scaling_process_single_component_messages_composition_v10": "the node tree is resolved again after each message (#3231)" ] + private static let knownCaseKeys: Set = [ + "action", + "args", + "catalogPaths", + "max_ratio", + "operation", + "payload", + "protocolVersion", + "sizes", + ] + + private static let knownArgsKeys: Set = [ + "validation_config" + ] + @Test(.enabled(if: ConformanceTiming.isEnabled, "set A2UI_RUN_TIMING=1 to run timing cases")) func messageProcessorScalingConformance() throws { let rawYAML = try ConformanceTestHelper.loadYAML(filename: Self.suite) @@ -49,22 +64,68 @@ struct MessageProcessorScalingConformanceTests { ) #expect(!cases.isEmpty, "\(Self.suite) should hold test cases") var executed = 0 - for (rawCase, testCase) in zip(rawCases, cases) - where ConformanceTiming.runsCase(named: testCase.name) { - if try runCase(rawCase, testCase) { + for (rawCase, testCase) in zip(rawCases, cases) { + if !ConformanceTiming.runsCase(named: testCase.name) { + ConformanceManifest.record( + suite: Self.suite, + name: testCase.name, + status: .skipped, + reason: "filtered by A2UI_TIMING_CASE" + ) + continue + } + if try runAndRecordCase(rawCase, testCase) { executed += 1 } } #expect(executed > 0, "no timing case of \(Self.suite) ran") } - /// Runs a timing case, returning false when it is a known gap and was skipped. - private func runCase(_ rawCase: [String: Any], _ testCase: ConformanceTestCase) throws -> Bool { - let name = testCase.name - if let reason = Self.knownGaps[name] { - print("[\(name)] skipped: \(reason)") + private func runAndRecordCase( + _ rawCase: [String: Any], + _ testCase: ConformanceTestCase + ) throws -> Bool { + if let reason = Self.knownGaps[testCase.name] { + print("[\(testCase.name)] skipped: \(reason)") + ConformanceManifest.record( + suite: Self.suite, + name: testCase.name, + status: .skipped, + reason: reason + ) return false } + do { + try runCase(rawCase, testCase) + ConformanceManifest.record(suite: Self.suite, name: testCase.name, status: .passed) + return true + } catch { + ConformanceManifest.record( + suite: Self.suite, + name: testCase.name, + status: .failed, + reason: String(describing: error) + ) + throw error + } + } + + /// Runs a timing case and asserts its median growth ratio stays within `max_ratio`. + private func runCase(_ rawCase: [String: Any], _ testCase: ConformanceTestCase) throws { + let name = testCase.name + try ConformanceTestHelper.assertKnownKeys( + rawCase, + Self.knownCaseKeys, + "\(Self.suite) case '\(name)'" + ) + if let argsDict = rawCase["args"] as? [String: Any] { + try ConformanceTestHelper.assertKnownKeys( + argsDict, + Self.knownArgsKeys, + "\(Self.suite) args in '\(name)'", + allowMetadata: false + ) + } #expect(testCase.action == "measure_growth", "[\(name)] unknown action") #expect(rawCase["operation"] as? String == "process_messages", "[\(name)] unknown operation") let template = try #require(rawCase["payload"] as? [Any], "[\(name)] has no payload") @@ -108,6 +169,8 @@ struct MessageProcessorScalingConformanceTests { + "at \(sizes[0]): \(result.baseline), ratio \(result.ratio) (max \(maxRatio))" print(report) #expect(result.ratio <= maxRatio, "\(report)") - return true + if result.ratio > maxRatio { + throw ConformanceTimingError(report) + } } } diff --git a/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift b/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift index f2c5fa78c3..59af47a5af 100644 --- a/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/SuiteRegistryTests.swift @@ -71,6 +71,12 @@ struct SuiteRegistryTests { for knownIssueName in NodeResolutionConformanceTests.placeholderKnownIssues.keys { #expect(nodeResNames.contains(knownIssueName), "Unused known issue '\(knownIssueName)'") } + + let scalingNames = Set( + ConformanceTestHelper.cases("core/message_processor_scaling.yaml").map(\.name)) + for gapName in MessageProcessorScalingConformanceTests.knownGaps.keys { + #expect(scalingNames.contains(gapName), "Unused scaling knownGap '\(gapName)'") + } } private func verifySupportedSuiteActions() throws { diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index 5ece67c066..4dfa07df4f 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -538,6 +538,18 @@ function verifyHarnessInvariants(discoveredSuites, state) { ); } } + const skipNameKeys = + SKIP_TEST_NAMES instanceof Map ? SKIP_TEST_NAMES.keys() : SKIP_TEST_NAMES.values(); + for (const name of skipNameKeys) { + if (!state.matchedSkippedNames.has(name)) { + recordInvariantFailure( + state, + 'SKIP_TEST_NAMES', + name, + 'Skipped test name matches no test case.', + ); + } + } } function loadSuiteTestCases(filePath, relativePath, state) { From 40ab5df29c1aab7f558c13b542590903eb181241 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 07:14:58 +0000 Subject: [PATCH 14/48] test(core): fix Dart scaling known keys and Swift conformance build Allow the payload key in Dart message_processor_scaling_conformance_test, handle .integer in ConformanceTestHelper.fromJSONValue, isolate ConformanceManifest.runAsync to MainActor, and unwrap optional action string interpolation in ValidatorConformanceTests. --- .../conformance/message_processor_scaling_conformance_test.dart | 2 +- swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift | 1 + .../core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift | 2 ++ .../Tests/A2UIConformanceTests/ValidatorConformanceTests.swift | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart b/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart index dafebd8bdc..232fca4764 100644 --- a/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/message_processor_scaling_conformance_test.dart @@ -35,9 +35,9 @@ const Map> _knownKeys = { ...caseMetadataKeys, 'args', 'catalogPaths', - 'generator', 'max_ratio', 'operation', + 'payload', 'protocolVersion', 'sizes', }, diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift index ad88dcf1e8..f72547eb36 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift @@ -127,6 +127,7 @@ public enum ConformanceManifest { } /// Executes an asynchronous `body` for `testCase` and records the outcome in the manifest. + @MainActor public static func runAsync( _ testCase: ConformanceCase, skippedReason: String? = nil, diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift index 2163f1f2ad..9b9c2cdac9 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift @@ -1306,6 +1306,8 @@ extension ConformanceTestHelper { return str case .boolean(let bool): return bool + case .integer(let int): + return int case .number(let double): if let exactInt = Int(exactly: double) { return exactInt diff --git a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift index 252c10d8a0..9c4345f32a 100644 --- a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift @@ -97,7 +97,7 @@ struct ValidatorConformanceTests { ) guard testCase.action == "validate" else { throw UnknownConformanceKeyError( - "[\(conformanceCase.name)] Unknown validator action '\(testCase.action)'" + "[\(conformanceCase.name)] Unknown validator action '\(testCase.action ?? "")'" ) } let targetVersion = resolveTargetVersion(for: testCase, versionPrefix: versionPrefix) From 61b83408282270be0a57494965440609d7167f09 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 07:20:21 +0000 Subject: [PATCH 15/48] test(swift-core): avoid exclusivity conflict on captured expression errors Wrap captured RendererError values in a reference box in ExpressionsConformanceTests so outboundListener does not mutate a local array while assertValidateStepError holds an active reference. --- .../ExpressionsConformanceTests.swift | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift index a12f1478f6..5cecaa4d7e 100644 --- a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift @@ -19,6 +19,10 @@ import OrderedCollections import OrderedJSON import Testing +private final class ExpressionErrorCaptureBox: @unchecked Sendable { + var errors: [RendererError] = [] +} + /// Runs the shared `conformance/core/expressions.yaml` suite. struct ExpressionsConformanceTests { private static let knownParseCaseKeys: Set = [ @@ -171,25 +175,25 @@ struct ExpressionsConformanceTests { ) // Expression errors raised while resolving nodes reach the surface as `EXPRESSION_ERROR` // rather than being thrown out of `processMessages`. - var capturedErrors: [RendererError] = [] + let capturedErrors = ExpressionErrorCaptureBox() processor.outboundListener = { message in MainActor.assumeIsolated { if case .error(let error) = message { - capturedErrors.append(error) + capturedErrors.errors.append(error) } } } for (stepIndex, step) in testCase.steps.enumerated() { guard let payload = step.payload else { continue } - capturedErrors.removeAll() + capturedErrors.errors.removeAll() let isFinalStep = stepIndex == testCase.steps.count - 1 if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { assertValidateStepError( payload: payload, processor: processor, - capturedErrors: &capturedErrors, + capturedErrors: capturedErrors, expectedError: expectedError, testName: testCase.name, stepIndex: stepIndex @@ -199,8 +203,8 @@ struct ExpressionsConformanceTests { try processor.processMessages(payload) #expect( - capturedErrors.isEmpty, - "\(testCase.name): unexpected error at step \(stepIndex): \(capturedErrors)" + capturedErrors.errors.isEmpty, + "\(testCase.name): unexpected error at step \(stepIndex): \(capturedErrors.errors)" ) let expectedSurfaces = (step.expect ?? testCase.expect)?["surfaces"]?.objectValue ?? [:] @@ -216,7 +220,7 @@ struct ExpressionsConformanceTests { private func assertValidateStepError( payload: JSONValue, processor: MessageProcessor, - capturedErrors: inout [RendererError], + capturedErrors: ExpressionErrorCaptureBox, expectedError: ConformanceExpectError, testName: String, stepIndex: Int @@ -228,7 +232,7 @@ struct ExpressionsConformanceTests { let message = (error as? (any A2UIError))?.message ?? error.localizedDescription errorMessages.append(message) } - for error in capturedErrors { + for error in capturedErrors.errors { switch error { case .validationFailed(let validationError): errorMessages.append(validationError.message) From 69a25ab1a3d2b67b57fc63c4aa2a1c92e83fbf3e Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 07:25:28 +0000 Subject: [PATCH 16/48] test(swift-core): handle DataError category in ValidatorConformanceTests Add DataError case to ValidatorConformanceTests.assertErrorCategory so test_error_response_update_data_model_prototype_pollution verifies A2UIDataError. --- .../A2UIConformanceTests/ValidatorConformanceTests.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift index 9c4345f32a..17ecf58c3e 100644 --- a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift @@ -562,6 +562,11 @@ struct ValidatorConformanceTests { error is A2UICatalogError, "[\(testName)] Expected A2UICatalogError for '\(category)', got \(type(of: error))" ) + case "DataError": + #expect( + error is A2UIDataError, + "[\(testName)] Expected A2UIDataError for '\(category)', got \(type(of: error))" + ) default: Issue.record("[\(testName)] Unknown error category '\(category)'") throw UnknownConformanceKeyError("[\(testName)] Unknown error category '\(category)'") From 596975f6e27597aea7ad1403a06329719322686f Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 07:34:14 +0000 Subject: [PATCH 17/48] test(web_core): disable Wireit caching for test:conformance Why: Wireit cached test:conformance with an empty output list, skipping conformance_test.mjs on cache hits and leaving A2UI_CONFORMANCE_MANIFEST_DIR empty when CI runs check_executed_cases.py. What: Remove the files and output properties from the test:conformance Wireit script so it runs on every test invocation like test and test:timing. --- typescript/core/web_core/package.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/typescript/core/web_core/package.json b/typescript/core/web_core/package.json index baf2cbc99c..a4db48560d 100644 --- a/typescript/core/web_core/package.json +++ b/typescript/core/web_core/package.json @@ -252,12 +252,7 @@ "command": "node tests/conformance/conformance_test.mjs", "dependencies": [ "build" - ], - "files": [ - "tests/conformance/**", - "../../../conformance/**" - ], - "output": [] + ] }, "test:timing": { "command": "node --expose-gc --test --test-concurrency=1 \"tests/timing/**/*_test.mjs\"", From c3e39e9951b1c22d7bc93b0958a67a7fd41066de Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 16:45:01 +0000 Subject: [PATCH 18/48] test(swift-core): reconcile validate runner with upstream/main --- .../BasicCatalogConformanceTests.swift | 31 ++- .../ConformanceTestHelper.swift | 233 +++++++++++++++++- .../ExpressionsConformanceTests.swift | 132 +--------- 3 files changed, 251 insertions(+), 145 deletions(-) diff --git a/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift index ea13e778e6..6ee3d7960e 100644 --- a/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/BasicCatalogConformanceTests.swift @@ -384,21 +384,25 @@ struct BasicCatalogConformanceTests { @Test(arguments: ConformanceTestHelper.cases("core/functions.yaml", action: "validate")) func functionsValidateConformance(_ conformanceCase: ConformanceCase) throws { try ConformanceManifest.run(conformanceCase) { - try runValidatedCatalogCase(conformanceCase, suiteName: "functions.yaml") + let testCase = try validateAndParseCatalogCase( + conformanceCase, + suiteName: "functions.yaml" + ) + try ConformanceTestHelper.runValidateCase(testCase) } } @Test(arguments: ConformanceTestHelper.cases("core/index_function.yaml", action: "validate")) func indexFunctionConformance(_ conformanceCase: ConformanceCase) throws { try ConformanceManifest.run(conformanceCase) { - try runValidatedCatalogCase(conformanceCase, suiteName: "index_function.yaml") + try runIndexFunctionCase(conformanceCase) } } - private func runValidatedCatalogCase( + private func validateAndParseCatalogCase( _ conformanceCase: ConformanceCase, suiteName: String - ) throws { + ) throws -> ConformanceTestCase { let rawDict = conformanceCase.rawDictionary try ConformanceTestHelper.assertKnownKeys( rawDict, @@ -423,10 +427,17 @@ struct BasicCatalogConformanceTests { } } } - let testCase = try #require( + return try #require( conformanceCase.parsedTestCase(), "[\(conformanceCase.name)] Could not parse test case" ) + } + + private func runIndexFunctionCase(_ conformanceCase: ConformanceCase) throws { + let testCase = try validateAndParseCatalogCase( + conformanceCase, + suiteName: "index_function.yaml" + ) let errorHandler = IndexConformanceErrorHandler() let processor = MessageProcessor( catalogs: ConformanceTestHelper.basicCatalogs(for: .v10), @@ -434,10 +445,11 @@ struct BasicCatalogConformanceTests { validationConfig: ValidationConfig(targetVersion: "1.0") ) - for step in testCase.steps { + for (stepIndex, step) in testCase.steps.enumerated() { guard let payload = step.payload else { continue } try executeIndexStep( step: step, + isFinalStep: stepIndex == testCase.steps.count - 1, payload: payload, testCase: testCase, processor: processor, @@ -448,12 +460,14 @@ struct BasicCatalogConformanceTests { private func executeIndexStep( step: ConformanceStep, + isFinalStep: Bool, payload: JSONValue, testCase: ConformanceTestCase, processor: MessageProcessor, errorHandler: IndexConformanceErrorHandler ) throws { - if let expectedError = step.expectError { + // The case's expectError applies to the last step only. + if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { errorHandler.capturedErrors.removeAll() var caughtError: Error? do { @@ -471,7 +485,8 @@ struct BasicCatalogConformanceTests { } try processor.processMessages(payload) - guard let expectObj = testCase.expect?.objectValue else { return } + let caseExpect = isFinalStep ? testCase.expect : nil + guard let expectObj = (step.expect ?? caseExpect)?.objectValue else { return } try ConformanceTestHelper.assertKnownKeys( expectObj, ["surfaces"], diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift index 9b9c2cdac9..9b191dfef9 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceTestHelper.swift @@ -709,18 +709,16 @@ public enum ConformanceTestHelper { let stepDicts = (dictionary["steps"] as? [[String: Any]]) ?? (dictionary["validate"] as? [[String: Any]]) + // Explicit steps carry only their own expectations. The case's `expectError` and `expect` + // apply to the final step, which each runner checks against `ConformanceTestCase`. if let stepDicts { - let lastIndex = stepDicts.count - 1 - return stepDicts.enumerated().map { index, stepDictionary in + return stepDicts.map { stepDictionary in let stepPayload = (stepDictionary["payload"] ?? stepDictionary["messages"]).map { toJSONValue($0) } - let fallbackError = index == lastIndex ? defaultExpectError : nil let stepError = parseExpectError(stepDictionary["expect_error"] ?? stepDictionary["expectError"]) - ?? fallbackError - let stepExpect = - (stepDictionary["expect"] as? [String: Any]).map { toJSONValue($0) } ?? defaultExpect + let stepExpect = (stepDictionary["expect"] as? [String: Any]).map { toJSONValue($0) } return ConformanceStep(payload: stepPayload, expectError: stepError, expect: stepExpect) } } @@ -1357,6 +1355,229 @@ extension ConformanceTestHelper { let single = try decoder.decode(AgentToRendererMessage.self, from: data) return [single] } + + /// Runs a `validate` case that renders components on a surface: processes each step's payload, + /// checks the errors a step expects, and compares the resolved properties of the expected + /// components. + /// + /// The case-level `expectError` and `expect` apply to the final step only. + @MainActor + public static func runValidateCase(_ testCase: ConformanceTestCase) throws { + // Without a declared version, `PayloadValidator` and `MessageProcessor` infer it from the + // catalog and the surface. + let targetVersion = + testCase.protocolVersion + ?? testCase.catalogConfiguration?["protocolVersion"]?.stringValue + ?? testCase.catalogConfiguration?["protocol_version"]?.stringValue + let processor = MessageProcessor( + catalogs: try ConformanceTestHelper.buildCatalogs(for: testCase), + validationConfig: ValidationConfig(targetVersion: targetVersion) + ) + // Expression errors raised while resolving nodes reach the surface as `EXPRESSION_ERROR` + // rather than being thrown out of `processMessages`. + let capturedErrors = ValidateErrorCaptureBox() + processor.outboundListener = { message in + MainActor.assumeIsolated { + if case .error(let error) = message { + capturedErrors.errors.append(error) + } + } + } + + for (stepIndex, step) in testCase.steps.enumerated() { + guard let payload = step.payload else { + continue + } + capturedErrors.errors.removeAll() + + let isFinalStep = stepIndex == testCase.steps.count - 1 + if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { + try assertValidateStepFails( + payload: payload, + processor: processor, + expectedError: expectedError, + capturedErrors: { capturedErrors.errors }, + context: "\(testCase.name) step \(stepIndex)" + ) + continue + } + + try processor.processMessages(payload) + #expect( + capturedErrors.errors.isEmpty, + "\(testCase.name): unexpected error at step \(stepIndex): \(capturedErrors.errors)" + ) + + let activeExpect = step.expect ?? (isFinalStep ? testCase.expect : nil) + if let expectObj = activeExpect?.objectValue { + try assertKnownKeys( + expectObj, + ["surfaces"], + "expect in '\(testCase.name)'", + allowMetadata: false + ) + } + let expectedSurfaces = activeExpect?["surfaces"]?.objectValue ?? [:] + for (surfaceID, expectedSurface) in expectedSurfaces { + if let surfObj = expectedSurface.objectValue { + try assertKnownKeys( + surfObj, + ["components"], + "expect.surfaces.\(surfaceID) in '\(testCase.name)'", + allowMetadata: false + ) + } + let surface = try #require( + processor.surfaceGroupModel[surfaceID], + "\(testCase.name): expected surface '\(surfaceID)'" + ) + let expectedComponents = expectedSurface["components"]?.objectValue ?? [:] + for (componentID, expectedComponent) in expectedComponents { + let node = try #require( + surface.findNode(id: componentID), + "\(testCase.name): expected resolved node '\(componentID)'" + ) + try assertValidateNode( + node, + matches: expectedComponent, + context: "\(testCase.name): '\(componentID)'" + ) + } + } + } + } + + /// Processes a `validate` step that must fail and checks the error against `expectedError`. + /// + /// A thrown error must be an `A2UIError` of the expected category. An error raised while + /// resolving nodes, such as an expression that fails to parse, is reported on the surface + /// instead of thrown; like the Dart runner, it counts as a `ValidationError`. + @MainActor + private static func assertValidateStepFails( + payload: JSONValue, + processor: MessageProcessor, + expectedError: ConformanceExpectError, + capturedErrors: () -> [RendererError], + context: String + ) throws { + var thrownError: (any Error)? + do { + try processor.processMessages(payload) + } catch { + thrownError = error + } + let reportedMessages = capturedErrors().map { error in + switch error { + case .validationFailed(let validationError): validationError.message + case .generic(let genericError): genericError.message + } + } + + var errorMessages: [String] = [] + if let thrownError { + guard let a2uiError = thrownError as? any A2UIError else { + Issue.record("\(context): unexpected non-A2UI error: \(thrownError)") + return + } + errorMessages.append(a2uiError.message) + if let category = expectedError.category { + let matchesCategory = try errorMatchesCategory( + a2uiError, + category: category, + context: context + ) + #expect( + matchesCategory, + "\(context): expected a \(category), got \(type(of: a2uiError)): \(a2uiError.message)" + ) + } + } else if !reportedMessages.isEmpty, let category = expectedError.category { + #expect( + category == "ValidationError", + "\(context): expected a thrown \(category), got surface errors \(reportedMessages)" + ) + } + errorMessages += reportedMessages + + #expect(!errorMessages.isEmpty, "\(context): expected an error") + if let expectedMessage = expectedError.message { + #expect( + errorMessages.contains { messageMatches($0, expected: expectedMessage) }, + "\(context): messages \(errorMessages) do not match '\(expectedMessage)'" + ) + } + } + + /// Whether `error` belongs to a conformance `expectError.category`, as in the Dart runner. + private static func errorMatchesCategory( + _ error: any A2UIError, + category: String, + context: String + ) throws -> Bool { + switch category { + case "ValidationError": + return error is A2UIValidationError || error is A2UIIntegrityError + || error is A2UIRecursionError || error is A2UICatalogError + case "CatalogError": return error is A2UICatalogError + case "IntegrityError": return error is A2UIIntegrityError + case "RecursionError": return error is A2UIRecursionError + case "DataError": return error is A2UIDataError + case "ParseError": return error is A2UIParseError + default: + throw UnknownConformanceKeyError("\(context): unknown error category '\(category)'") + } + } + + /// Checks a resolved node against the expected component of a `validate` case: its type and + /// the text of each listed string property. + @MainActor + private static func assertValidateNode( + _ node: A2UICore.Node, + matches expectedComponent: JSONValue, + context: String + ) throws { + if let expectedType = expectedComponent["component"]?.stringValue { + #expect( + node.type == expectedType, + "\(context) has type '\(node.type)', expected '\(expectedType)'" + ) + } + for (property, expectedValue) in expectedComponent.objectValue ?? [:] + where property != "component" { + let binding = try #require( + node.properties[property] as? DataBinding, + "\(context).\(property) is not a string binding" + ) + // A `Text` renders a scalar result as its JSON text. + let expectedText: String? + switch expectedValue { + case .string(let string): + expectedText = string + case .boolean(let boolean): + expectedText = String(boolean) + case .integer(let integer): + expectedText = String(integer) + case .number(let number): + expectedText = Int(exactly: number).map(String.init) ?? String(number) + case .null: + expectedText = nil + case .array, .object: + Issue.record("\(context).\(property): unsupported expected value \(expectedValue)") + continue + } + #expect( + binding.value == expectedText, + """ + \(context).\(property) is \(String(describing: binding.value)), \ + expected \(String(describing: expectedText)) + """ + ) + } + } +} + +private final class ValidateErrorCaptureBox: @unchecked Sendable { + var errors: [RendererError] = [] } @MainActor diff --git a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift index 5cecaa4d7e..9016be0595 100644 --- a/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ExpressionsConformanceTests.swift @@ -15,14 +15,9 @@ import A2UICore import BasicCatalog import Foundation -import OrderedCollections import OrderedJSON import Testing -private final class ExpressionErrorCaptureBox: @unchecked Sendable { - var errors: [RendererError] = [] -} - /// Runs the shared `conformance/core/expressions.yaml` suite. struct ExpressionsConformanceTests { private static let knownParseCaseKeys: Set = [ @@ -163,132 +158,7 @@ struct ExpressionsConformanceTests { conformanceCase.parsedTestCase(), "[\(conformanceCase.name)] Could not parse validate case" ) - try runValidateCase(testCase) - } - } - - @MainActor - private func runValidateCase(_ testCase: ConformanceTestCase) throws { - let processor = MessageProcessor( - catalogs: try ConformanceTestHelper.buildCatalogs(for: testCase), - validationConfig: ValidationConfig(targetVersion: "1.0") - ) - // Expression errors raised while resolving nodes reach the surface as `EXPRESSION_ERROR` - // rather than being thrown out of `processMessages`. - let capturedErrors = ExpressionErrorCaptureBox() - processor.outboundListener = { message in - MainActor.assumeIsolated { - if case .error(let error) = message { - capturedErrors.errors.append(error) - } - } - } - - for (stepIndex, step) in testCase.steps.enumerated() { - guard let payload = step.payload else { continue } - capturedErrors.errors.removeAll() - - let isFinalStep = stepIndex == testCase.steps.count - 1 - if let expectedError = step.expectError ?? (isFinalStep ? testCase.expectError : nil) { - assertValidateStepError( - payload: payload, - processor: processor, - capturedErrors: capturedErrors, - expectedError: expectedError, - testName: testCase.name, - stepIndex: stepIndex - ) - continue - } - - try processor.processMessages(payload) - #expect( - capturedErrors.errors.isEmpty, - "\(testCase.name): unexpected error at step \(stepIndex): \(capturedErrors.errors)" - ) - - let expectedSurfaces = (step.expect ?? testCase.expect)?["surfaces"]?.objectValue ?? [:] - try assertValidateSurfaces( - OrderedDictionary(uniqueKeysWithValues: expectedSurfaces.map { ($0.key, $0.value) }), - on: processor, - testName: testCase.name - ) - } - } - - @MainActor - private func assertValidateStepError( - payload: JSONValue, - processor: MessageProcessor, - capturedErrors: ExpressionErrorCaptureBox, - expectedError: ConformanceExpectError, - testName: String, - stepIndex: Int - ) { - var errorMessages: [String] = [] - do { - try processor.processMessages(payload) - } catch { - let message = (error as? (any A2UIError))?.message ?? error.localizedDescription - errorMessages.append(message) - } - for error in capturedErrors.errors { - switch error { - case .validationFailed(let validationError): - errorMessages.append(validationError.message) - case .generic(let genericError): - errorMessages.append(genericError.message) - } - } - #expect( - !errorMessages.isEmpty, - "\(testName): expected an error at step \(stepIndex)" - ) - if let expectedMessage = expectedError.message { - #expect( - errorMessages.contains { $0.localizedCaseInsensitiveContains(expectedMessage) }, - "\(testName): messages \(errorMessages) do not contain '\(expectedMessage)'" - ) - } - } - - @MainActor - private func assertValidateSurfaces( - _ expectedSurfaces: OrderedDictionary, - on processor: MessageProcessor, - testName: String - ) throws { - for (surfaceID, expectedSurface) in expectedSurfaces { - if let surfObj = expectedSurface.objectValue { - try ConformanceTestHelper.assertKnownKeys( - surfObj, - ["components"], - "expressions.yaml expect.surfaces.\(surfaceID) in '\(testName)'", - allowMetadata: false - ) - } - let surface = try #require( - processor.surfaceGroupModel[surfaceID], - "\(testName): expected surface '\(surfaceID)'" - ) - let expectedComponents = expectedSurface["components"]?.objectValue ?? [:] - for (componentID, expectedComponent) in expectedComponents { - let node = try #require( - surface.findNode(id: componentID), - "\(testName): expected resolved node '\(componentID)'" - ) - for (property, expectedValue) in expectedComponent.objectValue ?? [:] - where property != "component" { - let binding = try #require( - node.properties[property] as? DataBinding, - "\(testName): '\(componentID).\(property)' is not a string binding" - ) - #expect( - binding.value == expectedValue.stringValue, - "\(testName): '\(componentID).\(property)' is '\(String(describing: binding.value))'" - ) - } - } + try ConformanceTestHelper.runValidateCase(testCase) } } From 3480e5743bad223263ef4b349d0f192704210ada Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:09:44 +0000 Subject: [PATCH 19/48] test(a2ui_a2a,a2ui_adk): strict dispatch and executed-case manifest in the extension harnesses --- .github/workflows/python_ci.yml | 10 + ...onformance_harness_implementation_guide.md | 25 + .../extensions/a2a/a2a_integration.yaml | 60 --- .../tests/conformance/test_a2a_integration.py | 351 +++++++++---- python/a2ui_a2a/tests/conftest.py | 114 +++++ .../tests/conformance/test_adk_extensions.py | 468 +++++++++++++----- python/a2ui_adk/tests/conftest.py | 114 +++++ 7 files changed, 881 insertions(+), 261 deletions(-) create mode 100644 python/a2ui_a2a/tests/conftest.py create mode 100644 python/a2ui_adk/tests/conftest.py diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml index 5a7ddb576f..413b3b9b62 100644 --- a/.github/workflows/python_ci.yml +++ b/.github/workflows/python_ci.yml @@ -119,6 +119,16 @@ jobs: --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ --harness python-core \ --corpus core + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --corpus extensions \ + --only 'extensions/a2a/**' \ + --harness python-a2a + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --corpus extensions \ + --only 'extensions/adk/**' \ + --harness python-adk - name: Set Up Dart uses: ./.github/actions/setup-dart - name: Run CLI Python Runtime Tests diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index 6c4eda1120..c829bf829c 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -340,6 +340,31 @@ Each case in `core/accessibility.yaml` (`action: accessibility_check`) provides Where a browser-based test runner cannot read YAML from disk at test time, parse the suite files in the runner configuration and inject them as JSON. For example, [typescript/renderers/angular/karma.conf.js](../typescript/renderers/angular/karma.conf.js) parses `accessibility.yaml`, `adapter_conformance.yaml`, and `catalogs/basic/v1/catalog.json` in Node and exposes them to Jasmine browser specs via `client.conformanceSuites`. +## Extension harnesses (`conformance/extensions/`) + +Protocol and agent-framework integrations (such as A2A and ADK) live outside the Core and Inference SDK packages and run the suites under `conformance/extensions/`. Each extension harness dispatches cases through an explicit `RUNNERS` map, rejects unknown actions and unknown `args`, `expect`, or `expectError` keys with `_assert_known_keys` against a per-handler `_KNOWN_KEYS` allowlist, and writes its executed-case manifest to `$A2UI_CONFORMANCE_MANIFEST_DIR`. + +### A2A extension (`conformance/extensions/a2a/a2a_integration.yaml`) + +1. `create_a2ui_part`: Call `create_a2ui_part(args.data, version=args.version)`, verify `is_a2ui_part(part)` is `true`, extract the data part with `get_a2ui_datapart(part)`, and compare its `metadata.mimeType` with `expect.mimeType`. +2. `is_a2ui_part`: Build a `DataPart` with `metadata: {"mimeType": args.mimeType}`, call `is_a2ui_part`, and compare the boolean result with `expect`. +3. `try_activate_extension`: Build a request context with `requested_extensions = args.uris` and an agent card advertising every URI `get_a2ui_extension_uri` produces for `v0.8`, `v0.9`, and `1.0`. Call `try_activate_a2ui_extension(context, card)` and assert `(result is not None) == expect`. +4. `get_extension`: Call `get_a2ui_agent_extension(args.version)` with optional `accepts_inline_catalogs` (`args.acceptsInlineCatalogs`) and `supported_catalog_ids` (`args.supportedCatalogIds`). Compare `uri` with `expect.uri` and `params` with `expect.params`. +5. `try_activate`: Build a request context requesting `args.requested` and an agent card advertising `args.advertised`. Call `try_activate_a2ui_extension(context, card)` and verify `expect.activated` (`null` when no extension is activated, or the activated URI together with `expect.version`). +6. `select_newest`: Call the version selector with `args.requested` and `args.advertised` and compare the selected URI with `expect.newest`. + +### ADK extension (`conformance/extensions/adk/adk_extensions.yaml`) + +1. `convert_event`: Dispatches to one of two runners based on `args` keys, failing the case if neither key set matches: + 1. Subagent surface ownership (`subagent` and `message` in `args`, with optional `surfaceId` and `state`): Wrap `args.message` with `create_a2ui_part`, call `A2uiSubagentMap.update_from_server_event(part, subagent, session_service, session)`, and either verify `SurfaceIdAlreadyExistsError` (`expectError.category == "IntegrityError"` and `expectError.message`) or verify that the appended session event's `actions.state_delta` equals `expect.stateDelta`. + 2. Event converter (`errorCode`, `errorMessage`, `contentText`, or `hasCatalog` in `args`): Construct an ADK `Event` with `error_code` and `error_message` or text `content`, invoke `A2uiEventConverter` with a provider that returns a Direct JSON `A2uiRequestProcessor` when `hasCatalog` is `true` and `None` otherwise, and check `expectEmpty: true` (empty event list) or `expect` (`type`, `state`, and `message`). +2. `execute_tool`: Construct `SendA2uiToClientToolset`'s tool with a Direct JSON `A2uiRequestProcessor`. Pass `args.a2uiJson` under the tool's payload argument name (or raw `args` when testing missing arguments). Check `expect.success`: when `true`, verify no error key is returned and check `expect.containsValidatedJson`; when `false`, verify the error key is present and check `expect.errorContains`. +3. `data_model`: Copy `args.clientDataModel`, run `A2uiSubagentMap.strip_unowned_surfaces_from_data_model(args.subagent, client_data_model, args.state)`, and compare the stripped dictionary with `expect.clientDataModel`. + +### `UNSUPPORTED` and manifest checking for extensions + +If an extension harness does not implement an action or a nested key path, list `""` or `"."` (or `"#"`) in its `UNSUPPORTED` map with a non-empty reason so matching cases are recorded as `skipped` in the manifest. Any `UNSUPPORTED` or `EXPECTED_FAILURES` entry that matches no case fails the test run. Because `conformance/extensions/` groups suites for separate extension packages, pass `--only 'extensions/a2a/**'` or `--only 'extensions/adk/**'` to `check_executed_cases.py --corpus extensions` so each harness checks its own extension suite. + ## Reference harnesses Inspect these existing implementations for concrete code examples: diff --git a/conformance/extensions/a2a/a2a_integration.yaml b/conformance/extensions/a2a/a2a_integration.yaml index 18710b3378..f518a813cf 100644 --- a/conformance/extensions/a2a/a2a_integration.yaml +++ b/conformance/extensions/a2a/a2a_integration.yaml @@ -45,66 +45,6 @@ uris: ["other"] expect: false -- name: test_handler_replaces_smart_quotes - action: handle_rpc - args: - request: - jsonrpc: "2.0" - method: "message/send" - id: 1 - params: - message: - contextId: "test-context" - parts: [] - metadata: - a2uiClientCapabilities: - supportedCatalogIds: - ["https://a2ui.org/specification/v0_8/standard_catalog_definition.json"] - runnerOutput: "Here is your chart:\n\n[\n {\n “beginRendering”: {\n “surfaceId”: “sales-dashboard”\n }\n }\n]\n\nEnjoy!" - expect: - parts: - - kind: "text" - text: "Here is your chart:" - - kind: "data" - mimeType: "application/a2ui+json" - data: {"beginRendering": {"surfaceId": "sales-dashboard"}} - - kind: "text" - text: "Enjoy!" - -- name: test_handler_passes_regular_text - action: handle_rpc - args: - request: - jsonrpc: "2.0" - method: "message/send" - id: 1 - params: - message: - contextId: "test-context" - parts: [] - runnerOutput: "This is a normal conversational turn." - expect: - parts: - - kind: "text" - text: "This is a normal conversational turn." - -- name: test_handler_gracefully_handles_invalid_json - action: handle_rpc - args: - request: - jsonrpc: "2.0" - method: "message/send" - id: 1 - params: - message: - contextId: "test-context" - parts: [] - runnerOutput: "Here is an invalid chart:\n\n[\n {\n “beginRendering”: { MISSING CLOSING BRACKETS...\n\nOops." - expect: - parts: - - kind: "text" - text: "Here is an invalid chart:\n\n[\n {\n “beginRendering”: { MISSING CLOSING BRACKETS...\n\nOops." - - name: test_get_extension_minimal action: get_extension args: diff --git a/python/a2ui_a2a/tests/conformance/test_a2a_integration.py b/python/a2ui_a2a/tests/conformance/test_a2a_integration.py index b49da7e16d..416977f368 100644 --- a/python/a2ui_a2a/tests/conformance/test_a2a_integration.py +++ b/python/a2ui_a2a/tests/conformance/test_a2a_integration.py @@ -12,17 +12,254 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Conformance harness for `conformance/extensions/a2a/a2a_integration.yaml`.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any +from unittest.mock import MagicMock + +from a2a.server.agent_execution import RequestContext +from a2a.types import DataPart, Part import pytest -from .conformance_helpers import get_conformance_path, load_conformance_yaml +from a2ui.a2a import ( + create_a2ui_part, + get_a2ui_agent_extension, + get_a2ui_datapart, + get_a2ui_extension_uri, + is_a2ui_part, + try_activate_a2ui_extension, +) +from a2ui.a2a.extension import _select_newest_a2ui_extension +from a2ui.core.common import to_protocol_version +from .conformance_helpers import load_conformance_yaml + +SUITE = "extensions/a2a/a2a_integration.yaml" + +# Actions or `.` entries that this harness does not implement, +# mapped to the reason each matching case is recorded as skipped. +UNSUPPORTED: dict[str, str] = {} + +# `#` entries that exercise a known gap, executed strictly as xfail. +EXPECTED_FAILURES: dict[str, str] = {} + +_CASE_METADATA_KEYS: frozenset[str] = frozenset({ + "name", + "description", + "features", +}) + +_KNOWN_KEYS: dict[str, frozenset[str]] = { + "case": _CASE_METADATA_KEYS | frozenset({"action", "args", "expect"}), + "create_a2ui_part.args": frozenset({"data", "version"}), + "create_a2ui_part.expect": frozenset({"mimeType"}), + "is_a2ui_part.args": frozenset({"mimeType"}), + "try_activate_extension.args": frozenset({"uris"}), + "get_extension.args": frozenset({ + "version", + "acceptsInlineCatalogs", + "supportedCatalogIds", + }), + "get_extension.expect": frozenset({"uri", "params"}), + "try_activate.args": frozenset({"requested", "advertised"}), + "try_activate.expect": frozenset({"activated", "version"}), + "select_newest.args": frozenset({"requested", "advertised"}), + "select_newest.expect": frozenset({"newest"}), +} + +_CURRENT_CASE_KEY: str = "" + + +def _assert_known_keys( + obj: Mapping[str, Any], + allowed: frozenset[str], + where: str, + case_key: str | None = None, +) -> None: + target = case_key or _CURRENT_CASE_KEY or str(obj.get("name", "")) + for key in obj: + if key not in allowed: + pytest.fail(f"Unknown {where} '{key}' in {target}") + + +def _has_nested_key_path(obj: Mapping[str, Any], dotted_path: str) -> bool: + cur: Any = obj + for segment in dotted_path.split("."): + if not isinstance(cur, Mapping) or segment not in cur: + return False + cur = cur[segment] + return cur is not None -def get_conformance_cases(filename): + +def _find_unsupported_reason(test_case: Mapping[str, Any]) -> str | None: + case_key = f"{SUITE}#{test_case.get('name', '')}" + if case_key in UNSUPPORTED: + return UNSUPPORTED[case_key] + action = str(test_case.get("action", "")) + if action in UNSUPPORTED: + return UNSUPPORTED[action] + prefix = f"{action}." + for key, reason in UNSUPPORTED.items(): + if key.startswith(prefix) and _has_nested_key_path( + test_case, key[len(prefix) :] + ): + return reason + return None + + +def get_conformance_cases(filename: str) -> list[tuple[str, dict[str, Any]]]: cases = load_conformance_yaml(filename) + assert isinstance(cases, list) and cases, f"No cases loaded from {filename}" return [(case["name"], case) for case in cases] -# --- A2A Integration Conformance --- -cases_a2a_integration = get_conformance_cases("extensions/a2a/a2a_integration.yaml") +def _run_create_a2ui_part(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["case"], "case key") + args = test_case.get("args", {}) + expect = test_case["expect"] + _assert_known_keys(args, _KNOWN_KEYS["create_a2ui_part.args"], "args key") + _assert_known_keys(expect, _KNOWN_KEYS["create_a2ui_part.expect"], "expect key") + + part = create_a2ui_part(args["data"], version=args.get("version")) + assert is_a2ui_part(part) + data_part = get_a2ui_datapart(part) + assert data_part is not None + assert data_part.metadata is not None + assert data_part.metadata.get("mimeType") == expect["mimeType"] + + +def _run_is_a2ui_part(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["case"], "case key") + args = test_case.get("args", {}) + _assert_known_keys(args, _KNOWN_KEYS["is_a2ui_part.args"], "args key") + expect = test_case["expect"] + assert isinstance(expect, bool) + + part = Part(root=DataPart(data={}, metadata={"mimeType": args["mimeType"]})) + assert is_a2ui_part(part) == expect + + +def _run_try_activate_extension(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["case"], "case key") + args = test_case.get("args", {}) + _assert_known_keys(args, _KNOWN_KEYS["try_activate_extension.args"], "args key") + expect = test_case["expect"] + assert isinstance(expect, bool) + + context = MagicMock(spec=RequestContext) + context.requested_extensions = args["uris"] + context.add_activated_extension = MagicMock() + + card = MagicMock() + extensions = [] + for ver in ("0.8", "0.9", "1.0"): + ext = MagicMock() + ext.uri = get_a2ui_extension_uri(ver) + extensions.append(ext) + card.capabilities.extensions = extensions + + result_version = try_activate_a2ui_extension(context, card) + assert (result_version is not None) == expect + + +def _run_get_extension(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["case"], "case key") + args = test_case.get("args", {}) + expect = test_case["expect"] + _assert_known_keys(args, _KNOWN_KEYS["get_extension.args"], "args key") + _assert_known_keys(expect, _KNOWN_KEYS["get_extension.expect"], "expect key") + + kwargs: dict[str, Any] = {} + if args.get("acceptsInlineCatalogs") is not None: + kwargs["accepts_inline_catalogs"] = args["acceptsInlineCatalogs"] + if args.get("supportedCatalogIds") is not None: + kwargs["supported_catalog_ids"] = args["supportedCatalogIds"] + + ext = get_a2ui_agent_extension(args["version"], **kwargs) + assert ext.uri == expect["uri"] + if expect["params"] is None: + assert ext.params is None + else: + assert ext.params == expect["params"] + + +def _run_try_activate(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["case"], "case key") + args = test_case.get("args", {}) + expect = test_case["expect"] + _assert_known_keys(args, _KNOWN_KEYS["try_activate.args"], "args key") + _assert_known_keys(expect, _KNOWN_KEYS["try_activate.expect"], "expect key") + + context = MagicMock(spec=RequestContext) + context.requested_extensions = args["requested"] + context.add_activated_extension = MagicMock() + + card = MagicMock() + extensions = [] + for uri in args["advertised"]: + ext = MagicMock() + ext.uri = uri + extensions.append(ext) + card.capabilities.extensions = extensions + + result_version = try_activate_a2ui_extension(context, card) + if expect["activated"] is None: + assert result_version is None + context.add_activated_extension.assert_not_called() + else: + # The YAML is shared across SDKs and names the version as a string; + # the Python helper returns the matching ProtocolVersion. + assert result_version == to_protocol_version(expect["version"]) + context.add_activated_extension.assert_called_once_with(expect["activated"]) + + +def _run_select_newest(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["case"], "case key") + args = test_case.get("args", {}) + expect = test_case["expect"] + _assert_known_keys(args, _KNOWN_KEYS["select_newest.args"], "args key") + _assert_known_keys(expect, _KNOWN_KEYS["select_newest.expect"], "expect key") + + result = _select_newest_a2ui_extension(args["requested"], args["advertised"]) + assert result == expect["newest"] + + +RUNNERS: dict[str, Callable[[dict[str, Any]], None]] = { + "create_a2ui_part": _run_create_a2ui_part, + "get_extension": _run_get_extension, + "is_a2ui_part": _run_is_a2ui_part, + "select_newest": _run_select_newest, + "try_activate": _run_try_activate, + "try_activate_extension": _run_try_activate_extension, +} + +cases_a2a_integration = get_conformance_cases(SUITE) + + +def test_unsupported_and_expected_failures_are_used() -> None: + """Every UNSUPPORTED and EXPECTED_FAILURES entry must match at least one case.""" + matched_unsupported: set[str] = set() + for name, case in cases_a2a_integration: + case_key = f"{SUITE}#{name}" + action = str(case.get("action", "")) + if case_key in UNSUPPORTED: + matched_unsupported.add(case_key) + if action in UNSUPPORTED: + matched_unsupported.add(action) + prefix = f"{action}." + for key in UNSUPPORTED: + if key.startswith(prefix) and _has_nested_key_path( + case, key[len(prefix) :] + ): + matched_unsupported.add(key) + unused_unsupported = sorted(set(UNSUPPORTED) - matched_unsupported) + assert not unused_unsupported, f"Unused UNSUPPORTED entries: {unused_unsupported}" + + loaded_keys = {f"{SUITE}#{name}" for name, _ in cases_a2a_integration} + unused_xfails = sorted(set(EXPECTED_FAILURES) - loaded_keys) + assert not unused_xfails, f"Unused EXPECTED_FAILURES entries: {unused_xfails}" @pytest.mark.parametrize( @@ -30,89 +267,25 @@ def get_conformance_cases(filename): cases_a2a_integration, ids=[c[0] for c in cases_a2a_integration], ) -def test_a2a_integration_conformance(name, test_case): - from a2a.types import DataPart, Part - from a2ui.a2a import ( - create_a2ui_part, - get_a2ui_agent_extension, - get_a2ui_datapart, - is_a2ui_part, - try_activate_a2ui_extension, - ) - from a2ui.a2a.extension import _select_newest_a2ui_extension - from a2ui.core.common import to_protocol_version - from unittest.mock import MagicMock - from a2a.server.agent_execution import RequestContext - - action = test_case["action"] - args = test_case.get("args", {}) +def test_a2a_integration_conformance(name: str, test_case: dict[str, Any]) -> None: + global _CURRENT_CASE_KEY + case_key = f"{SUITE}#{name}" + _CURRENT_CASE_KEY = case_key + + unsupported_reason = _find_unsupported_reason(test_case) + if unsupported_reason is not None: + pytest.skip(unsupported_reason) + + action = test_case.get("action") + if not isinstance(action, str) or action not in RUNNERS: + pytest.fail(f"unknown action {action!r}") - if action == "create_a2ui_part": - data = args["data"] - version = args.get("version") - part = create_a2ui_part(data, version=version) - assert is_a2ui_part(part) - expect = test_case["expect"] - data_part = get_a2ui_datapart(part) - exp_mime = expect.get("mimeType") - assert data_part.metadata.get("mimeType") == exp_mime - - elif action == "is_a2ui_part": - mime_type = args.get("mimeType") - part = Part(root=DataPart(data={}, metadata={"mimeType": mime_type})) - result = is_a2ui_part(part) - assert result == test_case["expect"] - - elif action == "get_extension": - version = args["version"] - accepts_inline_catalogs = args.get("acceptsInlineCatalogs") - supported_catalog_ids = args.get("supportedCatalogIds") - - kwargs = {} - if accepts_inline_catalogs is not None: - kwargs["accepts_inline_catalogs"] = accepts_inline_catalogs - if supported_catalog_ids is not None: - kwargs["supported_catalog_ids"] = supported_catalog_ids - - ext = get_a2ui_agent_extension(version, **kwargs) - expect = test_case["expect"] - assert ext.uri == expect["uri"] - if expect["params"] is None: - assert ext.params is None - else: - assert ext.params == expect["params"] - - elif action == "try_activate": - requested = args["requested"] - advertised = args["advertised"] - - context = MagicMock(spec=RequestContext) - context.requested_extensions = requested - context.add_activated_extension = MagicMock() - - card = MagicMock() - extensions = [] - for uri in advertised: - ext = MagicMock() - ext.uri = uri - extensions.append(ext) - card.capabilities.extensions = extensions - - result_version = try_activate_a2ui_extension(context, card) - expect = test_case["expect"] - - if expect["activated"] is None: - assert result_version is None - context.add_activated_extension.assert_not_called() - else: - # The YAML is shared across SDKs and names the version as a string; - # the Python helper returns the matching ProtocolVersion. - assert result_version == to_protocol_version(expect["version"]) - context.add_activated_extension.assert_called_once_with(expect["activated"]) - - elif action == "select_newest": - requested = args["requested"] - advertised = args["advertised"] - result = _select_newest_a2ui_extension(requested, advertised) - expect = test_case["expect"] - assert result == expect["newest"] + runner = RUNNERS[action] + if case_key in EXPECTED_FAILURES: + try: + runner(test_case) + except (Exception, pytest.fail.Exception): + pytest.xfail(EXPECTED_FAILURES[case_key]) + pytest.fail(f"Expected failure '{case_key}' passed unexpectedly.") + else: + runner(test_case) diff --git a/python/a2ui_a2a/tests/conftest.py b/python/a2ui_a2a/tests/conftest.py new file mode 100644 index 0000000000..e7eff6c562 --- /dev/null +++ b/python/a2ui_a2a/tests/conftest.py @@ -0,0 +1,114 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pytest configuration for the a2ui_a2a tests.""" + +from __future__ import annotations + +from collections.abc import Generator +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +_CONFORMANCE_TEST_NAME = "test_a2a_integration_conformance" +_SUITE_NAME = "extensions/a2a/a2a_integration.yaml" + + +def _extract_case_identity(item: pytest.Item) -> tuple[str, str] | None: + """Returns `(suite, case_name)` for a parameterized conformance item.""" + if getattr(item, "originalname", None) != _CONFORMANCE_TEST_NAME: + return None + callspec = getattr(item, "callspec", None) + params = getattr(callspec, "params", None) + if not isinstance(params, dict): + return None + case_name = params.get("name") + if not isinstance(case_name, str) or not case_name: + return None + return _SUITE_NAME, case_name + + +def _append_manifest_record( + suite: str, case_name: str, status: str, reason: str | None = None +) -> None: + """Appends one executed-case record when `A2UI_CONFORMANCE_MANIFEST_DIR` is set.""" + manifest_dir = os.environ.get("A2UI_CONFORMANCE_MANIFEST_DIR") + if not manifest_dir: + return + os.makedirs(manifest_dir, exist_ok=True) + manifest_path = Path(manifest_dir) / f"python-a2a-{os.getpid()}.jsonl" + record: dict[str, str] = { + "harness": "python-a2a", + "suite": suite, + "case": case_name, + "status": status, + } + if reason: + record["reason"] = reason + with manifest_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record) + "\n") + + +def _extract_skip_reason(report: pytest.TestReport) -> str: + """Extracts a human-readable skip or xfail reason from a pytest TestReport.""" + wasxfail = getattr(report, "wasxfail", None) + if isinstance(wasxfail, str) and wasxfail.strip(): + return wasxfail.strip() + if wasxfail is not None: + return "expected failure" + longrepr = getattr(report, "longrepr", None) + if isinstance(longrepr, tuple) and len(longrepr) >= 3: + text = str(longrepr[2]).removeprefix("Skipped: ").strip() + return text or "skipped" + if longrepr: + return str(longrepr).removeprefix("Skipped: ").strip() or "skipped" + return "skipped" + + +def _report_status_and_reason( + report: pytest.TestReport, +) -> tuple[str, str | None]: + """Maps a pytest TestReport to `(status, reason)` for the manifest.""" + wasxfail = getattr(report, "wasxfail", None) + if wasxfail is not None: + if report.outcome == "skipped": + return "xfailed", _extract_skip_reason(report) + return "failed", None + if report.outcome == "skipped": + return "skipped", _extract_skip_reason(report) + if report.outcome == "passed": + return "passed", None + return "failed", None + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[Any] +) -> Generator[None, Any, None]: + """Records executed conformance test outcomes to the manifest JSONL file.""" + del call + outcome = yield + report: pytest.TestReport = outcome.get_result() + if report.when != "call" and not ( + report.when == "setup" and report.outcome != "passed" + ): + return + identity = _extract_case_identity(item) + if identity is None: + return + status, reason = _report_status_and_reason(report) + _append_manifest_record(identity[0], identity[1], status, reason) diff --git a/python/a2ui_adk/tests/conformance/test_adk_extensions.py b/python/a2ui_adk/tests/conformance/test_adk_extensions.py index 0493eee3ee..8fb5681883 100644 --- a/python/a2ui_adk/tests/conformance/test_adk_extensions.py +++ b/python/a2ui_adk/tests/conformance/test_adk_extensions.py @@ -12,141 +12,385 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Conformance harness for `conformance/extensions/adk/adk_extensions.yaml`.""" + +from __future__ import annotations + import asyncio +from collections.abc import Callable, Mapping +import copy import json +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +from google.adk.events.event import Event +from google.adk.sessions.session import Session +from google.genai import types as genai_types import pytest -from .conformance_helpers import get_conformance_path, load_conformance_yaml +from a2ui.a2a import create_a2ui_part +from a2ui.adk import ( + A2uiEventConverter, + A2uiSubagentMap, + SendA2uiToClientToolset, + SurfaceIdAlreadyExistsError, +) +from a2ui.core.basic_catalog import BasicCatalog +from a2ui.inference_formats.direct_json import DirectJsonFormatFactory +from a2ui.processor import A2uiRequestProcessor +from .conformance_helpers import load_conformance_yaml -def get_conformance_cases(filename): - cases = load_conformance_yaml(filename) - return [(case["name"], case) for case in cases] +SUITE = "extensions/adk/adk_extensions.yaml" +# Actions or `.` entries that this harness does not implement, +# mapped to the reason each matching case is recorded as skipped. +UNSUPPORTED: dict[str, str] = {} -# --- ADK Extensions Conformance --- -cases_adk_extensions = get_conformance_cases("extensions/adk/adk_extensions.yaml") +# `#` entries that exercise a known gap, executed strictly as xfail. +EXPECTED_FAILURES: dict[str, str] = {} +_CASE_METADATA_KEYS: frozenset[str] = frozenset({ + "name", + "description", + "features", +}) -@pytest.mark.parametrize( - "name, test_case", - cases_adk_extensions, - ids=[c[0] for c in cases_adk_extensions], -) -def test_adk_extensions_conformance(name, test_case): - from unittest.mock import MagicMock, patch +_SUBAGENT_EVENT_ARG_KEYS: frozenset[str] = frozenset({ + "subagent", + "surfaceId", + "state", + "message", +}) + +_CONVERTER_EVENT_ARG_KEYS: frozenset[str] = frozenset({ + "errorCode", + "errorMessage", + "contentText", + "hasCatalog", +}) + +_KNOWN_KEYS: dict[str, frozenset[str]] = { + "execute_tool.case": _CASE_METADATA_KEYS | frozenset({"action", "args", "expect"}), + "execute_tool.args": frozenset({"a2uiJson", "wrongArg"}), + "execute_tool.expect": frozenset({ + "success", + "containsValidatedJson", + "errorContains", + }), + "convert_event.subagent_case": ( + _CASE_METADATA_KEYS | frozenset({"action", "args", "expect", "expectError"}) + ), + "convert_event.subagent_args": _SUBAGENT_EVENT_ARG_KEYS, + "convert_event.subagent_expect": frozenset({"stateDelta"}), + "convert_event.expect_error": frozenset({"category", "message"}), + "convert_event.converter_case": ( + _CASE_METADATA_KEYS | frozenset({"action", "args", "expect", "expectEmpty"}) + ), + "convert_event.converter_args": _CONVERTER_EVENT_ARG_KEYS, + "convert_event.converter_expect": frozenset({"type", "state", "message"}), + "data_model.case": _CASE_METADATA_KEYS | frozenset({"action", "args", "expect"}), + "data_model.args": frozenset({"subagent", "state", "clientDataModel"}), + "data_model.expect": frozenset({"clientDataModel"}), +} + +_CURRENT_CASE_KEY: str = "" + + +def _assert_known_keys( + obj: Mapping[str, Any], + allowed: frozenset[str], + where: str, + case_key: str | None = None, +) -> None: + target = case_key or _CURRENT_CASE_KEY or str(obj.get("name", "")) + for key in obj: + if key not in allowed: + pytest.fail(f"Unknown {where} '{key}' in {target}") + + +def _has_nested_key_path(obj: Mapping[str, Any], dotted_path: str) -> bool: + cur: Any = obj + for segment in dotted_path.split("."): + if not isinstance(cur, Mapping) or segment not in cur: + return False + cur = cur[segment] + return cur is not None - from a2ui.adk import SendA2uiToClientToolset - from a2ui.core.basic_catalog import BasicCatalog - from a2ui.inference_formats.direct_json import DirectJsonFormatFactory - from a2ui.processor import A2uiRequestProcessor - action = test_case["action"] +def _find_unsupported_reason(test_case: Mapping[str, Any]) -> str | None: + case_key = f"{SUITE}#{test_case.get('name', '')}" + if case_key in UNSUPPORTED: + return UNSUPPORTED[case_key] + action = str(test_case.get("action", "")) + if action in UNSUPPORTED: + return UNSUPPORTED[action] + prefix = f"{action}." + for key, reason in UNSUPPORTED.items(): + if key.startswith(prefix) and _has_nested_key_path( + test_case, key[len(prefix) :] + ): + return reason + return None + + +def get_conformance_cases(filename: str) -> list[tuple[str, dict[str, Any]]]: + cases = load_conformance_yaml(filename) + assert isinstance(cases, list) and cases, f"No cases loaded from {filename}" + return [(case["name"], case) for case in cases] + + +def _run_execute_tool(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["execute_tool.case"], "case key") args = test_case.get("args", {}) + expect = test_case["expect"] + _assert_known_keys(args, _KNOWN_KEYS["execute_tool.args"], "args key") + _assert_known_keys(expect, _KNOWN_KEYS["execute_tool.expect"], "expect key") + + tool_cls = SendA2uiToClientToolset._SendA2uiJsonToClientTool + a2ui_json_str = args.get("a2uiJson") + tool_args = ( + {tool_cls.A2UI_PAYLOAD_ARG_NAME: a2ui_json_str} + if a2ui_json_str is not None + else args + ) - if action == "execute_tool": - tool_cls = SendA2uiToClientToolset._SendA2uiJsonToClientTool - a2ui_json_str = args.get("a2uiJson") - tool_args = ( - {tool_cls.A2UI_PAYLOAD_ARG_NAME: a2ui_json_str} if a2ui_json_str else args + # The shared cases write the payload in Direct JSON. + tool = tool_cls( + A2uiRequestProcessor( + [BasicCatalog("0.8")], format_factory=DirectJsonFormatFactory() ) + ) - # The shared cases write the payload in Direct JSON. - tool = tool_cls( - A2uiRequestProcessor( - [BasicCatalog("0.8")], format_factory=DirectJsonFormatFactory() - ) + tool_context_mock = MagicMock() + tool_context_mock.state = {} + tool_context_mock.actions = MagicMock(skip_summarization=False) + + with patch("a2ui.adk.send_a2ui_to_client_toolset.validate_payload"): + result = asyncio.run( + tool.run_async(args=tool_args, tool_context=tool_context_mock) ) - tool_context_mock = MagicMock() - tool_context_mock.state = {} - tool_context_mock.actions = MagicMock(skip_summarization=False) + expect_success = expect["success"] + assert isinstance(expect_success, bool) - with patch("a2ui.adk.send_a2ui_to_client_toolset.validate_payload"): - # run_async is async in Python - result = asyncio.run( - tool.run_async(args=tool_args, tool_context=tool_context_mock) - ) + if expect_success: + assert "error" not in result + assert tool_cls.VALIDATED_A2UI_JSON_KEY in result + if expect.get("containsValidatedJson"): + validated_payload = result[tool_cls.VALIDATED_A2UI_JSON_KEY] + assert "beginRendering" in json.dumps(validated_payload) + else: + assert "error" in result + if "errorContains" in expect: + assert expect["errorContains"] in result["error"] + + +def _run_subagent_event(test_case: dict[str, Any]) -> None: + _assert_known_keys( + test_case, _KNOWN_KEYS["convert_event.subagent_case"], "case key" + ) + args = test_case.get("args", {}) + _assert_known_keys(args, _KNOWN_KEYS["convert_event.subagent_args"], "args key") + + subagent = args["subagent"] + state = args.get("state", {}) + message = args["message"] + a2a_part = create_a2ui_part(message) - expect = test_case["expect"] - expect_success = expect["success"] + session_service = AsyncMock() + session = MagicMock(spec=Session) + session.state = dict(state) - if expect_success: - assert "error" not in result - assert ( - SendA2uiToClientToolset._SendA2uiJsonToClientTool.VALIDATED_A2UI_JSON_KEY - in result + expect_error = test_case.get("expectError") + if expect_error is not None: + if isinstance(expect_error, Mapping): + _assert_known_keys( + expect_error, + _KNOWN_KEYS["convert_event.expect_error"], + "expectError key", ) - if expect.get("containsValidatedJson"): - validated_payload = result[ - SendA2uiToClientToolset._SendA2uiJsonToClientTool.VALIDATED_A2UI_JSON_KEY - ] - assert "beginRendering" in json.dumps(validated_payload) + category = expect_error.get("category") + if category is not None: + assert category == "IntegrityError" + expected_msg = expect_error.get("message", "") else: - assert "error" in result - err_contains = expect.get("errorContains") - if err_contains: - assert err_contains in result["error"] - - elif action == "convert_event": - # Handle Subagent Map or Event Converter - if "subagent" in args and "message" in args: - from a2ui.adk import A2uiSubagentMap, SurfaceIdAlreadyExistsError - from a2ui.a2a import create_a2ui_part - from unittest.mock import AsyncMock, MagicMock - from google.adk.sessions.session import Session - - subagent = args["subagent"] - state = args.get("state", {}) - message = args["message"] - - a2a_part = create_a2ui_part(message) - - session_service = AsyncMock() - session = MagicMock(spec=Session) - session.state = dict(state) - - expect_error = test_case.get("expectError") - if expect_error: - with pytest.raises(SurfaceIdAlreadyExistsError) as exc_info: - asyncio.run( - A2uiSubagentMap.update_from_server_event( - a2a_part, subagent, session_service, session - ) - ) - msg = ( - expect_error.get("message", "") - if isinstance(expect_error, dict) - else expect_error - ) - if msg: - assert msg in str(exc_info.value) - else: - asyncio.run( - A2uiSubagentMap.update_from_server_event( - a2a_part, subagent, session_service, session - ) + expected_msg = str(expect_error) + + with pytest.raises(SurfaceIdAlreadyExistsError) as exc_info: + asyncio.run( + A2uiSubagentMap.update_from_server_event( + a2a_part, subagent, session_service, session ) - expect = test_case["expect"] - if "stateDelta" in expect: - session_service.append_event.assert_called_once() - call_args = session_service.append_event.call_args[0] - event = call_args[1] - assert event.actions.state_delta == expect["stateDelta"] - - elif action == "data_model": - from a2ui.adk import A2uiSubagentMap - import copy - - subagent = args.get("subagent") - state = args.get("state", {}) - client_data_model = copy.deepcopy(args.get("clientDataModel", {})) - - asyncio.run( - A2uiSubagentMap.strip_unowned_surfaces_from_data_model( - subagent, client_data_model, state ) + if expected_msg: + assert expected_msg in str(exc_info.value) + return + + asyncio.run( + A2uiSubagentMap.update_from_server_event( + a2a_part, subagent, session_service, session ) + ) + expect = test_case["expect"] + _assert_known_keys( + expect, _KNOWN_KEYS["convert_event.subagent_expect"], "expect key" + ) + assert "stateDelta" in expect + session_service.append_event.assert_called_once() + call_args = session_service.append_event.call_args[0] + event = call_args[1] + assert event.actions.state_delta == expect["stateDelta"] + + +def _run_converter_event(test_case: dict[str, Any]) -> None: + _assert_known_keys( + test_case, _KNOWN_KEYS["convert_event.converter_case"], "case key" + ) + args = test_case.get("args", {}) + _assert_known_keys(args, _KNOWN_KEYS["convert_event.converter_args"], "args key") + + content = ( + genai_types.Content( + role="model", + parts=[genai_types.Part(text=args["contentText"])], + ) + if "contentText" in args + else None + ) + event = Event( + author="agent", + error_code=args.get("errorCode"), + error_message=args.get("errorMessage"), + content=content, + ) + + has_catalog = bool(args.get("hasCatalog", False)) + processor = ( + A2uiRequestProcessor( + [BasicCatalog("0.8")], format_factory=DirectJsonFormatFactory() + ) + if has_catalog + else None + ) + converter = A2uiEventConverter(a2ui_processor=lambda _: processor) + + invocation_context = MagicMock() + invocation_context.invocation_id = "inv-1" + invocation_context.session.id = "session-1" + invocation_context.session.state = {} + + events = converter( + event, invocation_context, task_id="task-1", context_id="context-1" + ) + + if test_case.get("expectEmpty"): + assert events == [] + return + + expect = test_case["expect"] + _assert_known_keys( + expect, _KNOWN_KEYS["convert_event.converter_expect"], "expect key" + ) + assert len(events) == 1 + actual: Any = events[0] + assert type(actual).__name__ == expect["type"] + assert actual.status.state.value.upper() == expect["state"] + assert actual.status.message is not None + assert actual.status.message.parts[0].root.text == expect["message"] + + +def _run_convert_event(test_case: dict[str, Any]) -> None: + args = test_case.get("args", {}) + is_subagent = "subagent" in args and "message" in args + is_converter = any(k in args for k in _CONVERTER_EVENT_ARG_KEYS) + if is_subagent and not is_converter: + _run_subagent_event(test_case) + elif is_converter and not is_subagent: + _run_converter_event(test_case) + else: + pytest.fail( + f"convert_event args {sorted(args)} do not match subagent or converter" + f" key set in {_CURRENT_CASE_KEY}" + ) + + +def _run_data_model(test_case: dict[str, Any]) -> None: + _assert_known_keys(test_case, _KNOWN_KEYS["data_model.case"], "case key") + args = test_case.get("args", {}) + expect = test_case["expect"] + _assert_known_keys(args, _KNOWN_KEYS["data_model.args"], "args key") + _assert_known_keys(expect, _KNOWN_KEYS["data_model.expect"], "expect key") + assert "clientDataModel" in expect + + subagent = args["subagent"] + state = args.get("state", {}) + client_data_model = copy.deepcopy(args.get("clientDataModel", {})) + + asyncio.run( + A2uiSubagentMap.strip_unowned_surfaces_from_data_model( + subagent, client_data_model, state + ) + ) + assert client_data_model == expect["clientDataModel"] + + +RUNNERS: dict[str, Callable[[dict[str, Any]], None]] = { + "convert_event": _run_convert_event, + "data_model": _run_data_model, + "execute_tool": _run_execute_tool, +} + +cases_adk_extensions = get_conformance_cases(SUITE) + + +def test_unsupported_and_expected_failures_are_used() -> None: + """Every UNSUPPORTED and EXPECTED_FAILURES entry must match at least one case.""" + matched_unsupported: set[str] = set() + for name, case in cases_adk_extensions: + case_key = f"{SUITE}#{name}" + action = str(case.get("action", "")) + if case_key in UNSUPPORTED: + matched_unsupported.add(case_key) + if action in UNSUPPORTED: + matched_unsupported.add(action) + prefix = f"{action}." + for key in UNSUPPORTED: + if key.startswith(prefix) and _has_nested_key_path( + case, key[len(prefix) :] + ): + matched_unsupported.add(key) + unused_unsupported = sorted(set(UNSUPPORTED) - matched_unsupported) + assert not unused_unsupported, f"Unused UNSUPPORTED entries: {unused_unsupported}" + + loaded_keys = {f"{SUITE}#{name}" for name, _ in cases_adk_extensions} + unused_xfails = sorted(set(EXPECTED_FAILURES) - loaded_keys) + assert not unused_xfails, f"Unused EXPECTED_FAILURES entries: {unused_xfails}" + + +@pytest.mark.parametrize( + "name, test_case", + cases_adk_extensions, + ids=[c[0] for c in cases_adk_extensions], +) +def test_adk_extensions_conformance(name: str, test_case: dict[str, Any]) -> None: + global _CURRENT_CASE_KEY + case_key = f"{SUITE}#{name}" + _CURRENT_CASE_KEY = case_key + + unsupported_reason = _find_unsupported_reason(test_case) + if unsupported_reason is not None: + pytest.skip(unsupported_reason) + + action = test_case.get("action") + if not isinstance(action, str) or action not in RUNNERS: + pytest.fail(f"unknown action {action!r}") - expected = test_case["expect"] - if "clientDataModel" in expected: - assert client_data_model == expected["clientDataModel"] + runner = RUNNERS[action] + if case_key in EXPECTED_FAILURES: + try: + runner(test_case) + except (Exception, pytest.fail.Exception): + pytest.xfail(EXPECTED_FAILURES[case_key]) + pytest.fail(f"Expected failure '{case_key}' passed unexpectedly.") + else: + runner(test_case) diff --git a/python/a2ui_adk/tests/conftest.py b/python/a2ui_adk/tests/conftest.py new file mode 100644 index 0000000000..37f214e130 --- /dev/null +++ b/python/a2ui_adk/tests/conftest.py @@ -0,0 +1,114 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pytest configuration for the a2ui_adk tests.""" + +from __future__ import annotations + +from collections.abc import Generator +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +_CONFORMANCE_TEST_NAME = "test_adk_extensions_conformance" +_SUITE_NAME = "extensions/adk/adk_extensions.yaml" + + +def _extract_case_identity(item: pytest.Item) -> tuple[str, str] | None: + """Returns `(suite, case_name)` for a parameterized conformance item.""" + if getattr(item, "originalname", None) != _CONFORMANCE_TEST_NAME: + return None + callspec = getattr(item, "callspec", None) + params = getattr(callspec, "params", None) + if not isinstance(params, dict): + return None + case_name = params.get("name") + if not isinstance(case_name, str) or not case_name: + return None + return _SUITE_NAME, case_name + + +def _append_manifest_record( + suite: str, case_name: str, status: str, reason: str | None = None +) -> None: + """Appends one executed-case record when `A2UI_CONFORMANCE_MANIFEST_DIR` is set.""" + manifest_dir = os.environ.get("A2UI_CONFORMANCE_MANIFEST_DIR") + if not manifest_dir: + return + os.makedirs(manifest_dir, exist_ok=True) + manifest_path = Path(manifest_dir) / f"python-adk-{os.getpid()}.jsonl" + record: dict[str, str] = { + "harness": "python-adk", + "suite": suite, + "case": case_name, + "status": status, + } + if reason: + record["reason"] = reason + with manifest_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record) + "\n") + + +def _extract_skip_reason(report: pytest.TestReport) -> str: + """Extracts a human-readable skip or xfail reason from a pytest TestReport.""" + wasxfail = getattr(report, "wasxfail", None) + if isinstance(wasxfail, str) and wasxfail.strip(): + return wasxfail.strip() + if wasxfail is not None: + return "expected failure" + longrepr = getattr(report, "longrepr", None) + if isinstance(longrepr, tuple) and len(longrepr) >= 3: + text = str(longrepr[2]).removeprefix("Skipped: ").strip() + return text or "skipped" + if longrepr: + return str(longrepr).removeprefix("Skipped: ").strip() or "skipped" + return "skipped" + + +def _report_status_and_reason( + report: pytest.TestReport, +) -> tuple[str, str | None]: + """Maps a pytest TestReport to `(status, reason)` for the manifest.""" + wasxfail = getattr(report, "wasxfail", None) + if wasxfail is not None: + if report.outcome == "skipped": + return "xfailed", _extract_skip_reason(report) + return "failed", None + if report.outcome == "skipped": + return "skipped", _extract_skip_reason(report) + if report.outcome == "passed": + return "passed", None + return "failed", None + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[Any] +) -> Generator[None, Any, None]: + """Records executed conformance test outcomes to the manifest JSONL file.""" + del call + outcome = yield + report: pytest.TestReport = outcome.get_result() + if report.when != "call" and not ( + report.when == "setup" and report.outcome != "passed" + ): + return + identity = _extract_case_identity(item) + if identity is None: + return + status, reason = _report_status_and_reason(report) + _append_manifest_record(identity[0], identity[1], status, reason) From 113bc1cf907ffc6bdabad403478de082416dca49 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:22:03 +0000 Subject: [PATCH 20/48] test(a2ui_agent-ts): executed-case manifest for the TypeScript agent harnesses --- .github/workflows/web_ci.yml | 2 + .../conformance/catalog-provider.test.ts | 81 ++-- .../conformance/catalog-resolution.test.ts | 65 +-- .../conformance/catalog-transformer.test.ts | 117 ++--- .../tests/conformance/conformance.test.ts | 49 +-- .../direct-json-conformance.test.ts | 244 +++++------ .../direct-json-prompt-generator.test.ts | 89 ++-- .../direct-json-response-streaming.test.ts | 34 +- .../conformance/express-conformance.test.ts | 405 +++++++++--------- .../express-response-streaming.test.ts | 28 +- .../a2ui_agent/tests/conformance/loader.ts | 45 +- .../a2ui_agent/tests/conformance/manifest.ts | 102 +++++ .../multi-catalog-conformance.test.ts | 52 ++- .../conformance/skill-conformance.test.ts | 88 ++-- .../tests/conformance/streamed-answer.test.ts | 87 ++-- .../tests/conformance/suite-helpers.ts | 59 ++- .../conformance/unsupported-suites.test.ts | 47 ++ 17 files changed, 883 insertions(+), 711 deletions(-) create mode 100644 typescript/agents/a2ui_agent/tests/conformance/manifest.ts create mode 100644 typescript/agents/a2ui_agent/tests/conformance/unsupported-suites.test.ts diff --git a/.github/workflows/web_ci.yml b/.github/workflows/web_ci.yml index e71af26873..8aed18d1eb 100644 --- a/.github/workflows/web_ci.yml +++ b/.github/workflows/web_ci.yml @@ -122,6 +122,8 @@ jobs: - uses: ./.github/actions/setup-python - name: Check web_core executed cases run: uv run --project conformance python conformance/scripts/check_executed_cases.py --manifest-dir "${{ runner.temp }}/a2ui-manifest" --harness web_core --corpus core + - name: Check TS agent executed cases + run: uv run --project conformance python conformance/scripts/check_executed_cases.py --manifest-dir "${{ runner.temp }}/a2ui-manifest" --corpus agent --harness ts-agent - name: Test React 18 Compatibility run: yarn workspace @a2ui/react run test:react18 - name: Verify NPM Publish Script diff --git a/typescript/agents/a2ui_agent/tests/conformance/catalog-provider.test.ts b/typescript/agents/a2ui_agent/tests/conformance/catalog-provider.test.ts index 1e565cabbe..bacb09ba8d 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/catalog-provider.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/catalog-provider.test.ts @@ -14,57 +14,58 @@ * limitations under the License. */ -import {describe, expect} from 'vitest'; +import {expect} from 'vitest'; import {FileSystemCatalogProvider, ProtocolVersion} from '../../src/index.js'; import {loadCases} from './loader.js'; import { conformancePath, + describeSuite, errorClassFor, expectCatalogContents, - registerCase, } from './suite-helpers.js'; /** Cases that run but do not pass yet, with the reason for each. */ const KNOWN_FAILURES = new Map([]); -describe('Conformance: catalog_provider.yaml', () => { - for (const testCase of loadCases([conformancePath('agent/catalog_provider.yaml')])) { - registerCase(testCase, KNOWN_FAILURES, async () => { - const args = testCase.args as { - provider: string; - path: string; - catalog_id?: string; - protocol_version?: ProtocolVersion; - }; - if (args.provider !== 'file_system') { - throw new Error(`Unknown provider '${args.provider}'`); - } - const provider = new FileSystemCatalogProvider( - conformancePath(args.path), - args.protocol_version, - args.catalog_id, - ); +describeSuite( + 'Conformance: catalog_provider.yaml', + loadCases([conformancePath('agent/catalog_provider.yaml')]), + async testCase => { + const args = testCase.args as { + provider: string; + path: string; + catalog_id?: string; + protocol_version?: ProtocolVersion; + }; + if (args.provider !== 'file_system') { + throw new Error(`Unknown provider '${args.provider}'`); + } + const provider = new FileSystemCatalogProvider( + conformancePath(args.path), + args.protocol_version, + args.catalog_id, + ); - if (testCase.expect_error) { - expect(() => provider.load()).toThrow(errorClassFor(testCase)); - return; - } + if (testCase.expect_error) { + expect(() => provider.load()).toThrow(errorClassFor(testCase)); + return; + } - const expected = testCase.expect as { - catalog_id?: string; - protocol_version?: string; - components?: string[]; - functions?: string[]; - }; - const catalog = provider.load(); - if (expected.catalog_id !== undefined) { - expect(catalog.id).toBe(expected.catalog_id); - } - if (expected.protocol_version !== undefined) { - expect(catalog.protocolVersion).toBe(expected.protocol_version); - } - expectCatalogContents(catalog, expected); - }); - } -}); + const expected = testCase.expect as { + catalog_id?: string; + protocol_version?: string; + components?: string[]; + functions?: string[]; + }; + const catalog = provider.load(); + if (expected.catalog_id !== undefined) { + expect(catalog.id).toBe(expected.catalog_id); + } + if (expected.protocol_version !== undefined) { + expect(catalog.protocolVersion).toBe(expected.protocol_version); + } + expectCatalogContents(catalog, expected); + }, + {knownFailures: KNOWN_FAILURES}, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/catalog-resolution.test.ts b/typescript/agents/a2ui_agent/tests/conformance/catalog-resolution.test.ts index 04204a556b..a68a3d1d8b 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/catalog-resolution.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/catalog-resolution.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {describe, expect} from 'vitest'; +import {expect} from 'vitest'; import {resolveCatalogs} from '../../src/index.js'; import {RendererCapabilities} from '../../src/internal/web-core.js'; @@ -22,45 +22,46 @@ import {loadCases} from './loader.js'; import { CatalogRegistration, conformancePath, + describeSuite, errorClassFor, expectCatalogContents, loadRegistrations, - registerCase, } from './suite-helpers.js'; /** Cases that run but do not pass yet, with the reason for each. */ const KNOWN_FAILURES = new Map(); -describe('Conformance: catalog_resolution.yaml', () => { - for (const testCase of loadCases([conformancePath('agent/catalog_resolution.yaml')])) { - registerCase(testCase, KNOWN_FAILURES, async () => { - const args = testCase.args as { - catalogs: CatalogRegistration[]; - renderer_capabilities?: Record; - accepts_inline_catalogs?: boolean; - }; - const configs = loadRegistrations(args.catalogs); +describeSuite( + 'Conformance: catalog_resolution.yaml', + loadCases([conformancePath('agent/catalog_resolution.yaml')]), + async testCase => { + const args = testCase.args as { + catalogs: CatalogRegistration[]; + renderer_capabilities?: Record; + accepts_inline_catalogs?: boolean; + }; + const configs = loadRegistrations(args.catalogs); - const resolve = () => - resolveCatalogs(configs, args.renderer_capabilities, args.accepts_inline_catalogs ?? false); + const resolve = () => + resolveCatalogs(configs, args.renderer_capabilities, args.accepts_inline_catalogs ?? false); - if (testCase.expect_error) { - expect(resolve).toThrow(errorClassFor(testCase)); - return; - } + if (testCase.expect_error) { + expect(resolve).toThrow(errorClassFor(testCase)); + return; + } - const expected = testCase.expect as { - active_catalog_ids: string[]; - catalogs?: Array<{catalog_id: string; components?: string[]; functions?: string[]}>; - }; - const active = resolve(); - // Order carries no meaning in the protocol, so the suite compares ids as a set. - expect(active.map(c => c.id).sort()).toEqual([...expected.active_catalog_ids].sort()); - for (const expectedCatalog of expected.catalogs ?? []) { - const catalog = active.find(c => c.id === expectedCatalog.catalog_id); - expect(catalog, `active catalog ${expectedCatalog.catalog_id}`).toBeDefined(); - expectCatalogContents(catalog!, expectedCatalog); - } - }); - } -}); + const expected = testCase.expect as { + active_catalog_ids: string[]; + catalogs?: Array<{catalog_id: string; components?: string[]; functions?: string[]}>; + }; + const active = resolve(); + // Order carries no meaning in the protocol, so the suite compares ids as a set. + expect(active.map(c => c.id).sort()).toEqual([...expected.active_catalog_ids].sort()); + for (const expectedCatalog of expected.catalogs ?? []) { + const catalog = active.find(c => c.id === expectedCatalog.catalog_id); + expect(catalog, `active catalog ${expectedCatalog.catalog_id}`).toBeDefined(); + expectCatalogContents(catalog!, expectedCatalog); + } + }, + {knownFailures: KNOWN_FAILURES}, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/catalog-transformer.test.ts b/typescript/agents/a2ui_agent/tests/conformance/catalog-transformer.test.ts index 8bf21331d8..45fa338085 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/catalog-transformer.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/catalog-transformer.test.ts @@ -14,9 +14,9 @@ * limitations under the License. */ -import * as fs from 'fs'; +import {readFileSync} from 'fs'; -import {describe, expect} from 'vitest'; +import {expect} from 'vitest'; import { ComponentPruningTransformer, @@ -24,69 +24,70 @@ import { FunctionPruningTransformer, } from '../../src/index.js'; import {loadCases} from './loader.js'; -import {conformancePath, expectCatalogContents, registerCase} from './suite-helpers.js'; +import {conformancePath, describeSuite, expectCatalogContents} from './suite-helpers.js'; /** Cases that run but do not pass yet, with the reason for each. */ const KNOWN_FAILURES = new Map([]); -describe('Conformance: catalog_transformer.yaml', () => { - for (const testCase of loadCases([conformancePath('agent/catalog_transformer.yaml')])) { - registerCase(testCase, KNOWN_FAILURES, async () => { - const args = (testCase.args ?? {}) as { - catalog: string; - transformers?: Array<{ - component_pruning?: string[] | null; - function_pruning?: string[] | null; - }>; - }; +describeSuite( + 'Conformance: catalog_transformer.yaml', + loadCases([conformancePath('agent/catalog_transformer.yaml')]), + async testCase => { + const args = (testCase.args ?? {}) as { + catalog: string; + transformers?: Array<{ + component_pruning?: string[] | null; + function_pruning?: string[] | null; + }>; + }; - const filePath = conformancePath(args.catalog); - const doc = JSON.parse(fs.readFileSync(filePath, 'utf8')) as Record; - let transformed = new FileSystemCatalogProvider( - filePath, - typeof doc.protocolVersion === 'string' ? undefined : '1.0', - typeof doc.catalogId === 'string' ? undefined : args.catalog, - ).load(); - for (const transformer of args.transformers ?? []) { - if (transformer?.component_pruning !== undefined) { - transformed = new ComponentPruningTransformer(transformer.component_pruning).transform( - transformed, - ); - } - if (transformer?.function_pruning !== undefined) { - transformed = new FunctionPruningTransformer(transformer.function_pruning).transform( - transformed, - ); - } + const filePath = conformancePath(args.catalog); + const doc = JSON.parse(readFileSync(filePath, 'utf8')) as Record; + let transformed = new FileSystemCatalogProvider( + filePath, + typeof doc.protocolVersion === 'string' ? undefined : '1.0', + typeof doc.catalogId === 'string' ? undefined : args.catalog, + ).load(); + for (const transformer of args.transformers ?? []) { + if (transformer?.component_pruning !== undefined) { + transformed = new ComponentPruningTransformer(transformer.component_pruning).transform( + transformed, + ); } + if (transformer?.function_pruning !== undefined) { + transformed = new FunctionPruningTransformer(transformer.function_pruning).transform( + transformed, + ); + } + } - const expected = testCase.expect as { - catalog_id?: string; - protocol_version?: string; - components?: string[]; - functions?: string[]; - serialized_defs_contain?: string[]; - serialized_defs_omit?: string[]; - }; + const expected = testCase.expect as { + catalog_id?: string; + protocol_version?: string; + components?: string[]; + functions?: string[]; + serialized_defs_contain?: string[]; + serialized_defs_omit?: string[]; + }; - if (expected.catalog_id !== undefined) { - expect(transformed.id).toBe(expected.catalog_id); - } - if (expected.protocol_version !== undefined) { - expect(transformed.protocolVersion).toBe(expected.protocol_version); - } - expectCatalogContents(transformed, expected); + if (expected.catalog_id !== undefined) { + expect(transformed.id).toBe(expected.catalog_id); + } + if (expected.protocol_version !== undefined) { + expect(transformed.protocolVersion).toBe(expected.protocol_version); + } + expectCatalogContents(transformed, expected); - if (expected.serialized_defs_contain || expected.serialized_defs_omit) { - const serialized = transformed.toJson() as {$defs?: Record}; - const defs = serialized.$defs ?? {}; - for (const defName of expected.serialized_defs_contain ?? []) { - expect(defs).toHaveProperty(defName); - } - for (const defName of expected.serialized_defs_omit ?? []) { - expect(defs).not.toHaveProperty(defName); - } + if (expected.serialized_defs_contain || expected.serialized_defs_omit) { + const serialized = transformed.toJson() as {$defs?: Record}; + const defs = serialized.$defs ?? {}; + for (const defName of expected.serialized_defs_contain ?? []) { + expect(defs).toHaveProperty(defName); + } + for (const defName of expected.serialized_defs_omit ?? []) { + expect(defs).not.toHaveProperty(defName); } - }); - } -}); + } + }, + {knownFailures: KNOWN_FAILURES}, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/conformance.test.ts b/typescript/agents/a2ui_agent/tests/conformance/conformance.test.ts index 29d77125ef..fdfca24f2d 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/conformance.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/conformance.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {describe, expect, test} from 'vitest'; +import {expect} from 'vitest'; import { A2uiCatalogError, @@ -33,7 +33,7 @@ import {toWireProtocolVersion} from '../../src/utils/protocol-version.js'; import {loadBasicCatalog} from '../helpers/basic-catalogs.js'; import {createCatalogConfig, createFileCatalogConfig} from './fixtures.js'; import {classify, KNOWN_FAILURES, loadCases, LoadedCase} from './loader.js'; -import {conformancePath} from './suite-helpers.js'; +import {conformancePath, describeSuite} from './suite-helpers.js'; // Cases that name no catalog run against the v1.0 basic catalog. const basicCatalogV10 = loadBasicCatalog('1.0'); @@ -256,33 +256,24 @@ const LEGACY_ACTION_HANDLERS: Record void | Pr select_catalog: runSelectCatalogCase, }; -describe('Conformance Harness (Legacy Agent Suites)', () => { - const yamlFiles = [ +describeSuite( + 'Conformance Harness (Legacy Agent Suites)', + loadCases([ conformancePath('agent/legacy/parser.yaml'), conformancePath('agent/legacy/inference_format.yaml'), - ]; - - const allCases = loadCases(yamlFiles); - - for (const testCase of allCases) { - const verdict = classify(testCase); - const {action, name} = testCase; - const testName = `${action} · ${name}`; - - const testFn = async () => { - const handler = LEGACY_ACTION_HANDLERS[action]; - if (!handler) { - throw new Error(`Unhandled conformance action: ${action}`); - } - await handler(testCase); - }; - - if (!verdict.runnable) { - test.skip(`${testName} (${verdict.reason})`, testFn); - } else if (KNOWN_FAILURES.has(name)) { - test.fails(`${testName} (known gap: ${KNOWN_FAILURES.get(name)})`, testFn); - } else { - test(testName, testFn); + ]), + async testCase => { + const handler = LEGACY_ACTION_HANDLERS[testCase.action]; + if (!handler) { + throw new Error(`Unhandled conformance action: ${testCase.action}`); } - } -}); + await handler(testCase); + }, + { + knownFailures: KNOWN_FAILURES, + skipReason: testCase => { + const verdict = classify(testCase); + return verdict.runnable ? undefined : verdict.reason; + }, + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/direct-json-conformance.test.ts b/typescript/agents/a2ui_agent/tests/conformance/direct-json-conformance.test.ts index 2534bb1c77..f7d2827bb5 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/direct-json-conformance.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/direct-json-conformance.test.ts @@ -14,12 +14,10 @@ * limitations under the License. */ -import {describe, test, expect} from 'vitest'; -import * as path from 'path'; -import {loadCases, CONFORMANCE_ROOT} from './loader.js'; -import {createFileCatalogConfig} from './fixtures.js'; -import {compileChecked} from './suite-helpers.js'; -import {AgentToRendererMessage} from '../../src/internal/web-core.js'; +import {resolve} from 'path'; + +import {expect} from 'vitest'; + import { A2uiCompilationParseError, A2uiCompilationValidationError, @@ -29,6 +27,10 @@ import { RawResponsePart, ResponsePart, } from '../../src/index.js'; +import {AgentToRendererMessage} from '../../src/internal/web-core.js'; +import {createFileCatalogConfig} from './fixtures.js'; +import {CONFORMANCE_ROOT, loadCases} from './loader.js'; +import {compileChecked, describeSuite} from './suite-helpers.js'; /** The error classes that count as each category the suites name. */ const ERRORS_BY_CATEGORY: Record Error>> = { @@ -123,143 +125,127 @@ function assertPartsMatch( } } -describe('Direct JSON Conformance Suite', () => { - const files = [ - path.resolve(CONFORMANCE_ROOT, 'agent/direct_json/compiler.yaml'), - path.resolve(CONFORMANCE_ROOT, 'agent/direct_json/decompiler.yaml'), - path.resolve(CONFORMANCE_ROOT, 'agent/direct_json/response_parser.yaml'), - ]; - - const cases = loadCases(files); - const UNSUPPORTED = new Set([]); - const KNOWN_FAILURES = new Map([]); - - for (const testCase of cases) { - const {name, action} = testCase; - const isUnsupported = UNSUPPORTED.has(name); - - const runCase = async () => { - const args = (testCase.args as Record) || {}; - const catalogPaths: string[] = []; - if (Array.isArray(args.catalogs)) { - catalogPaths.push(...(args.catalogs as string[])); - } else if (typeof args.catalog === 'string') { - catalogPaths.push(args.catalog); - } else { - catalogPaths.push('test_data/catalogs/simplified_catalog_v1_0.json'); +const UNSUPPORTED = new Map([]); +const KNOWN_FAILURES = new Map([]); + +describeSuite( + 'Direct JSON Conformance Suite', + loadCases([ + resolve(CONFORMANCE_ROOT, 'agent/direct_json/compiler.yaml'), + resolve(CONFORMANCE_ROOT, 'agent/direct_json/decompiler.yaml'), + resolve(CONFORMANCE_ROOT, 'agent/direct_json/response_parser.yaml'), + ]), + async testCase => { + const {action} = testCase; + const args = (testCase.args as Record) || {}; + const catalogPaths: string[] = []; + if (Array.isArray(args.catalogs)) { + catalogPaths.push(...(args.catalogs as string[])); + } else if (typeof args.catalog === 'string') { + catalogPaths.push(args.catalog); + } else { + catalogPaths.push('test_data/catalogs/simplified_catalog_v1_0.json'); + } + const catalogs = catalogPaths.map(p => createFileCatalogConfig(p).catalog); + const parser = new DirectJsonParser(catalogs); + + if (action === 'compile') { + const payload = testCase.input as string; + + if (testCase.expect_error) { + assertThrows( + () => { + compileChecked(parser, payload); + }, + testCase.expect_error as Record, + ); + return; } - const catalogs = catalogPaths.map(p => createFileCatalogConfig(p).catalog); - const parser = new DirectJsonParser(catalogs); - - if (action === 'compile') { - const payload = testCase.input as string; - - if (testCase.expect_error) { - assertThrows( - () => { - compileChecked(parser, payload); - }, - testCase.expect_error as Record, - ); - return; - } - const compiled = compileChecked(parser, payload); + const compiled = compileChecked(parser, payload); - if (testCase.expect_present) { - for (const pointer of testCase.expect_present as string[]) { - const val = resolvePointer(compiled, pointer); - expect(val !== null && val !== undefined && val !== '').toBe(true); - deletePointer(compiled, pointer); - } + if (testCase.expect_present) { + for (const pointer of testCase.expect_present as string[]) { + const val = resolvePointer(compiled, pointer); + expect(val !== null && val !== undefined && val !== '').toBe(true); + deletePointer(compiled, pointer); } + } - expect(compiled).toEqual(testCase.expect); - } else if (action === 'decompile') { - const messages = testCase.messages as AgentToRendererMessage[]; + expect(compiled).toEqual(testCase.expect); + } else if (action === 'decompile') { + const messages = testCase.messages as AgentToRendererMessage[]; - const notation = parser.decompile(messages); + const notation = parser.decompile(messages); - if (testCase.expect_contains) { - for (const fragment of testCase.expect_contains as string[]) { - expect(notation).toContain(fragment); - } + if (testCase.expect_contains) { + for (const fragment of testCase.expect_contains as string[]) { + expect(notation).toContain(fragment); } + } - if (testCase.expect_round_trip) { - expect(compileChecked(parser, notation)).toEqual(messages); + if (testCase.expect_round_trip) { + expect(compileChecked(parser, notation)).toEqual(messages); + } + } else if (action === 'unwrap') { + const expected = testCase.expect as Array>; + const actual = parser.unwrap(testCase.input as string); + assertRawPartsMatch(actual, expected); + const hasAny = expected.some(p => 'a2ui_raw' in p); + const hasComplete = expected.some(p => 'a2ui_raw' in p && (p.is_final ?? true) === true); + expect(parser.hasFormatContent(testCase.input as string)).toBe(hasAny); + expect(parser.hasFormatContent(testCase.input as string, {complete: true})).toBe(hasComplete); + } else if (action === 'has_format_content') { + const complete = (args.complete as boolean) ?? false; + expect(parser.hasFormatContent(testCase.input as string, {complete})).toBe(testCase.expect); + } else if (action === 'wrap') { + const parts = testCase.parts as Array>; + const rawParts: RawResponsePart[] = parts.map(p => { + if ('text' in p) { + return {type: 'text', text: p.text as string, isFinal: (p.is_final as boolean) ?? true}; } - } else if (action === 'unwrap') { - const expected = testCase.expect as Array>; - const actual = parser.unwrap(testCase.input as string); - assertRawPartsMatch(actual, expected); - const hasAny = expected.some(p => 'a2ui_raw' in p); - const hasComplete = expected.some(p => 'a2ui_raw' in p && (p.is_final ?? true) === true); - expect(parser.hasFormatContent(testCase.input as string)).toBe(hasAny); - expect(parser.hasFormatContent(testCase.input as string, {complete: true})).toBe( - hasComplete, - ); - } else if (action === 'has_format_content') { - const complete = (args.complete as boolean) ?? false; - expect(parser.hasFormatContent(testCase.input as string, {complete})).toBe(testCase.expect); - } else if (action === 'wrap') { - const parts = testCase.parts as Array>; - const rawParts: RawResponsePart[] = parts.map(p => { - if ('text' in p) { - return {type: 'text', text: p.text as string, isFinal: (p.is_final as boolean) ?? true}; - } - return { - type: 'a2ui', - a2uiRaw: p.a2ui_raw as string, - isFinal: (p.is_final as boolean) ?? true, - }; - }); + return { + type: 'a2ui', + a2uiRaw: p.a2ui_raw as string, + isFinal: (p.is_final as boolean) ?? true, + }; + }); - const output = parser.wrap(rawParts); - - if (testCase.expect_output !== undefined) { - expect(output).toBe(testCase.expect_output); - } - if (testCase.expect_contains) { - for (const fragment of testCase.expect_contains as string[]) { - expect(output).toContain(fragment); - } - } - if (testCase.expect_round_trip) { - const unwrapped = parser.unwrap(output); - assertRawPartsMatch(unwrapped, parts); - } - } else if (action === 'parse_response') { - const wrapped = (args.wrapped as boolean) ?? true; + const output = parser.wrap(rawParts); - if (testCase.expect_error) { - assertThrows( - () => { - parser.parseResponse(testCase.input as string, {wrapped}); - }, - testCase.expect_error as Record, - ); - return; + if (testCase.expect_output !== undefined) { + expect(output).toBe(testCase.expect_output); + } + if (testCase.expect_contains) { + for (const fragment of testCase.expect_contains as string[]) { + expect(output).toContain(fragment); } - - const parts = parser.parseResponse(testCase.input as string, {wrapped}); - assertPartsMatch(parts, testCase.expect as Array>); - } else { - throw new Error(`Unsupported action in direct_json conformance: ${action}`); } - }; + if (testCase.expect_round_trip) { + const unwrapped = parser.unwrap(output); + assertRawPartsMatch(unwrapped, parts); + } + } else if (action === 'parse_response') { + const wrapped = (args.wrapped as boolean) ?? true; + + if (testCase.expect_error) { + assertThrows( + () => { + parser.parseResponse(testCase.input as string, {wrapped}); + }, + testCase.expect_error as Record, + ); + return; + } - if (isUnsupported) { - test.skip(`${action} · ${name}`, runCase); - } else if (KNOWN_FAILURES.has(`${action} · ${name}`)) { - test.fails( - `${action} · ${name} (known gap: ${KNOWN_FAILURES.get(`${action} · ${name}`)})`, - runCase, - ); - } else if (KNOWN_FAILURES.has(name)) { - test.fails(`${action} · ${name} (known gap: ${KNOWN_FAILURES.get(name)})`, runCase); + const parts = parser.parseResponse(testCase.input as string, {wrapped}); + assertPartsMatch(parts, testCase.expect as Array>); } else { - test(`${action} · ${name}`, runCase); + throw new Error(`Unsupported action in direct_json conformance: ${action}`); } - } -}); + }, + { + knownFailures: KNOWN_FAILURES, + skipReason: testCase => UNSUPPORTED.get(testCase.name), + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/direct-json-prompt-generator.test.ts b/typescript/agents/a2ui_agent/tests/conformance/direct-json-prompt-generator.test.ts index 89e4ecde2a..ab05376b5e 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/direct-json-prompt-generator.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/direct-json-prompt-generator.test.ts @@ -14,63 +14,64 @@ * limitations under the License. */ -import * as fs from 'fs'; -import {describe, expect} from 'vitest'; +import {readFileSync} from 'fs'; + +import {expect} from 'vitest'; import {AgentToRendererMessage, DirectJsonPromptGenerator} from '../../src/index.js'; import {loadCases} from './loader.js'; import { CatalogRegistration, conformancePath, + describeSuite, errorClassFor, loadRegistrations, - registerCase, } from './suite-helpers.js'; /** Cases that run but do not pass yet, with the reason for each. */ const KNOWN_FAILURES = new Map([]); -describe('Conformance: direct_json/prompt_generator.yaml', () => { - for (const testCase of loadCases([conformancePath('agent/direct_json/prompt_generator.yaml')])) { - registerCase(testCase, KNOWN_FAILURES, async () => { - const args = testCase.args as { - format: string; - catalogs: CatalogRegistration[]; - examples?: string[]; - allowed_messages?: string[]; - }; - if (args.format !== 'direct_json') { - throw new Error(`Unexpected format '${args.format}'`); - } +describeSuite( + 'Conformance: direct_json/prompt_generator.yaml', + loadCases([conformancePath('agent/direct_json/prompt_generator.yaml')]), + async testCase => { + const args = testCase.args as { + format: string; + catalogs: CatalogRegistration[]; + examples?: string[]; + allowed_messages?: string[]; + }; + if (args.format !== 'direct_json') { + throw new Error(`Unexpected format '${args.format}'`); + } - const catalogs = loadRegistrations(args.catalogs).map(c => c.transformedCatalog); - const turns = (args.examples ?? []).map( - file => - JSON.parse(fs.readFileSync(conformancePath(file), 'utf8')) as AgentToRendererMessage[], - ); - const examples = turns.length > 0 ? turns : undefined; - const generate = () => - new DirectJsonPromptGenerator(catalogs, examples, args.allowed_messages).generate(); + const catalogs = loadRegistrations(args.catalogs).map(c => c.transformedCatalog); + const turns = (args.examples ?? []).map( + file => JSON.parse(readFileSync(conformancePath(file), 'utf8')) as AgentToRendererMessage[], + ); + const examples = turns.length > 0 ? turns : undefined; + const generate = () => + new DirectJsonPromptGenerator(catalogs, examples, args.allowed_messages).generate(); - if (testCase.expect_error) { - expect(generate).toThrow(errorClassFor(testCase)); - return; - } + if (testCase.expect_error) { + expect(generate).toThrow(errorClassFor(testCase)); + return; + } - const snippet = generate(); - if (typeof testCase.expect_file === 'string') { - const expectedText = fs.readFileSync(conformancePath(testCase.expect_file), 'utf8'); - expect(snippet).toBe(expectedText); - } - for (const needle of (testCase.expect_contains as string[] | undefined) ?? []) { - expect(snippet, `snippet should contain '${needle}'`).toContain(needle); - } - for (const needle of (testCase.expect_absent as string[] | undefined) ?? []) { - expect(snippet, `snippet should not contain '${needle}'`).not.toContain(needle); - } - if (testCase.expect_deterministic) { - expect(generate()).toBe(snippet); - } - }); - } -}); + const snippet = generate(); + if (typeof testCase.expect_file === 'string') { + const expectedText = readFileSync(conformancePath(testCase.expect_file), 'utf8'); + expect(snippet).toBe(expectedText); + } + for (const needle of (testCase.expect_contains as string[] | undefined) ?? []) { + expect(snippet, `snippet should contain '${needle}'`).toContain(needle); + } + for (const needle of (testCase.expect_absent as string[] | undefined) ?? []) { + expect(snippet, `snippet should not contain '${needle}'`).not.toContain(needle); + } + if (testCase.expect_deterministic) { + expect(generate()).toBe(snippet); + } + }, + {knownFailures: KNOWN_FAILURES}, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/direct-json-response-streaming.test.ts b/typescript/agents/a2ui_agent/tests/conformance/direct-json-response-streaming.test.ts index 9855282466..9501d0fe71 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/direct-json-response-streaming.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/direct-json-response-streaming.test.ts @@ -14,13 +14,14 @@ * limitations under the License. */ -import {describe, expect, test} from 'vitest'; +import {expect} from 'vitest'; + import {DirectJsonParser} from '../../src/inference-formats/direct-json/parser.js'; -import {FileSystemCatalogProvider} from '../../src/processor/catalog-providers.js'; -import {ResponseStream} from '../../src/parser/response-stream.js'; import {StreamPart} from '../../src/parser/response-part.js'; +import {ResponseStream} from '../../src/parser/response-stream.js'; +import {FileSystemCatalogProvider} from '../../src/processor/catalog-providers.js'; import {LoadedCase, loadCases} from './loader.js'; -import {categoryOf, conformancePath, registerCase} from './suite-helpers.js'; +import {categoryOf, conformancePath, describeSuite} from './suite-helpers.js'; /** * Cases that run but do not pass yet, with the reason for each. @@ -168,16 +169,15 @@ function runCase(testCase: LoadedCase): void { } } -describe('Conformance: direct_json/response_streaming.yaml', () => { - for (const testCase of loadCases([ - conformancePath('agent/direct_json/response_streaming.yaml'), - ])) { - const version = declaredVersion(testCase.args as unknown as StreamingArgs); - const unsupported = version ? UNSUPPORTED_VERSIONS.get(version) : undefined; - if (unsupported) { - test.skip(`${testCase.action} · ${testCase.name} (${unsupported})`, () => {}); - continue; - } - registerCase(testCase, KNOWN_FAILURES, () => runCase(testCase)); - } -}); +describeSuite( + 'Conformance: direct_json/response_streaming.yaml', + loadCases([conformancePath('agent/direct_json/response_streaming.yaml')]), + testCase => runCase(testCase), + { + knownFailures: KNOWN_FAILURES, + skipReason: testCase => { + const version = declaredVersion(testCase.args as unknown as StreamingArgs); + return version ? UNSUPPORTED_VERSIONS.get(version) : undefined; + }, + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/express-conformance.test.ts b/typescript/agents/a2ui_agent/tests/conformance/express-conformance.test.ts index 6d21ed5d10..71541070f2 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/express-conformance.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/express-conformance.test.ts @@ -14,13 +14,11 @@ * limitations under the License. */ -import * as fs from 'fs'; -import * as path from 'path'; -import {describe, test, expect} from 'vitest'; -import {CONFORMANCE_ROOT, loadCases} from './loader.js'; -import {createFileCatalogConfig} from './fixtures.js'; -import {compileChecked} from './suite-helpers.js'; -import {Catalog, CatalogApi} from '../../src/internal/web-core.js'; +import {readFileSync} from 'fs'; +import {resolve} from 'path'; + +import {expect} from 'vitest'; + import { A2uiCatalogError, A2uiCompilationError, @@ -37,6 +35,10 @@ import { RawResponsePart, ResponsePart, } from '../../src/index.js'; +import {Catalog, CatalogApi} from '../../src/internal/web-core.js'; +import {createFileCatalogConfig} from './fixtures.js'; +import {CONFORMANCE_ROOT, loadCases} from './loader.js'; +import {compileChecked, describeSuite} from './suite-helpers.js'; const CONFORMANCE_SURFACE_ID = 'default_surface'; const DEFAULT_CATALOG = 'test_data/catalogs/simplified_catalog_v1_0.json'; @@ -175,234 +177,219 @@ function assertPartsMatch( } } -describe('Express Conformance Suite', () => { - const files = [ - path.resolve(CONFORMANCE_ROOT, 'agent/express/compiler.yaml'), - path.resolve(CONFORMANCE_ROOT, 'agent/express/decompiler.yaml'), - path.resolve(CONFORMANCE_ROOT, 'agent/express/response_parser.yaml'), - path.resolve(CONFORMANCE_ROOT, 'agent/express/prompt_generator.yaml'), - ]; - - const cases = loadCases(files); - - for (const testCase of cases) { - const {name, action} = testCase; - const isUnsupported = UNSUPPORTED.has(name); - const isKnownFailure = KNOWN_FAILURES.has(name); - - const runCase = async () => { - const args = (testCase.args as Record) || {}; - - if (action === 'compile') { - const parser = await getParser(args); - const payload = testCase.input as string; - - if (testCase.expect_error) { - assertThrows( - () => { - compileChecked(parser, payload); - }, - testCase.expect_error as Record, - ); - return; - } +describeSuite( + 'Express Conformance Suite', + loadCases([ + resolve(CONFORMANCE_ROOT, 'agent/express/compiler.yaml'), + resolve(CONFORMANCE_ROOT, 'agent/express/decompiler.yaml'), + resolve(CONFORMANCE_ROOT, 'agent/express/response_parser.yaml'), + resolve(CONFORMANCE_ROOT, 'agent/express/prompt_generator.yaml'), + ]), + async testCase => { + const {action} = testCase; + const args = (testCase.args as Record) || {}; + + if (action === 'compile') { + const parser = await getParser(args); + const payload = testCase.input as string; + + if (testCase.expect_error) { + assertThrows( + () => { + compileChecked(parser, payload); + }, + testCase.expect_error as Record, + ); + return; + } - const compiled = compileChecked(parser, payload); + const compiled = compileChecked(parser, payload); - if (testCase.expect_present) { - for (const pointer of testCase.expect_present as string[]) { - const val = resolvePointer(compiled, pointer); - expect(val !== null && val !== undefined && val !== '').toBe(true); - deletePointer(compiled, pointer); - } + if (testCase.expect_present) { + for (const pointer of testCase.expect_present as string[]) { + const val = resolvePointer(compiled, pointer); + expect(val !== null && val !== undefined && val !== '').toBe(true); + deletePointer(compiled, pointer); } + } - expect(compiled).toEqual(testCase.expect); - } else if (action === 'decompile') { - const parser = await getParser(args); - const messages = testCase.messages as AgentToRendererMessage[]; - - if (testCase.expect_error) { - assertThrows( - () => { - parser.decompile(messages); - }, - testCase.expect_error as Record, - ); - return; - } + expect(compiled).toEqual(testCase.expect); + } else if (action === 'decompile') { + const parser = await getParser(args); + const messages = testCase.messages as AgentToRendererMessage[]; + + if (testCase.expect_error) { + assertThrows( + () => { + parser.decompile(messages); + }, + testCase.expect_error as Record, + ); + return; + } - const notation = parser.decompile(messages); + const notation = parser.decompile(messages); - if (testCase.expect_contains) { - for (const fragment of testCase.expect_contains as string[]) { - expect(notation).toContain(fragment); - } + if (testCase.expect_contains) { + for (const fragment of testCase.expect_contains as string[]) { + expect(notation).toContain(fragment); } - if (testCase.expect_absent) { - for (const fragment of testCase.expect_absent as string[]) { - expect(notation).not.toContain(fragment); - } + } + if (testCase.expect_absent) { + for (const fragment of testCase.expect_absent as string[]) { + expect(notation).not.toContain(fragment); } + } - if (testCase.expect_round_trip) { - expect(compileChecked(parser, notation)).toEqual(messages); - } - if (testCase.expect_recompiled) { - expect(compileChecked(parser, notation)).toEqual(testCase.expect_recompiled); + if (testCase.expect_round_trip) { + expect(compileChecked(parser, notation)).toEqual(messages); + } + if (testCase.expect_recompiled) { + expect(compileChecked(parser, notation)).toEqual(testCase.expect_recompiled); + } + } else if (action === 'unwrap') { + const parser = await getParser(args); + const expected = testCase.expect as Array>; + const actual = parser.unwrap(testCase.input as string); + assertRawPartsMatch(actual, expected); + const hasAny = expected.some(p => 'a2ui_raw' in p); + const hasComplete = expected.some(p => 'a2ui_raw' in p && (p.is_final ?? true) === true); + expect(parser.hasFormatContent(testCase.input as string)).toBe(hasAny); + expect(parser.hasFormatContent(testCase.input as string, {complete: true})).toBe(hasComplete); + } else if (action === 'has_format_content') { + const parser = await getParser(args); + const complete = (args.complete as boolean) ?? false; + expect(parser.hasFormatContent(testCase.input as string, {complete})).toBe(testCase.expect); + } else if (action === 'wrap') { + const parser = await getParser(args); + const parts = testCase.parts as Array>; + const rawParts: RawResponsePart[] = parts.map(p => { + if ('text' in p) { + return {type: 'text', text: p.text as string, isFinal: (p.is_final as boolean) ?? true}; } - } else if (action === 'unwrap') { - const parser = await getParser(args); - const expected = testCase.expect as Array>; - const actual = parser.unwrap(testCase.input as string); - assertRawPartsMatch(actual, expected); - const hasAny = expected.some(p => 'a2ui_raw' in p); - const hasComplete = expected.some(p => 'a2ui_raw' in p && (p.is_final ?? true) === true); - expect(parser.hasFormatContent(testCase.input as string)).toBe(hasAny); - expect(parser.hasFormatContent(testCase.input as string, {complete: true})).toBe( - hasComplete, - ); - } else if (action === 'has_format_content') { - const parser = await getParser(args); - const complete = (args.complete as boolean) ?? false; - expect(parser.hasFormatContent(testCase.input as string, {complete})).toBe(testCase.expect); - } else if (action === 'wrap') { - const parser = await getParser(args); - const parts = testCase.parts as Array>; - const rawParts: RawResponsePart[] = parts.map(p => { - if ('text' in p) { - return {type: 'text', text: p.text as string, isFinal: (p.is_final as boolean) ?? true}; - } - return { - type: 'a2ui', - a2uiRaw: p.a2ui_raw as string, - isFinal: (p.is_final as boolean) ?? true, - }; - }); + return { + type: 'a2ui', + a2uiRaw: p.a2ui_raw as string, + isFinal: (p.is_final as boolean) ?? true, + }; + }); - const output = parser.wrap(rawParts); + const output = parser.wrap(rawParts); - if (testCase.expect_output !== undefined) { - expect(output).toBe(testCase.expect_output); - } - if (testCase.expect_contains) { - for (const fragment of testCase.expect_contains as string[]) { - expect(output).toContain(fragment); - } - } - if (testCase.expect_round_trip) { - const unwrapped = parser.unwrap(output); - assertRawPartsMatch(unwrapped, parts); - } - } else if (action === 'parse_response') { - const parser = await getParser(args); - const wrapped = (args.wrapped as boolean) ?? true; - - if (testCase.expect_error) { - assertThrows( - () => { - parser.parseResponse(testCase.input as string, {wrapped}); - }, - testCase.expect_error as Record, - ); - return; + if (testCase.expect_output !== undefined) { + expect(output).toBe(testCase.expect_output); + } + if (testCase.expect_contains) { + for (const fragment of testCase.expect_contains as string[]) { + expect(output).toContain(fragment); } + } + if (testCase.expect_round_trip) { + const unwrapped = parser.unwrap(output); + assertRawPartsMatch(unwrapped, parts); + } + } else if (action === 'parse_response') { + const parser = await getParser(args); + const wrapped = (args.wrapped as boolean) ?? true; + + if (testCase.expect_error) { + assertThrows( + () => { + parser.parseResponse(testCase.input as string, {wrapped}); + }, + testCase.expect_error as Record, + ); + return; + } - const parts = parser.parseResponse(testCase.input as string, {wrapped}); - assertPartsMatch(parts, testCase.expect as Array>); - } else if (action === 'generate_prompt_snippet') { - const catEntries = - (args.catalogs as Array<{ - catalog: string; - transformers?: Array>; - }>) || []; - - if (testCase.expect_error) { - assertThrows( - () => { - new ExpressFormat([]); - }, - testCase.expect_error as Record, - ); - return; - } + const parts = parser.parseResponse(testCase.input as string, {wrapped}); + assertPartsMatch(parts, testCase.expect as Array>); + } else if (action === 'generate_prompt_snippet') { + const catEntries = + (args.catalogs as Array<{ + catalog: string; + transformers?: Array>; + }>) || []; + + if (testCase.expect_error) { + assertThrows( + () => { + new ExpressFormat([]); + }, + testCase.expect_error as Record, + ); + return; + } - const loadedCatalogs: CatalogApi[] = []; - for (const entry of catEntries) { - const fullPath = path.resolve(CONFORMANCE_ROOT, entry.catalog); - const schema = JSON.parse(fs.readFileSync(fullPath, 'utf8')); - let catalog = Catalog.fromJson(schema); - - if (entry.transformers) { - for (const trans of entry.transformers) { - if (trans.component_pruning) { - catalog = new ComponentPruningTransformer(trans.component_pruning).transform( - catalog, - ); - } - if (trans.function_pruning) { - catalog = new FunctionPruningTransformer(trans.function_pruning).transform(catalog); - } + const loadedCatalogs: CatalogApi[] = []; + for (const entry of catEntries) { + const fullPath = resolve(CONFORMANCE_ROOT, entry.catalog); + const schema = JSON.parse(readFileSync(fullPath, 'utf8')); + let catalog = Catalog.fromJson(schema); + + if (entry.transformers) { + for (const trans of entry.transformers) { + if (trans.component_pruning) { + catalog = new ComponentPruningTransformer(trans.component_pruning).transform(catalog); + } + if (trans.function_pruning) { + catalog = new FunctionPruningTransformer(trans.function_pruning).transform(catalog); } } - loadedCatalogs.push(catalog); } + loadedCatalogs.push(catalog); + } - let examples: Record | undefined; - if (Array.isArray(args.examples) && args.examples.length > 0) { - examples = {}; - for (const exPath of args.examples as string[]) { - const fullExPath = path.resolve(CONFORMANCE_ROOT, exPath); - const exContent = JSON.parse(fs.readFileSync(fullExPath, 'utf8')); - const messages: AgentToRendererMessage[] = Array.isArray(exContent) - ? exContent - : [exContent]; - const firstMsg = messages[0]; - const catId = - (firstMsg && 'createSurface' in firstMsg - ? firstMsg.createSurface.catalogId - : undefined) || loadedCatalogs[0]?.id; - if (catId) { - if (!examples[catId]) { - examples[catId] = []; - } - examples[catId].push(messages); + let examples: Record | undefined; + if (Array.isArray(args.examples) && args.examples.length > 0) { + examples = {}; + for (const exPath of args.examples as string[]) { + const fullExPath = resolve(CONFORMANCE_ROOT, exPath); + const exContent = JSON.parse(readFileSync(fullExPath, 'utf8')); + const messages: AgentToRendererMessage[] = Array.isArray(exContent) + ? exContent + : [exContent]; + const firstMsg = messages[0]; + const catId = + (firstMsg && 'createSurface' in firstMsg + ? firstMsg.createSurface.catalogId + : undefined) || loadedCatalogs[0]?.id; + if (catId) { + if (!examples[catId]) { + examples[catId] = []; } + examples[catId].push(messages); } } + } - const generator = new ExpressPromptGenerator( - loadedCatalogs, - examples, - args.allowed_messages as string[] | undefined, - ); - const output = generator.generate(); + const generator = new ExpressPromptGenerator( + loadedCatalogs, + examples, + args.allowed_messages as string[] | undefined, + ); + const output = generator.generate(); - if (testCase.expect_contains) { - for (const fragment of testCase.expect_contains as string[]) { - expect(output).toContain(fragment); - } + if (testCase.expect_contains) { + for (const fragment of testCase.expect_contains as string[]) { + expect(output).toContain(fragment); } - if (testCase.expect_absent) { - for (const fragment of testCase.expect_absent as string[]) { - expect(output).not.toContain(fragment); - } - } - if (testCase.expect_deterministic) { - const secondOutput = generator.generate(); - expect(output).toBe(secondOutput); + } + if (testCase.expect_absent) { + for (const fragment of testCase.expect_absent as string[]) { + expect(output).not.toContain(fragment); } - } else { - throw new Error(`Unsupported action in express conformance: ${action}`); } - }; - - if (isUnsupported) { - test.skip(`${action} · ${name}`, runCase); - } else if (isKnownFailure) { - test.fails(`${action} · ${name}`, runCase); + if (testCase.expect_deterministic) { + const secondOutput = generator.generate(); + expect(output).toBe(secondOutput); + } } else { - test(`${action} · ${name}`, runCase); + throw new Error(`Unsupported action in express conformance: ${action}`); } - } -}); + }, + { + knownFailures: KNOWN_FAILURES, + skipReason: testCase => UNSUPPORTED.get(testCase.name), + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/express-response-streaming.test.ts b/typescript/agents/a2ui_agent/tests/conformance/express-response-streaming.test.ts index 7fd976f65d..2852a51ee0 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/express-response-streaming.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/express-response-streaming.test.ts @@ -14,13 +14,14 @@ * limitations under the License. */ -import {describe, expect, test} from 'vitest'; +import {expect} from 'vitest'; + import {ExpressParser} from '../../src/inference-formats/express/parser.js'; import {StreamPart} from '../../src/parser/response-part.js'; import {ResponseStream} from '../../src/parser/response-stream.js'; import {FileSystemCatalogProvider} from '../../src/processor/catalog-providers.js'; import {LoadedCase, loadCases} from './loader.js'; -import {categoryOf, conformancePath, registerCase} from './suite-helpers.js'; +import {categoryOf, conformancePath, describeSuite} from './suite-helpers.js'; const KNOWN_FAILURES = new Map(); const UNSUPPORTED_VERSIONS = new Map([['0.8', 'this SDK does not implement v0.8']]); @@ -167,14 +168,15 @@ function runCase(testCase: LoadedCase): void { } } -describe('Conformance: express/response_streaming.yaml', () => { - for (const testCase of loadCases([conformancePath('agent/express/response_streaming.yaml')])) { - const version = declaredVersion(testCase.args as unknown as StreamingArgs); - const unsupported = version ? UNSUPPORTED_VERSIONS.get(version) : undefined; - if (unsupported) { - test.skip(`${testCase.action} · ${testCase.name} (${unsupported})`, () => {}); - continue; - } - registerCase(testCase, KNOWN_FAILURES, () => runCase(testCase)); - } -}); +describeSuite( + 'Conformance: express/response_streaming.yaml', + loadCases([conformancePath('agent/express/response_streaming.yaml')]), + testCase => runCase(testCase), + { + knownFailures: KNOWN_FAILURES, + skipReason: testCase => { + const version = declaredVersion(testCase.args as unknown as StreamingArgs); + return version ? UNSUPPORTED_VERSIONS.get(version) : undefined; + }, + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/loader.ts b/typescript/agents/a2ui_agent/tests/conformance/loader.ts index 9b959ac9f6..6089cb34ce 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/loader.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/loader.ts @@ -14,16 +14,19 @@ * limitations under the License. */ -import * as fs from 'fs'; -import * as path from 'path'; -import * as yaml from 'js-yaml'; +import {readFileSync} from 'fs'; +import {dirname, relative, resolve} from 'path'; import {fileURLToPath} from 'url'; +import {loadAll} from 'js-yaml'; + +import {recordCase} from './manifest.js'; + // Using __dirname isn't directly available in ESM unless constructed const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); +const __dirname = dirname(__filename); -export const CONFORMANCE_ROOT = path.resolve(__dirname, '../../../../../conformance'); +export const CONFORMANCE_ROOT = resolve(__dirname, '../../../../../conformance'); export interface TestCase { name: string; @@ -118,15 +121,22 @@ const SUPPORTED_FORMATS = new Set(['direct_json', 'express']); */ const UNIMPLEMENTED_ACTIONS = new Map([]); +function skipVerdict(testCase: TestCase, reason: string) { + if ('sourceFile' in testCase && typeof testCase.sourceFile === 'string') { + recordCase(testCase.sourceFile, testCase.name, 'skipped', reason); + } + return {runnable: false, reason}; +} + export function classify(testCase: TestCase) { const superseded = SUPERSEDED.get(testCase.name) ?? SUPERSEDED_ACTIONS.get(testCase.action); if (superseded) { - return {runnable: false, reason: superseded}; + return skipVerdict(testCase, superseded); } const unimplemented = UNIMPLEMENTED_ACTIONS.get(testCase.action); if (unimplemented) { - return {runnable: false, reason: unimplemented}; + return skipVerdict(testCase, unimplemented); } let version = 'unversioned'; @@ -148,10 +158,10 @@ export function classify(testCase: TestCase) { } if (version !== 'unversioned' && !SUPPORTED_PROTOCOL_VERSIONS.has(version)) { - return {runnable: false, reason: `Unsupported protocol version: ${version}`}; + return skipVerdict(testCase, `Unsupported protocol version: ${version}`); } if (!SUPPORTED_FORMATS.has(format)) { - return {runnable: false, reason: `Unsupported format: ${format}`}; + return skipVerdict(testCase, `Unsupported format: ${format}`); } return {runnable: true, reason: ''}; @@ -160,17 +170,22 @@ export function classify(testCase: TestCase) { export function loadCases(files: string[]): LoadedCase[] { const allCases: LoadedCase[] = []; for (const file of files) { - const content = fs.readFileSync(file, 'utf-8'); - const docs = yaml.loadAll(content) as Array< - Record | Record[] - >; + const sourceFile = relative(CONFORMANCE_ROOT, file).replace(/\\/g, '/'); + const content = readFileSync(file, 'utf-8'); + const docs = loadAll(content) as Array | Record[]>; for (const doc of docs) { if (Array.isArray(doc)) { for (const item of doc) { - allCases.push({...(item as TestCase), sourceFile: path.basename(file)}); + allCases.push({...(item as TestCase), sourceFile}); + } + } else if (doc && Array.isArray(doc.tests)) { + for (const item of doc.tests as Record[]) { + const name = (item.name ?? item.id) as string; + const action = (item.action ?? 'build') as string; + allCases.push({...(item as TestCase), name, action, sourceFile}); } } else if (doc) { - allCases.push({...(doc as TestCase), sourceFile: path.basename(file)}); + allCases.push({...(doc as TestCase), sourceFile}); } } } diff --git a/typescript/agents/a2ui_agent/tests/conformance/manifest.ts b/typescript/agents/a2ui_agent/tests/conformance/manifest.ts new file mode 100644 index 0000000000..24226928d4 --- /dev/null +++ b/typescript/agents/a2ui_agent/tests/conformance/manifest.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {appendFileSync, mkdirSync} from 'fs'; +import {join} from 'path'; + +import {afterEach} from 'vitest'; + +export type CaseStatus = 'passed' | 'failed' | 'skipped' | 'xfailed'; + +export interface ActiveCase { + suite: string; + name: string; + knownFailure?: string; +} + +let activeCase: ActiveCase | undefined; + +export function setActiveCase(current: ActiveCase | undefined): void { + activeCase = current; +} + +const recordedSkipped = new Set(); + +/** + * Appends one executed-case manifest entry for the TypeScript agent harness when + * `A2UI_CONFORMANCE_MANIFEST_DIR` is set. + */ +export function recordCase( + suite: string, + testCase: string, + status: CaseStatus, + reason?: string, +): void { + const manifestDir = process.env['A2UI_CONFORMANCE_MANIFEST_DIR']; + if (!manifestDir) { + return; + } + if (status === 'skipped') { + const key = `${suite}::${testCase}`; + if (recordedSkipped.has(key)) { + return; + } + recordedSkipped.add(key); + } + mkdirSync(manifestDir, {recursive: true}); + const record: Record = { + harness: 'ts-agent', + suite, + case: testCase, + status, + }; + if (reason) { + record['reason'] = reason; + } + const filePath = join(manifestDir, `ts-agent-${process.pid}.jsonl`); + appendFileSync(filePath, `${JSON.stringify(record)}\n`, 'utf8'); +} + +/** + * Registers a Vitest `afterEach` hook in the current suite that records the outcome of the + * active conformance case from `ctx.task.result?.state`. + */ +export function registerAfterEachManifestRecorder(): void { + afterEach(ctx => { + const current = activeCase; + activeCase = undefined; + if (!current) { + return; + } + const state = ctx.task.result?.state; + if (ctx.task.fails) { + // Vitest flips `task.result.state` for `test.fails` after `afterEach` completes, so + // `state === 'fail'` inside `afterEach` means the expected failure threw as expected. + if (state === 'fail') { + recordCase(current.suite, current.name, 'xfailed', current.knownFailure ?? 'known failure'); + } else { + recordCase(current.suite, current.name, 'failed', 'Expected test to fail'); + } + return; + } + if (state === 'pass') { + recordCase(current.suite, current.name, 'passed'); + } else { + const message = ctx.task.result?.errors?.[0]?.message ?? 'test failed'; + recordCase(current.suite, current.name, 'failed', message); + } + }); +} diff --git a/typescript/agents/a2ui_agent/tests/conformance/multi-catalog-conformance.test.ts b/typescript/agents/a2ui_agent/tests/conformance/multi-catalog-conformance.test.ts index fa35157f8f..cd889d7fb3 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/multi-catalog-conformance.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/multi-catalog-conformance.test.ts @@ -14,8 +14,9 @@ * limitations under the License. */ -import * as fs from 'fs'; -import {describe, expect, test} from 'vitest'; +import {readFileSync} from 'fs'; + +import {expect} from 'vitest'; import { A2uiCatalogError, @@ -47,6 +48,7 @@ import { CatalogRegistration, compileChecked, conformancePath, + describeSuite, expectCatalogContents, } from './suite-helpers.js'; @@ -151,7 +153,7 @@ function loadExamples( const examples: Record = {}; for (let i = 0; i < examplePaths.length; i++) { const fullPath = conformancePath(examplePaths[i]); - const content = JSON.parse(fs.readFileSync(fullPath, 'utf8')); + const content = JSON.parse(readFileSync(fullPath, 'utf8')); const messages: AgentToRendererMessage[] = Array.isArray(content) ? content : [content]; const firstSurfaceCatId = (messages[0] as {createSurface?: {catalogId?: string}} | undefined) ?.createSurface?.catalogId; @@ -443,29 +445,25 @@ function runFormatCase(testCase: LoadedCase): void { } } -describe('Conformance: multi_catalog_formats.yaml & request_processor.yaml', () => { - const files = [ +describeSuite( + 'Conformance: multi_catalog_formats.yaml & request_processor.yaml', + loadCases([ conformancePath('agent/multi_catalog_formats.yaml'), conformancePath('agent/request_processor.yaml'), - ]; - - for (const testCase of loadCases(files)) { - const args = (testCase.args as Record) || {}; - const fmt = (args.format as string) ?? 'express'; - const fmtOverride = args.format_override as string | undefined; - const isUnsupportedFormat = - (fmt !== 'express' && fmt !== 'direct_json') || - (fmtOverride !== undefined && fmtOverride !== 'express' && fmtOverride !== 'direct_json'); - - const title = `${testCase.action} · ${testCase.name}`; - if (isUnsupportedFormat) { - test.skip(title, () => { - runFormatCase(testCase); - }); - } else { - test(title, () => { - runFormatCase(testCase); - }); - } - } -}); + ]), + testCase => runFormatCase(testCase), + { + skipReason: testCase => { + const args = (testCase.args as Record) || {}; + const fmt = (args.format as string) ?? 'express'; + const fmtOverride = args.format_override as string | undefined; + if (fmt !== 'express' && fmt !== 'direct_json') { + return `Unsupported format: ${fmt}`; + } + if (fmtOverride !== undefined && fmtOverride !== 'express' && fmtOverride !== 'direct_json') { + return `Unsupported format override: ${fmtOverride}`; + } + return undefined; + }, + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/skill-conformance.test.ts b/typescript/agents/a2ui_agent/tests/conformance/skill-conformance.test.ts index 3cbe3a4e4f..d01826eeec 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/skill-conformance.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/skill-conformance.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import * as fs from 'fs'; -import * as os from 'os'; -import * as path from 'path'; +import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'fs'; +import {tmpdir} from 'os'; +import {dirname, join, resolve} from 'path'; -import {describe, expect, test} from 'vitest'; +import {expect} from 'vitest'; import { CatalogConfig, @@ -31,7 +31,7 @@ import { import {CatalogApi} from '../../src/internal/web-core.js'; import {loadBasicCatalog, REPO_ROOT} from '../helpers/basic-catalogs.js'; import {classify, KNOWN_FAILURES, loadCases, LoadedCase} from './loader.js'; -import {conformancePath, errorClassFor} from './suite-helpers.js'; +import {conformancePath, describeSuite, errorClassFor} from './suite-helpers.js'; const basicCatalogV10 = loadBasicCatalog('1.0'); @@ -46,7 +46,7 @@ function buildSkillContext(testCase: LoadedCase): SkillCaseContext { const catRelPath = args.catalog as string | undefined; let catalog: CatalogApi | undefined; if (catRelPath) { - const absCatPath = path.resolve(REPO_ROOT, catRelPath); + const absCatPath = resolve(REPO_ROOT, catRelPath); const catConfig = CatalogConfig.fromPath(absCatPath); catalog = catConfig.transformedCatalog; } @@ -64,7 +64,7 @@ function runFromFormatCase(testCase: LoadedCase, {args, generator}: SkillCaseCon const monoName = (args.name as string | undefined) ?? 'a2ui'; const skillObj = generator.generateSkill({name: monoName}); const expectedRel = testCase.expected_file as string; - const expectedContent = fs.readFileSync(path.resolve(REPO_ROOT, expectedRel), 'utf-8'); + const expectedContent = readFileSync(resolve(REPO_ROOT, expectedRel), 'utf-8'); expect(skillObj.toMarkdown()).toBe(expectedContent); } @@ -72,14 +72,14 @@ function runCoreSyntaxCase(testCase: LoadedCase, {args, generator}: SkillCaseCon const coreName = (args.name as string | undefined) ?? 'a2ui-core'; const skillObj = generator.generateCoreSkill({name: coreName}); const expectedRel = testCase.expected_file as string; - const expectedContent = fs.readFileSync(path.resolve(REPO_ROOT, expectedRel), 'utf-8'); + const expectedContent = readFileSync(resolve(REPO_ROOT, expectedRel), 'utf-8'); expect(skillObj.toMarkdown()).toBe(expectedContent); } function runFromCatalogCase(testCase: LoadedCase, {catalog, generator}: SkillCaseContext): void { const skillObj = generator.generateCatalogSkill({catalog}); const expectedRel = testCase.expected_file as string; - const expectedContent = fs.readFileSync(path.resolve(REPO_ROOT, expectedRel), 'utf-8'); + const expectedContent = readFileSync(resolve(REPO_ROOT, expectedRel), 'utf-8'); expect(skillObj.toMarkdown()).toBe(expectedContent); } @@ -92,7 +92,7 @@ function runSkillSetCase(testCase: LoadedCase, {generator}: SkillCaseContext): v } const expectedFiles = (testCase.expected_skill_files as Record) || {}; for (const [k, goldenRel] of Object.entries(expectedFiles)) { - const golden = fs.readFileSync(path.resolve(REPO_ROOT, goldenRel), 'utf-8'); + const golden = readFileSync(resolve(REPO_ROOT, goldenRel), 'utf-8'); expect(dict[k], `content of ${k}`).toBe(golden); } } @@ -110,12 +110,12 @@ interface LiteralSkill { */ function runExportToDirectoryCase(testCase: LoadedCase): void { const args = (testCase.args as Record) || {}; - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'a2ui-skill-export-')); + const tempDir = mkdtempSync(join(tmpdir(), 'a2ui-skill-export-')); try { for (const rel of (args.pre_existing_files as string[] | undefined) ?? []) { - const abs = path.join(tempDir, rel); - fs.mkdirSync(path.dirname(abs), {recursive: true}); - fs.writeFileSync(abs, 'pre-existing', 'utf-8'); + const abs = join(tempDir, rel); + mkdirSync(dirname(abs), {recursive: true}); + writeFileSync(abs, 'pre-existing', 'utf-8'); } const run = () => { @@ -132,13 +132,13 @@ function runExportToDirectoryCase(testCase: LoadedCase): void { } for (const rel of (testCase.expected_files as string[] | undefined) ?? []) { - expect(fs.existsSync(path.join(tempDir, rel)), `expected ${rel} to exist`).toBe(true); + expect(existsSync(join(tempDir, rel)), `expected ${rel} to exist`).toBe(true); } for (const rel of (testCase.expected_absent_files as string[] | undefined) ?? []) { - expect(fs.existsSync(path.join(tempDir, rel)), `expected ${rel} to be removed`).toBe(false); + expect(existsSync(join(tempDir, rel)), `expected ${rel} to be removed`).toBe(false); } } finally { - fs.rmSync(tempDir, {recursive: true, force: true}); + rmSync(tempDir, {recursive: true, force: true}); } } @@ -152,33 +152,29 @@ const SKILL_ACTION_HANDLERS: Record< skill_set: runSkillSetCase, }; -describe('Conformance: skill.yaml', () => { - for (const testCase of loadCases([conformancePath('agent/skill.yaml')])) { - const verdict = classify(testCase); - const {action, name} = testCase; - const testName = `${action} · ${name}`; - - const testFn = async () => { - if (action === 'export_to_directory') { - runExportToDirectoryCase(testCase); - return; - } - const handler = Object.hasOwn(SKILL_ACTION_HANDLERS, action) - ? SKILL_ACTION_HANDLERS[action] - : undefined; - if (!handler) { - throw new Error(`Unhandled conformance action: ${action}`); - } - const ctx = buildSkillContext(testCase); - await handler(testCase, ctx); - }; - - if (!verdict.runnable) { - test.skip(`${testName} (${verdict.reason})`, testFn); - } else if (KNOWN_FAILURES.has(name)) { - test.fails(`${testName} (known gap: ${KNOWN_FAILURES.get(name)})`, testFn); - } else { - test(testName, testFn); +describeSuite( + 'Conformance: skill.yaml', + loadCases([conformancePath('agent/skill.yaml')]), + async testCase => { + const {action} = testCase; + if (action === 'export_to_directory') { + runExportToDirectoryCase(testCase); + return; } - } -}); + const handler = Object.hasOwn(SKILL_ACTION_HANDLERS, action) + ? SKILL_ACTION_HANDLERS[action] + : undefined; + if (!handler) { + throw new Error(`Unhandled conformance action: ${action}`); + } + const ctx = buildSkillContext(testCase); + await handler(testCase, ctx); + }, + { + knownFailures: KNOWN_FAILURES, + skipReason: testCase => { + const verdict = classify(testCase); + return verdict.runnable ? undefined : verdict.reason; + }, + }, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/streamed-answer.test.ts b/typescript/agents/a2ui_agent/tests/conformance/streamed-answer.test.ts index 888c957ca2..330b6c8c97 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/streamed-answer.test.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/streamed-answer.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {describe, expect} from 'vitest'; +import {expect} from 'vitest'; import { A2uiRequestProcessor, @@ -25,7 +25,7 @@ import { } from '../../src/index.js'; import {MessageProcessor} from '../../src/internal/web-core.js'; import {loadCases} from './loader.js'; -import {categoryOf, conformancePath, registerCase} from './suite-helpers.js'; +import {categoryOf, conformancePath, describeSuite} from './suite-helpers.js'; /** Cases that run but do not pass yet, with the reason for each. */ const KNOWN_FAILURES = new Map(); @@ -54,48 +54,49 @@ function partToDict(part: AnswerPart, renderer: MessageProcessor): Record { - for (const testCase of loadCases([conformancePath('agent/streamed_answer.yaml')])) { - registerCase(testCase, KNOWN_FAILURES, () => { - const args = testCase.args as unknown as AnswerArgs; - if (args.format !== 'direct_json') { - throw new Error(`Unexpected format '${args.format}'`); - } - const processor = new A2uiRequestProcessor( - [load(args.catalog)], - undefined, - new DirectJsonFormatFactory(), - ); - const renderer = new MessageProcessor( - (args.renderer_catalogs ?? [args.catalog]).map(load) as never, - ); - const open = () => new StreamedAnswer(processor, {renderer}); +describeSuite( + 'Conformance: streamed_answer.yaml', + loadCases([conformancePath('agent/streamed_answer.yaml')]), + testCase => { + const args = testCase.args as unknown as AnswerArgs; + if (args.format !== 'direct_json') { + throw new Error(`Unexpected format '${args.format}'`); + } + const processor = new A2uiRequestProcessor( + [load(args.catalog)], + undefined, + new DirectJsonFormatFactory(), + ); + const renderer = new MessageProcessor( + (args.renderer_catalogs ?? [args.catalog]).map(load) as never, + ); + const open = () => new StreamedAnswer(processor, {renderer}); - const expectError = testCase.expect_error as {category: string} | undefined; - if (expectError) { - let thrown: unknown; - try { - open(); - } catch (e) { - thrown = e; - } - expect(categoryOf(thrown)).toBe(expectError.category); - return; + const expectError = testCase.expect_error as {category: string} | undefined; + if (expectError) { + let thrown: unknown; + try { + open(); + } catch (e) { + thrown = e; } + expect(categoryOf(thrown)).toBe(expectError.category); + return; + } - const answer = open(); - const parts = () => answer.parts.map(part => partToDict(part, renderer)); - for (const [index, step] of (testCase.steps as Step[]).entries()) { - answer.feed(step.input); - expect(parts(), `step ${index}: ${step.input}`).toEqual(step.expect_parts); - if (step.expect_in_block !== undefined) { - expect(answer.isInBlock, `step ${index}: in block`).toBe(step.expect_in_block); - } - } - answer.close(); - if (testCase.expect_close) { - expect(parts(), 'close').toEqual(testCase.expect_close); + const answer = open(); + const parts = () => answer.parts.map(part => partToDict(part, renderer)); + for (const [index, step] of (testCase.steps as Step[]).entries()) { + answer.feed(step.input); + expect(parts(), `step ${index}: ${step.input}`).toEqual(step.expect_parts); + if (step.expect_in_block !== undefined) { + expect(answer.isInBlock, `step ${index}: in block`).toBe(step.expect_in_block); } - }); - } -}); + } + answer.close(); + if (testCase.expect_close) { + expect(parts(), 'close').toEqual(testCase.expect_close); + } + }, + {knownFailures: KNOWN_FAILURES}, +); diff --git a/typescript/agents/a2ui_agent/tests/conformance/suite-helpers.ts b/typescript/agents/a2ui_agent/tests/conformance/suite-helpers.ts index 9d58ec9794..2c9dd85916 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/suite-helpers.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/suite-helpers.ts @@ -18,14 +18,14 @@ * Shared helpers for the harnesses that run the agent suites written against the module * blueprint (`conformance/agent/*.yaml` and `conformance/agent/direct-json/*.yaml`). * - * Those suites spell their keys in snake_case (`expect_error`, `catalog_id`), unlike the + * Those suites write their keys in snake_case (`expect_error`, `catalog_id`), unlike the * legacy suites that `conformance.test.ts` runs. */ -import * as path from 'path'; -import {expect, test} from 'vitest'; +import {resolve} from 'path'; + +import {describe, expect, test} from 'vitest'; -import {CatalogApi} from '../../src/internal/web-core.js'; import { A2uiCatalogError, A2uiCompilationError, @@ -43,7 +43,9 @@ import { Parser, validatePayload, } from '../../src/index.js'; +import {CatalogApi} from '../../src/internal/web-core.js'; import {CONFORMANCE_ROOT, LoadedCase} from './loader.js'; +import {recordCase, registerAfterEachManifestRecorder, setActiveCase} from './manifest.js'; /** Error categories the suites name, mapped to the SDK's error classes. */ const ERROR_BY_CATEGORY: Record Error> = { @@ -72,7 +74,7 @@ export function categoryOf(error: unknown): string { } export function conformancePath(relative: string): string { - return path.resolve(CONFORMANCE_ROOT, relative); + return resolve(CONFORMANCE_ROOT, relative); } /** The error class for the category a case names in `expect_error`. */ @@ -141,7 +143,7 @@ export function expectCatalogContents( /** * Registers one case with vitest, as an expected failure when it is listed in - * `knownFailures`. + * `knownFailures` or as a skip when `skipReason` is given. * * An expected failure still runs, and the suite asserts that it fails, so fixing the gap * turns the test red and prompts removing the entry. @@ -150,12 +152,51 @@ export function registerCase( testCase: LoadedCase, knownFailures: ReadonlyMap, run: () => void | Promise, + skipReason?: string, ): void { const title = `${testCase.action} · ${testCase.name}`; - const knownFailure = knownFailures.get(testCase.name); + if (skipReason) { + recordCase(testCase.sourceFile, testCase.name, 'skipped', skipReason); + test.skip(`${title} (${skipReason})`, () => {}); + return; + } + const knownFailure = knownFailures.get(testCase.name) ?? knownFailures.get(title); + const wrappedRun = async () => { + setActiveCase({ + suite: testCase.sourceFile, + name: testCase.name, + knownFailure, + }); + await run(); + }; if (knownFailure) { - test.fails(`${title} (known gap: ${knownFailure})`, run); + test.fails(`${title} (known gap: ${knownFailure})`, wrappedRun); } else { - test(title, run); + test(title, wrappedRun); } } + +export interface DescribeSuiteOptions { + knownFailures?: ReadonlyMap; + skipReason?: (testCase: LoadedCase) => string | undefined; +} + +/** + * Declares a Vitest `describe` block for one or more conformance suites, registers the + * `afterEach` manifest recorder, and registers each loaded case. + */ +export function describeSuite( + suiteTitle: string, + cases: readonly LoadedCase[], + run: (testCase: LoadedCase) => void | Promise, + options: DescribeSuiteOptions = {}, +): void { + const knownFailures = options.knownFailures ?? new Map(); + describe(suiteTitle, () => { + registerAfterEachManifestRecorder(); + for (const testCase of cases) { + const reason = options.skipReason?.(testCase); + registerCase(testCase, knownFailures, () => run(testCase), reason); + } + }); +} diff --git a/typescript/agents/a2ui_agent/tests/conformance/unsupported-suites.test.ts b/typescript/agents/a2ui_agent/tests/conformance/unsupported-suites.test.ts new file mode 100644 index 0000000000..55822e5483 --- /dev/null +++ b/typescript/agents/a2ui_agent/tests/conformance/unsupported-suites.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {loadCases} from './loader.js'; +import {conformancePath, describeSuite} from './suite-helpers.js'; + +const UNSUPPORTED_SUITES: ReadonlyArray<{suite: string; reason: string}> = [ + { + suite: 'agent/builder/builder.yaml', + reason: 'the fluent builder API is a Python-only helper not implemented in TypeScript', + }, + { + suite: 'agent/macros/macros.yaml', + reason: 'the composite macro API is a Python-only helper not implemented in TypeScript', + }, + { + suite: 'agent/legacy/streaming_parser.yaml', + reason: 'superseded by direct_json/response_streaming.yaml and express/response_streaming.yaml', + }, + { + suite: 'agent/direct_json/scaling.yaml', + reason: 'timing suite runs separately under yarn test:timing', + }, + { + suite: 'agent/express/scaling.yaml', + reason: 'timing suite runs separately under yarn test:timing', + }, +]; + +for (const {suite, reason} of UNSUPPORTED_SUITES) { + describeSuite(`Conformance (skipped): ${suite}`, loadCases([conformancePath(suite)]), () => {}, { + skipReason: () => reason, + }); +} From c110059bf263ea767b7ce803709f272dfa286b76 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:23:43 +0000 Subject: [PATCH 21/48] test(dart/a2ui_agent): executed-case manifest for the Dart agent harnesses Add strict key/action dispatch, an executed-case manifest writer, reasoned skip records for unsupported and performance-tagged agent suites, and the CI manifest check for dart/a2ui_agent. --- .github/workflows/flutter_packages_test.yml | 12 +- .../catalog_provider_conformance_test.dart | 30 +++ .../catalog_resolution_conformance_test.dart | 35 +++ .../catalog_transformer_conformance_test.dart | 28 ++- .../direct_json_conformance_test.dart | 105 ++++++++- .../conformance/express_conformance_test.dart | 111 ++++++++- .../a2ui_agent/test/conformance/manifest.dart | 213 ++++++++++++++++++ ...ulti_catalog_formats_conformance_test.dart | 34 ++- .../request_processor_conformance_test.dart | 94 +++++++- .../conformance/scaling_conformance_test.dart | 40 +++- .../streamed_answer_conformance_test.dart | 32 +++ dart/a2ui_agent/test/conformance/suites.dart | 116 +++++++++- .../conformance/unsupported_suites_test.dart | 211 +++++++++++++++++ 13 files changed, 1020 insertions(+), 41 deletions(-) create mode 100644 dart/a2ui_agent/test/conformance/manifest.dart create mode 100644 dart/a2ui_agent/test/conformance/unsupported_suites_test.dart diff --git a/.github/workflows/flutter_packages_test.yml b/.github/workflows/flutter_packages_test.yml index ad74816826..d0478f4c36 100644 --- a/.github/workflows/flutter_packages_test.yml +++ b/.github/workflows/flutter_packages_test.yml @@ -131,7 +131,7 @@ jobs: - name: Run tests working-directory: ${{ matrix.package.path }} env: - A2UI_CONFORMANCE_MANIFEST_DIR: ${{ matrix.package.path == 'dart/a2ui_core' && format('{0}/a2ui-manifest', runner.temp) || '' }} + A2UI_CONFORMANCE_MANIFEST_DIR: ${{ (matrix.package.path == 'dart/a2ui_core' || matrix.package.path == 'dart/a2ui_agent') && format('{0}/a2ui-manifest', runner.temp) || '' }} run: | if [ -d "test" ]; then flutter test --test-randomize-ordering-seed=random @@ -149,6 +149,16 @@ jobs: --harness dart-core \ --corpus core \ --exclude "core/message_processor_scaling.yaml=performance: runs with -P performance" + - name: Set up Python for agent conformance manifest check + if: matrix.package.path == 'dart/a2ui_agent' + uses: ./.github/actions/setup-python + - name: Check executed agent conformance cases + if: matrix.package.path == 'dart/a2ui_agent' + run: | + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --corpus agent \ + --harness dart-agent # The e2e_test packages are excluded from the matrix above because their # tests call a live Gemini model, which needs credentials and quota that are diff --git a/dart/a2ui_agent/test/conformance/catalog_provider_conformance_test.dart b/dart/a2ui_agent/test/conformance/catalog_provider_conformance_test.dart index f37dec1283..5da9a39e35 100644 --- a/dart/a2ui_agent/test/conformance/catalog_provider_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/catalog_provider_conformance_test.dart @@ -47,12 +47,42 @@ void main() { } } +const Map> _knownKeys = { + 'case:provide_catalog': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + }, + 'args:provide_catalog': { + 'catalog_id', + 'path', + 'protocol_version', + 'provider', + }, +}; + void _runCase( Map testCase, SuiteVersion version, Directory temp, ) { + switch (testCase['action']) { + case 'provide_catalog': + assertKnownKeys( + testCase, + _knownKeys['case:provide_catalog']!, + 'case:provide_catalog', + ); + default: + throw UnsupportedError('Unknown action ${testCase['action']}'); + } final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:provide_catalog']!, + 'args:provide_catalog', + ); expect(args['provider'], 'file_system'); final bool lower = version == SuiteVersion.v0_9 && args['protocol_version'] != 'v0.9'; diff --git a/dart/a2ui_agent/test/conformance/catalog_resolution_conformance_test.dart b/dart/a2ui_agent/test/conformance/catalog_resolution_conformance_test.dart index 79ae45decc..f37fadae1b 100644 --- a/dart/a2ui_agent/test/conformance/catalog_resolution_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/catalog_resolution_conformance_test.dart @@ -33,8 +33,38 @@ void main() { } } +const Map> _knownKeys = { + 'case:resolve_catalogs': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + }, + 'args:resolve_catalogs': { + 'accepts_inline_catalogs', + 'catalogs', + 'renderer_capabilities', + }, + 'expect:resolve_catalogs': {'active_catalog_ids', 'catalogs'}, +}; + void _runCase(Map testCase, SuiteVersion version) { + switch (testCase['action']) { + case 'resolve_catalogs': + assertKnownKeys( + testCase, + _knownKeys['case:resolve_catalogs']!, + 'case:resolve_catalogs', + ); + default: + throw UnsupportedError('Unknown action ${testCase['action']}'); + } final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:resolve_catalogs']!, + 'args:resolve_catalogs', + ); final capabilities = args['renderer_capabilities'] as Map?; List resolve() => resolveCatalogs( [ @@ -51,6 +81,11 @@ void _runCase(Map testCase, SuiteVersion version) { } final List active = resolve(); final expected = testCase['expect']! as Map; + assertKnownKeys( + expected, + _knownKeys['expect:resolve_catalogs']!, + 'expect:resolve_catalogs', + ); expect( active.map((c) => c.id), unorderedEquals(expected['active_catalog_ids']! as List), diff --git a/dart/a2ui_agent/test/conformance/catalog_transformer_conformance_test.dart b/dart/a2ui_agent/test/conformance/catalog_transformer_conformance_test.dart index 577de9e17a..1ea1e60531 100644 --- a/dart/a2ui_agent/test/conformance/catalog_transformer_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/catalog_transformer_conformance_test.dart @@ -16,6 +16,16 @@ import 'package:test/test.dart'; import 'suites.dart'; +const Map> _knownKeys = { + 'case:transform_catalog': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + }, + 'args:transform_catalog': {'catalog', 'transformers'}, +}; + /// Runs the shared `conformance/agent/catalog_transformer.yaml` suite, in /// each [SuiteVersion]. void main() { @@ -26,8 +36,24 @@ void main() { test('suite is not empty', () => expect(cases, isNotEmpty)); for (final testCase in cases) { test(testCase['name']! as String, () { + switch (testCase['action']) { + case 'transform_catalog': + assertKnownKeys( + testCase, + _knownKeys['case:transform_catalog']!, + 'case:transform_catalog', + ); + default: + throw UnsupportedError('Unknown action ${testCase['action']}'); + } + final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:transform_catalog']!, + 'args:transform_catalog', + ); version.expectCatalog( - version.catalogConfig(testCase['args']).transformedCatalog, + version.catalogConfig(args).transformedCatalog, testCase['expect']! as Map, ); }); diff --git a/dart/a2ui_agent/test/conformance/direct_json_conformance_test.dart b/dart/a2ui_agent/test/conformance/direct_json_conformance_test.dart index c3abb687f1..2fe6cccdb7 100644 --- a/dart/a2ui_agent/test/conformance/direct_json_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/direct_json_conformance_test.dart @@ -31,7 +31,83 @@ import 'suites.dart'; /// catalog declares v0.8 is skipped, since this SDK does not implement it. /// /// As written in v1.0, only the cases whose catalogs target v1.0 run. +const Map> _knownKeys = { + 'case:compile': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + 'input', + }, + 'case:decompile': { + ...caseMetadataKeys, + 'args', + 'expect_contains', + 'expect_round_trip', + 'messages', + }, + 'case:generate_prompt_snippet': { + ...caseMetadataKeys, + 'args', + 'expect_deterministic', + 'expect_error', + 'expect_file', + }, + 'case:has_format_content': {...caseMetadataKeys, 'args', 'expect', 'input'}, + 'case:parse_chunk': { + ...caseMetadataKeys, + 'args', + 'expect_close', + 'expect_matches_single_shot', + 'steps', + }, + 'case:parse_response': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + 'input', + }, + 'case:unwrap': {...caseMetadataKeys, 'args', 'expect', 'input'}, + 'case:wrap': { + ...caseMetadataKeys, + 'args', + 'expect_contains', + 'expect_output', + 'expect_round_trip', + 'parts', + }, + 'args:compile': {'catalog', 'format'}, + 'args:decompile': {'catalog', 'catalogs', 'format'}, + 'args:generate_prompt_snippet': { + 'allowed_messages', + 'catalogs', + 'examples', + 'format', + }, + 'args:has_format_content': {'complete', 'format'}, + 'args:parse_chunk': { + 'after_rejected_block', + 'catalog', + 'catalogs', + 'format', + 'progressive_keys', + 'wrapped', + }, + 'args:parse_response': {'catalog', 'format', 'wrapped'}, + 'args:unwrap': {'format'}, + 'args:wrap': {'format'}, + 'step:parse_chunk': { + 'expect', + 'expect_error', + 'expect_in_block', + 'expect_partial_results', + 'input', + }, +}; + void main() { + final allCaseNames = {}; for (final suite in [ 'compiler', 'decompiler', @@ -45,16 +121,15 @@ void main() { final List> cases = loadSuite(path); test('suite is not empty', () => expect(cases, isNotEmpty)); for (final testCase in cases) { + final name = testCase['name']! as String; + allCaseNames.add(name); if (version == SuiteVersion.v1_0 && !_targetsV1(testCase)) continue; - test( - testCase['name']! as String, - () => _runCase(testCase, version), - skip: _skipReason(testCase, version), - ); + test(name, () => _runCase(testCase, version)); } }); } } + verifyNoUnusedEntries('direct_json', allCaseNames, _v1OnlyCases); } /// v1.0 streaming cases that rely on what v0.9 cannot state: a component @@ -95,7 +170,20 @@ String? _skipReason(Map testCase, SuiteVersion version) { } void _runCase(Map testCase, SuiteVersion version) { - switch (testCase['action']) { + if (_skipReason(testCase, version) case final String reason) { + markTestSkipped(reason); + return; + } + final action = testCase['action'] as String?; + final Set? allowedCaseKeys = _knownKeys['case:$action']; + if (allowedCaseKeys == null) { + throw UnsupportedError('Unknown action $action'); + } + assertKnownKeys(testCase, allowedCaseKeys, 'case:$action'); + final args = testCase['args']! as Map; + assertKnownKeys(args, _knownKeys['args:$action']!, 'args:$action'); + expect(args['format'], 'direct_json'); + switch (action) { case 'parse_chunk': _runChunks(testCase, version); return; @@ -228,6 +316,11 @@ void _runChunks(Map testCase, SuiteVersion version) { for (final (int i, Object? step) in (testCase['steps']! as List).indexed) { final stepCase = step! as Map; + assertKnownKeys( + stepCase, + _knownKeys['step:parse_chunk']!, + 'step:parse_chunk', + ); final String chunk = written.text( stepCase['input']! as String, catalogId: catalogId, diff --git a/dart/a2ui_agent/test/conformance/express_conformance_test.dart b/dart/a2ui_agent/test/conformance/express_conformance_test.dart index 774696dd63..bd480659e3 100644 --- a/dart/a2ui_agent/test/conformance/express_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/express_conformance_test.dart @@ -30,7 +30,87 @@ import 'suites.dart'; /// /// A case that needs something this SDK does not implement is skipped with /// the reason. +const Map> _knownKeys = { + 'case:compile': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + 'expect_present', + 'input', + }, + 'case:decompile': { + ...caseMetadataKeys, + 'args', + 'expect_contains', + 'expect_error', + 'expect_recompiled', + 'expect_round_trip', + 'messages', + }, + 'case:generate_prompt_snippet': { + ...caseMetadataKeys, + 'args', + 'expect_absent', + 'expect_contains', + 'expect_deterministic', + 'expect_error', + }, + 'case:has_format_content': {...caseMetadataKeys, 'args', 'expect', 'input'}, + 'case:parse_chunk': { + ...caseMetadataKeys, + 'args', + 'expect_close', + 'expect_error', + 'expect_matches_single_shot', + 'steps', + }, + 'case:parse_response': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + 'input', + }, + 'case:unwrap': {...caseMetadataKeys, 'args', 'expect', 'input'}, + 'case:wrap': { + ...caseMetadataKeys, + 'args', + 'expect_contains', + 'expect_output', + 'expect_round_trip', + 'parts', + }, + 'args:compile': {'catalog', 'catalogs', 'format'}, + 'args:decompile': {'catalog', 'catalogs', 'format'}, + 'args:generate_prompt_snippet': { + 'allowed_messages', + 'catalogs', + 'examples', + 'format', + }, + 'args:has_format_content': {'complete', 'format'}, + 'args:parse_chunk': { + 'after_rejected_block', + 'catalog', + 'format', + 'progressive_keys', + 'wrapped', + }, + 'args:parse_response': {'catalog', 'format', 'wrapped'}, + 'args:unwrap': {'format'}, + 'args:wrap': {'format'}, + 'step:parse_chunk': { + 'expect', + 'expect_error', + 'expect_in_block', + 'expect_partial_results', + 'input', + }, +}; + void main() { + final allCaseNames = {}; for (final suite in [ 'compiler', 'response_parser', @@ -44,15 +124,14 @@ void main() { final List> cases = loadSuite(path); test('suite is not empty', () => expect(cases, isNotEmpty)); for (final testCase in cases) { - test( - testCase['name']! as String, - () => _runCase(testCase, version), - skip: _skipReason(testCase, version), - ); + final name = testCase['name']! as String; + allCaseNames.add(name); + test(name, () => _runCase(testCase, version)); } }); } } + verifyNoUnusedEntries('express', allCaseNames, _v1OnlyCases.keys); } const String _oneCatalogPerSurface = @@ -127,7 +206,20 @@ String? _skipReason(Map testCase, SuiteVersion version) { } void _runCase(Map testCase, SuiteVersion version) { - if (testCase['action'] == 'parse_chunk') { + if (_skipReason(testCase, version) case final String reason) { + markTestSkipped(reason); + return; + } + final action = testCase['action'] as String?; + final Set? allowedCaseKeys = _knownKeys['case:$action']; + if (allowedCaseKeys == null) { + throw UnsupportedError('Unknown action $action'); + } + assertKnownKeys(testCase, allowedCaseKeys, 'case:$action'); + final args = testCase['args']! as Map; + assertKnownKeys(args, _knownKeys['args:$action']!, 'args:$action'); + expect(args['format'], 'express'); + if (action == 'parse_chunk') { _runChunks(testCase, version); return; } @@ -137,7 +229,7 @@ void _runCase(Map testCase, SuiteVersion version) { return; } final Object? result = _perform(testCase, version); - switch (testCase['action']) { + switch (action) { case 'generate_prompt_snippet': _checkSnippet(testCase, version, result! as String); case 'wrap': @@ -178,6 +270,11 @@ void _runChunks(Map testCase, SuiteVersion version) { for (final (int i, Object? step) in (testCase['steps']! as List).indexed) { final stepCase = step! as Map; + assertKnownKeys( + stepCase, + _knownKeys['step:parse_chunk']!, + 'step:parse_chunk', + ); final chunk = stepCase['input']! as String; chunks.add(chunk); if (stepCase['expect_error'] case final Object error) { diff --git a/dart/a2ui_agent/test/conformance/manifest.dart b/dart/a2ui_agent/test/conformance/manifest.dart new file mode 100644 index 0000000000..27536c1569 --- /dev/null +++ b/dart/a2ui_agent/test/conformance/manifest.dart @@ -0,0 +1,213 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:test/test.dart'; + +import 'suites.dart'; + +/// Harness identifier written to each executed-case manifest entry. +const String harnessName = 'dart-agent'; + +class _VersionOutcome { + const _VersionOutcome(this.status, this.reason); + + final String status; + final String? reason; +} + +class _CaseEntry { + final Map outcomes = {}; + + (String status, String? reason) combined() { + String? failedReason; + var hasFailed = false; + String? xfailedReason; + var hasXfailed = false; + var hasPassed = false; + String? skippedReason; + + for (final _VersionOutcome outcome in outcomes.values) { + switch (outcome.status) { + case 'failed': + hasFailed = true; + failedReason ??= outcome.reason; + case 'xfailed': + hasXfailed = true; + xfailedReason ??= outcome.reason; + case 'passed': + hasPassed = true; + skippedReason ??= outcome.reason; + case 'skipped': + skippedReason ??= outcome.reason; + } + } + + if (hasFailed) { + return ('failed', failedReason ?? skippedReason); + } + if (hasXfailed) { + return ('xfailed', xfailedReason ?? skippedReason); + } + if (hasPassed) { + return ('passed', skippedReason); + } + return ('skipped', skippedReason); + } +} + +final Map> _recordsByDir = {}; + +/// Records the outcome of conformance case [name] in [suite] to the +/// executed-case manifest when `A2UI_CONFORMANCE_MANIFEST_DIR` (or +/// [manifestDir]) is set. +/// +/// A case executed in both [SuiteVersion]s is written once: `failed` if either +/// run failed, `xfailed` if either run was an expected failure, `passed` if at +/// least one run passed (keeping any v0.9 lowering skip [reason]), and +/// `skipped` otherwise. +void recordCase( + String suite, + String name, { + required String status, + String? reason, + SuiteVersion? version, + String? manifestDir, +}) { + final String? dir = + manifestDir ?? Platform.environment['A2UI_CONFORMANCE_MANIFEST_DIR']; + if (dir == null || dir.isEmpty) return; + + final Map<(String, String), _CaseEntry> entries = _recordsByDir.putIfAbsent( + dir, + () => {}, + ); + final _CaseEntry entry = entries.putIfAbsent((suite, name), _CaseEntry.new); + final String? normalizedReason = reason != null && reason.trim().isNotEmpty + ? reason.trim() + : null; + entry.outcomes[version] = _VersionOutcome(status, normalizedReason); + + Directory(dir).createSync(recursive: true); + final file = File('$dir/$harnessName-$pid-${Isolate.current.hashCode}.jsonl'); + final buffer = StringBuffer(); + for (final MapEntry<(String, String), _CaseEntry>( + key: (String recordSuite, String recordName), + value: _CaseEntry caseEntry, + ) + in entries.entries) { + final (String combinedStatus, String? combinedReason) = caseEntry + .combined(); + final record = { + 'harness': harnessName, + 'suite': recordSuite, + 'case': recordName, + 'status': combinedStatus, + 'reason': ?combinedReason, + }; + buffer.writeln(jsonEncode(record)); + } + file.writeAsStringSync(buffer.toString(), flush: true); +} + +/// Registers group-scoped `setUp` and `tearDown` hooks that record each test in +/// [caseNames] to the executed-case manifest for [suite]. +void registerSuiteManifestRecorder(String suite, Set caseNames) { + if (Zone.current[#test.declarer] == null || + Zone.current[#test.invoker] != null) { + return; + } + + String? skipReason; + StreamSubscription? messageSubscription; + + setUp(() { + skipReason = null; + final Object? invoker = Zone.current[#test.invoker]; + if (invoker == null) return; + // ignore: avoid_dynamic_calls + final Object? liveTest = (invoker as dynamic).liveTest; + // ignore: avoid_dynamic_calls + final messages = (liveTest as dynamic).onMessage as Stream; + messageSubscription = messages.listen((Object? message) { + // ignore: avoid_dynamic_calls + if ('${(message as dynamic).type}' == 'skip') { + // ignore: avoid_dynamic_calls + final text = (message as dynamic).text as String; + if (text.trim().isNotEmpty) { + skipReason = text.trim(); + } + } + }); + }); + + tearDown(() async { + await messageSubscription?.cancel(); + messageSubscription = null; + final Object? invoker = Zone.current[#test.invoker]; + if (invoker == null) return; + // ignore: avoid_dynamic_calls + final Object? liveTest = (invoker as dynamic).liveTest; + // ignore: avoid_dynamic_calls + final caseName = (liveTest as dynamic).individualName as String; + if (!caseNames.contains(caseName)) return; + + // ignore: avoid_dynamic_calls + final groups = (liveTest as dynamic).groups as List; + var groupName = ''; + if (groups.isNotEmpty) { + // ignore: avoid_dynamic_calls + groupName = (groups.last as dynamic).name as String; + } + final SuiteVersion? version = groupName.endsWith(SuiteVersion.v0_9.label) + ? SuiteVersion.v0_9 + : groupName.endsWith(SuiteVersion.v1_0.label) + ? SuiteVersion.v1_0 + : null; + + // ignore: avoid_dynamic_calls + final result = '${(liveTest as dynamic).state.result}'; + switch (result) { + case 'success': + recordCase(suite, caseName, status: 'passed', version: version); + case 'skipped': + recordCase( + suite, + caseName, + status: 'skipped', + reason: skipReason, + version: version, + ); + default: + // ignore: avoid_dynamic_calls + final errors = (liveTest as dynamic).errors as List; + String? failureReason; + if (errors.isNotEmpty) { + // ignore: avoid_dynamic_calls + failureReason = '${(errors.first as dynamic).error}'; + } + recordCase( + suite, + caseName, + status: 'failed', + reason: failureReason, + version: version, + ); + } + }); +} diff --git a/dart/a2ui_agent/test/conformance/multi_catalog_formats_conformance_test.dart b/dart/a2ui_agent/test/conformance/multi_catalog_formats_conformance_test.dart index f739a818ea..f8b56efadd 100644 --- a/dart/a2ui_agent/test/conformance/multi_catalog_formats_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/multi_catalog_formats_conformance_test.dart @@ -24,6 +24,30 @@ import 'suites.dart'; /// The suite is about a surface drawing on several catalogs, which only v1.0 /// can state: a component or function call names its own `catalogId`. So it /// runs as written in v1.0 only. +const Map> _knownKeys = { + 'case:compile': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + 'input', + }, + 'case:create_format': {...caseMetadataKeys, 'args', 'expect_error'}, + 'case:decompile': { + ...caseMetadataKeys, + 'args', + 'expect_absent', + 'expect_contains', + 'expect_round_trip', + 'messages', + }, + 'case:parse_response': {...caseMetadataKeys, 'args', 'expect', 'input'}, + 'args:compile': {'catalogs', 'format'}, + 'args:create_format': {'catalogs', 'format'}, + 'args:decompile': {'catalogs', 'format'}, + 'args:parse_response': {'catalogs', 'format'}, +}; + void main() { const path = 'agent/multi_catalog_formats.yaml'; const SuiteVersion version = SuiteVersion.v1_0; @@ -37,13 +61,21 @@ void main() { } void _runCase(Map testCase, SuiteVersion version) { + final action = testCase['action'] as String?; + final Set? allowedCaseKeys = _knownKeys['case:$action']; + if (allowedCaseKeys == null) { + throw UnsupportedError('Unknown action $action'); + } + assertKnownKeys(testCase, allowedCaseKeys, 'case:$action'); + final args = testCase['args']! as Map; + assertKnownKeys(args, _knownKeys['args:$action']!, 'args:$action'); final Object? error = testCase['expect_error']; if (error != null) { expectThrowsCategory(() => _perform(testCase, version), error); return; } final Object? result = _perform(testCase, version); - switch (testCase['action']) { + switch (action) { case 'decompile': _checkDecompiled(testCase, version, result! as String); default: diff --git a/dart/a2ui_agent/test/conformance/request_processor_conformance_test.dart b/dart/a2ui_agent/test/conformance/request_processor_conformance_test.dart index 8688d2a4ac..da7df1e910 100644 --- a/dart/a2ui_agent/test/conformance/request_processor_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/request_processor_conformance_test.dart @@ -24,25 +24,70 @@ import 'suites.dart'; /// /// Lowered to v0.9, capabilities, examples and responses are lowered, and /// parsed messages lifted back, as `suites.dart` describes. +const Map> _knownKeys = { + 'case:create_format': {...caseMetadataKeys, 'args', 'expect', 'expect_error'}, + 'case:create_processor': { + ...caseMetadataKeys, + 'args', + 'expect', + 'expect_error', + 'then_parse', + }, + 'args:create_format': {'catalogs', 'examples', 'format', 'probe_chunks'}, + 'args:create_processor': { + 'catalogs', + 'examples', + 'format', + 'format_override', + 'renderer_capabilities', + }, + 'expect:create_format': { + 'parser_state_isolated', + 'parsers_are_distinct', + 'prompt_snippet_absent', + 'prompt_snippet_contains', + }, + 'expect:create_processor': { + 'active_catalog_ids', + 'catalogs', + 'generator_catalogs_unchanged', + 'prompt_snippet_absent', + 'prompt_snippet_contains', + }, + 'then_parse': {'expect', 'expect_error', 'input'}, +}; + void main() { const path = 'agent/request_processor.yaml'; + final allCaseNames = {}; for (final SuiteVersion version in SuiteVersion.values) { group('conformance $path ${version.label}', () { final List> cases = loadSuite(path); test('suite is not empty', () => expect(cases, isNotEmpty)); for (final testCase in cases) { - test( - testCase['name']! as String, - skip: _skipReason(testCase, version), - () => switch (testCase['action']) { - 'create_format' => _createFormat(testCase, version), - 'create_processor' => _createProcessor(testCase, version), - _ => throw UnsupportedError('Unknown action ${testCase['action']}'), - }, - ); + final name = testCase['name']! as String; + allCaseNames.add(name); + test(name, () { + if (_skipReason(testCase, version) case final String reason) { + markTestSkipped(reason); + return; + } + switch (testCase['action']) { + case 'create_format': + _createFormat(testCase, version); + case 'create_processor': + _createProcessor(testCase, version); + default: + throw UnsupportedError('Unknown action ${testCase['action']}'); + } + }); } }); } + verifyNoUnusedEntries('request_processor', allCaseNames, [ + ..._v1OnlyCases.keys, + 'test_express_create_format_rejects_several_v0_9_catalogs', + ]); } /// Cases v0.9 cannot state, with the reason for each. @@ -91,7 +136,17 @@ List> _examples( ]; void _createFormat(Map testCase, SuiteVersion version) { + assertKnownKeys( + testCase, + _knownKeys['case:create_format']!, + 'case:create_format', + ); final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:create_format']!, + 'args:create_format', + ); InferenceFormat create() => _factory( args['format'], ).createFormat(version.catalogs(args), examples: _examples(args, version)); @@ -101,6 +156,11 @@ void _createFormat(Map testCase, SuiteVersion version) { } final InferenceFormat format = create(); final expected = testCase['expect']! as Map; + assertKnownKeys( + expected, + _knownKeys['expect:create_format']!, + 'expect:create_format', + ); expectSnippet(format.promptGenerator.generate(), expected); if (expected['parsers_are_distinct'] == true) { expect(format.createParser(), isNot(same(format.createParser()))); @@ -126,7 +186,17 @@ void _createFormat(Map testCase, SuiteVersion version) { } void _createProcessor(Map testCase, SuiteVersion version) { + assertKnownKeys( + testCase, + _knownKeys['case:create_processor']!, + 'case:create_processor', + ); final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:create_processor']!, + 'args:create_processor', + ); final List configs = [ for (final Object? entry in args['catalogs']! as List) version.catalogConfig(entry), @@ -157,6 +227,11 @@ void _createProcessor(Map testCase, SuiteVersion version) { } final A2uiRequestProcessor processor = create(); final expected = testCase['expect']! as Map; + assertKnownKeys( + expected, + _knownKeys['expect:create_processor']!, + 'expect:create_processor', + ); if (expected['active_catalog_ids'] case final List ids) { expect(processor.activeCatalogs.map((c) => c.id), ids); } @@ -173,6 +248,7 @@ void _createProcessor(Map testCase, SuiteVersion version) { } if (testCase['then_parse'] case final Map parse) { + assertKnownKeys(parse, _knownKeys['then_parse']!, 'then_parse'); List parseResponse() => version.partsJson( processor.parseResponse(version.text(parse['input']! as String)), ); diff --git a/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart b/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart index 05bbddc0f2..b1d36cf87f 100644 --- a/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/scaling_conformance_test.dart @@ -32,6 +32,30 @@ import 'package:test/test.dart'; import 'suites.dart'; +const Map> _knownKeys = { + 'case:measure_growth': { + ...caseMetadataKeys, + 'args', + 'chunk_size', + 'max_ratio', + 'operation', + 'payload', + 'sizes', + }, + 'case:measure_overhead': { + ...caseMetadataKeys, + 'args', + 'baseline', + 'chunk_size', + 'max_ratio', + 'operation', + 'payload', + 'size', + }, + 'args:measure_growth': {'catalog', 'format', 'progressive_keys'}, + 'args:measure_overhead': {'catalog', 'format', 'progressive_keys'}, +}; + void main() { for (final path in const [ 'agent/direct_json/scaling.yaml', @@ -48,13 +72,19 @@ void main() { } void _runCase(Map testCase) { - final InferenceFormat format = _format( - testCase['args']! as Map, - ); + final action = testCase['action'] as String?; + final Set? allowedCaseKeys = _knownKeys['case:$action']; + if (allowedCaseKeys == null) { + throw UnsupportedError('Unknown action $action'); + } + assertKnownKeys(testCase, allowedCaseKeys, 'case:$action'); + final args = testCase['args']! as Map; + assertKnownKeys(args, _knownKeys['args:$action']!, 'args:$action'); + final InferenceFormat format = _format(args); final operation = testCase['operation']! as String; final chunkSize = testCase['chunk_size'] as int?; final maxRatio = testCase['max_ratio']! as num; - switch (testCase['action']) { + switch (action) { case 'measure_growth': final [int smaller, int larger] = (testCase['sizes']! as List) .cast(); @@ -73,7 +103,7 @@ void _runCase(Map testCase) { what: '$operation against ${testCase['baseline']}', ); default: - throw UnsupportedError('Unknown action ${testCase['action']}'); + throw UnsupportedError('Unknown action $action'); } } diff --git a/dart/a2ui_agent/test/conformance/streamed_answer_conformance_test.dart b/dart/a2ui_agent/test/conformance/streamed_answer_conformance_test.dart index 4ba80eab91..2c571f9c1b 100644 --- a/dart/a2ui_agent/test/conformance/streamed_answer_conformance_test.dart +++ b/dart/a2ui_agent/test/conformance/streamed_answer_conformance_test.dart @@ -37,8 +37,35 @@ void main() { } } +const Map> _knownKeys = { + 'case:stream_answer': { + ...caseMetadataKeys, + 'args', + 'expect_close', + 'expect_error', + 'steps', + }, + 'args:stream_answer': {'catalog', 'format', 'renderer_catalogs'}, + 'step:stream_answer': {'expect_in_block', 'expect_parts', 'input'}, +}; + void _runCase(Map testCase, SuiteVersion version) { + switch (testCase['action']) { + case 'stream_answer': + assertKnownKeys( + testCase, + _knownKeys['case:stream_answer']!, + 'case:stream_answer', + ); + default: + throw UnsupportedError('Unknown action ${testCase['action']}'); + } final args = testCase['args']! as Map; + assertKnownKeys( + args, + _knownKeys['args:stream_answer']!, + 'args:stream_answer', + ); expect(args['format'], 'direct_json'); final processor = A2uiRequestProcessor( activeCatalogs: version.catalogs(args), @@ -67,6 +94,11 @@ void _runCase(Map testCase, SuiteVersion version) { for (final (int i, Object? step) in (testCase['steps']! as List).indexed) { final stepCase = step! as Map; + assertKnownKeys( + stepCase, + _knownKeys['step:stream_answer']!, + 'step:stream_answer', + ); final input = stepCase['input']! as String; answer.feed(version.text(input, catalogId: catalogId)); expect( diff --git a/dart/a2ui_agent/test/conformance/suites.dart b/dart/a2ui_agent/test/conformance/suites.dart index 4791d62af6..9ae3b08fab 100644 --- a/dart/a2ui_agent/test/conformance/suites.dart +++ b/dart/a2ui_agent/test/conformance/suites.dart @@ -50,14 +50,90 @@ import 'package:a2ui_core/a2ui_core.dart'; import 'package:test/test.dart'; import 'package:yaml/yaml.dart'; +import 'manifest.dart'; + +export 'manifest.dart' show harnessName, recordCase; + +/// Common metadata keys allowed at the top level of every conformance case. +const Set caseMetadataKeys = { + 'action', + 'description', + 'features', + 'name', +}; + +/// Keys recognized inside an `expect_error` block by [expectThrowsCategory]. +const Set knownExpectErrorKeys = {'category', 'message'}; + +/// Keys recognized inside a catalog expectation map by +/// [SuiteVersion.expectCatalog]. +const Set knownCatalogExpectKeys = { + 'catalog_id', + 'components', + 'functions', + 'protocol_version', +}; + +/// Fails the current test if [obj] contains any keys not in [allowed]. +void assertKnownKeys( + Map obj, + Set allowed, + String where, +) { + final List unknown = + obj.keys.where((k) => !allowed.contains(k)).toList()..sort(); + if (unknown.isNotEmpty) { + fail('Unhandled keys at $where: $unknown'); + } +} + +/// Registers a test that fails if any key in [entries] is absent from +/// [knownCaseNames]. +void verifyNoUnusedEntries( + String label, + Set knownCaseNames, + Iterable entries, +) { + final List unused = + entries.where((k) => !knownCaseNames.contains(k)).toList()..sort(); + if (unused.isNotEmpty) { + test('$label: no unused skip entries', () { + fail('Unused skip entries in $label: $unused'); + }); + } +} + /// Reads the cases of the suite at [path], relative to `conformance/`, as -/// plain Dart maps. -List> loadSuite(String path) => [ - for (final Object? node - in loadYaml(File('$conformanceRoot/$path').readAsStringSync()) - as YamlList) - _plain(node)! as Map, -]; +/// plain Dart maps, and registers a `tearDown` recorder when called inside a +/// test group. +List> loadSuite(String path) { + final String text = File('$conformanceRoot/$path').readAsStringSync(); + Object? parsed; + try { + parsed = loadYaml(text); + } on YamlException { + parsed = loadYaml( + text.replaceAll( + RegExp(r'\\ud[89a-f][0-9a-f]{2}', caseSensitive: false), + r'\ufffd', + ), + ); + } + final YamlList list = switch (parsed) { + final YamlList items => items, + final YamlMap map => map['tests']! as YamlList, + _ => throw StateError('Unsupported suite YAML structure in $path.'), + }; + final List> cases = [ + for (final Object? node in list) _plain(node)! as Map, + ]; + final Set caseNames = { + for (final Map testCase in cases) + (testCase['id'] ?? testCase['name'])! as String, + }; + registerSuiteManifestRecorder(path, caseNames); + return cases; +} /// The protocol version a harness runs a suite's cases in. enum SuiteVersion { @@ -133,6 +209,7 @@ enum SuiteVersion { /// Checks the catalog-level assertions of [expected] against [catalog]; /// see [expectCatalog]. As written, the version is checked too. void expectCatalog(CatalogApi catalog, Map expected) { + assertKnownKeys(expected, knownCatalogExpectKeys, 'expect.catalog'); _expectCatalog(catalog, expected); if (expected['protocol_version'] case final String version when !_lowers) { expect(catalog.protocolVersion?.jsonValue, version); @@ -158,15 +235,32 @@ final Map _errors = { 'CatalogError': isA(), }; -/// Expects [body] to throw the error category of a case's `expect_error`. +/// Expects [body] to throw the error category (and optional message pattern) +/// of a case's `expect_error`. void expectThrowsCategory( void Function() body, Object? expectError, { String? reason, }) { - final category = - (expectError! as Map)['category']! as String; - expect(body, throwsA(_errors[category]), reason: reason); + final error = expectError! as Map; + assertKnownKeys(error, knownExpectErrorKeys, 'expect_error'); + final category = error['category']! as String; + final Matcher? categoryMatcher = _errors[category]; + if (categoryMatcher == null) { + fail('Unknown error category: $category'); + } + final Matcher matcher = switch (error['message']) { + final String pattern => allOf( + categoryMatcher, + isA().having( + (e) => e.message, + 'message', + matches(RegExp(pattern, caseSensitive: false)), + ), + ), + _ => categoryMatcher, + }; + expect(body, throwsA(matcher), reason: reason); } /// The catalog a suite names by [entry]: a path relative to `conformance/`, diff --git a/dart/a2ui_agent/test/conformance/unsupported_suites_test.dart b/dart/a2ui_agent/test/conformance/unsupported_suites_test.dart new file mode 100644 index 0000000000..53cd13ba53 --- /dev/null +++ b/dart/a2ui_agent/test/conformance/unsupported_suites_test.dart @@ -0,0 +1,211 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:test/test.dart'; + +import 'suites.dart'; + +/// Conformance suites executed in the default test run by the per-suite +/// `*_conformance_test.dart` files. +const Set activeAgentSuites = { + 'agent/catalog_provider.yaml', + 'agent/catalog_resolution.yaml', + 'agent/catalog_transformer.yaml', + 'agent/direct_json/compiler.yaml', + 'agent/direct_json/decompiler.yaml', + 'agent/direct_json/prompt_generator.yaml', + 'agent/direct_json/response_parser.yaml', + 'agent/direct_json/response_streaming.yaml', + 'agent/express/compiler.yaml', + 'agent/express/decompiler.yaml', + 'agent/express/prompt_generator.yaml', + 'agent/express/response_parser.yaml', + 'agent/express/response_streaming.yaml', + 'agent/multi_catalog_formats.yaml', + 'agent/request_processor.yaml', + 'agent/streamed_answer.yaml', +}; + +/// `agent/**/*.yaml` suites not executed in the default `dart-agent` test run, +/// paired with the reason recorded in the executed-case manifest. +const Map unsupportedAgentSuites = { + 'agent/builder/builder.yaml': + 'The fluent builder AST suite targets the Python builder API, which ' + 'a2ui_agent for Dart does not implement.', + 'agent/direct_json/scaling.yaml': + 'Timing suite tagged performance; runs in the performance job with ' + 'dart test -P performance.', + 'agent/express/scaling.yaml': + 'Timing suite tagged performance; runs in the performance job with ' + 'dart test -P performance.', + 'agent/legacy/inference_format.yaml': + 'Pre-blueprint legacy suite retained only for backward compatibility; ' + 'superseded by the current agent suites.', + 'agent/legacy/parser.yaml': + 'Pre-blueprint legacy suite retained only for backward compatibility; ' + 'superseded by the current agent suites.', + 'agent/legacy/streaming_parser.yaml': + 'Pre-blueprint legacy suite retained only for backward compatibility; ' + 'superseded by the current agent suites.', + 'agent/macros/macros.yaml': + 'The composite macro suite targets the Python macro API, which ' + 'a2ui_agent for Dart does not implement.', + 'agent/skill.yaml': + 'a2ui_agent for Dart does not implement the skill generation API.', +}; + +Set _discoverAgentSuites() { + final agentDir = Directory('$conformanceRoot/agent'); + final suites = {}; + for (final FileSystemEntity entity in agentDir.listSync(recursive: true)) { + if (entity is File && entity.path.endsWith('.yaml')) { + final String relative = entity.path + .substring(conformanceRoot.length + 1) + .replaceAll(r'\', '/'); + suites.add(relative); + } + } + return suites; +} + +void main() { + group('agent suite coverage and unsupported suites', () { + test('accounts for every conformance/agent/**/*.yaml file', () { + final Set onDisk = _discoverAgentSuites(); + final Set unsupported = unsupportedAgentSuites.keys.toSet(); + expect( + activeAgentSuites.intersection(unsupported), + isEmpty, + reason: + 'A suite cannot be in both activeAgentSuites and ' + 'unsupportedAgentSuites.', + ); + expect( + activeAgentSuites.union(unsupported), + equals(onDisk), + reason: + 'Every conformance/agent/**/*.yaml file must be in ' + 'activeAgentSuites or unsupportedAgentSuites.', + ); + }); + + for (final MapEntry(:String key, :String value) + in unsupportedAgentSuites.entries) { + test('$key records skipped manifest entries with a reason', () { + final String reason = value.trim(); + expect(reason, isNotEmpty); + final List> cases = loadSuite(key); + expect(cases, isNotEmpty); + for (final testCase in cases) { + final name = (testCase['id'] ?? testCase['name'])! as String; + recordCase(key, name, status: 'skipped', reason: reason); + } + }); + } + }); + + group('recordCase', () { + test( + 'merges outcomes across SuiteVersions and preserves v0.9 skip reasons', + () { + final Directory tempDir = Directory.systemTemp.createTempSync( + 'a2ui_agent_manifest_test_', + ); + addTearDown(() => tempDir.deleteSync(recursive: true)); + + recordCase( + 'agent/sample.yaml', + 'case_both_pass', + status: 'passed', + version: SuiteVersion.v0_9, + manifestDir: tempDir.path, + ); + recordCase( + 'agent/sample.yaml', + 'case_both_pass', + status: 'passed', + version: SuiteVersion.v1_0, + manifestDir: tempDir.path, + ); + + recordCase( + 'agent/sample.yaml', + 'case_v09_skipped_v10_passed', + status: 'skipped', + reason: 'v0.9 cannot state what the case carries.', + version: SuiteVersion.v0_9, + manifestDir: tempDir.path, + ); + recordCase( + 'agent/sample.yaml', + 'case_v09_skipped_v10_passed', + status: 'passed', + version: SuiteVersion.v1_0, + manifestDir: tempDir.path, + ); + + recordCase( + 'agent/sample.yaml', + 'case_either_failed', + status: 'passed', + version: SuiteVersion.v0_9, + manifestDir: tempDir.path, + ); + recordCase( + 'agent/sample.yaml', + 'case_either_failed', + status: 'failed', + reason: 'Expected 1, got 2.', + version: SuiteVersion.v1_0, + manifestDir: tempDir.path, + ); + + final List files = tempDir.listSync().whereType().toList(); + expect(files, hasLength(1)); + final List> records = files.single + .readAsLinesSync() + .map((line) => jsonDecode(line) as Map) + .toList(); + expect( + records, + equals([ + { + 'harness': 'dart-agent', + 'suite': 'agent/sample.yaml', + 'case': 'case_both_pass', + 'status': 'passed', + }, + { + 'harness': 'dart-agent', + 'suite': 'agent/sample.yaml', + 'case': 'case_v09_skipped_v10_passed', + 'status': 'passed', + 'reason': 'v0.9 cannot state what the case carries.', + }, + { + 'harness': 'dart-agent', + 'suite': 'agent/sample.yaml', + 'case': 'case_either_failed', + 'status': 'failed', + 'reason': 'Expected 1, got 2.', + }, + ]), + ); + }, + ); + }); +} From 98a897ff9541e330dfe37e24b601edc01e674525 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:50:33 +0000 Subject: [PATCH 22/48] test(conformance): address review findings for agent and extension harnesses - Remove duplicate recordCase call from skipVerdict in the TypeScript agent conformance loader and drop the redundant recordedSkipped deduplication set in manifest.ts. - Assert context.add_activated_extension invocations in the Python A2A try_activate_a2ui_extension conformance runner. - Document the Python A2A and ADK extension conformance harnesses in the harness implementation guide. --- .../conformance_harness_implementation_guide.md | 3 +++ .../tests/conformance/test_a2a_integration.py | 4 ++++ .../agents/a2ui_agent/tests/conformance/loader.ts | 15 +++++---------- .../a2ui_agent/tests/conformance/manifest.ts | 9 --------- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index c829bf829c..b1b2bf815f 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -401,3 +401,6 @@ Inspect these existing implementations for concrete code examples: 3. Angular (Karma and Jasmine browser runner): [typescript/renderers/angular/src/adapter_conformance.spec.ts](../typescript/renderers/angular/src/adapter_conformance.spec.ts) and [typescript/renderers/angular/karma.conf.js](../typescript/renderers/angular/karma.conf.js) 4. Flutter (`flutter_test` widget and `SemanticsNode` tree checks): [dart/a2ui_flutter/test/adapter_conformance_test.dart](../dart/a2ui_flutter/test/adapter_conformance_test.dart) 5. SwiftUI (`Testing` on `Surface`, `SurfaceViewModel`, and `A2UIAccessibilityModifier`): [swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift](../swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift) +6. Extension harnesses (`conformance/extensions/`): + 1. Python A2A (`a2a_integration.yaml`): [python/a2ui_a2a/tests/conformance/test_a2a_integration.py](../python/a2ui_a2a/tests/conformance/test_a2a_integration.py) and [python/a2ui_a2a/tests/conftest.py](../python/a2ui_a2a/tests/conftest.py) + 2. Python ADK (`adk_extensions.yaml`): [python/a2ui_adk/tests/conformance/test_adk_extensions.py](../python/a2ui_adk/tests/conformance/test_adk_extensions.py) and [python/a2ui_adk/tests/conftest.py](../python/a2ui_adk/tests/conftest.py) diff --git a/python/a2ui_a2a/tests/conformance/test_a2a_integration.py b/python/a2ui_a2a/tests/conformance/test_a2a_integration.py index 416977f368..79ba3e5aac 100644 --- a/python/a2ui_a2a/tests/conformance/test_a2a_integration.py +++ b/python/a2ui_a2a/tests/conformance/test_a2a_integration.py @@ -162,6 +162,10 @@ def _run_try_activate_extension(test_case: dict[str, Any]) -> None: result_version = try_activate_a2ui_extension(context, card) assert (result_version is not None) == expect + if expect: + context.add_activated_extension.assert_called_once() + else: + context.add_activated_extension.assert_not_called() def _run_get_extension(test_case: dict[str, Any]) -> None: diff --git a/typescript/agents/a2ui_agent/tests/conformance/loader.ts b/typescript/agents/a2ui_agent/tests/conformance/loader.ts index 6089cb34ce..122b8cbff3 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/loader.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/loader.ts @@ -20,8 +20,6 @@ import {fileURLToPath} from 'url'; import {loadAll} from 'js-yaml'; -import {recordCase} from './manifest.js'; - // Using __dirname isn't directly available in ESM unless constructed const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -121,22 +119,19 @@ const SUPPORTED_FORMATS = new Set(['direct_json', 'express']); */ const UNIMPLEMENTED_ACTIONS = new Map([]); -function skipVerdict(testCase: TestCase, reason: string) { - if ('sourceFile' in testCase && typeof testCase.sourceFile === 'string') { - recordCase(testCase.sourceFile, testCase.name, 'skipped', reason); - } +function skipVerdict(reason: string) { return {runnable: false, reason}; } export function classify(testCase: TestCase) { const superseded = SUPERSEDED.get(testCase.name) ?? SUPERSEDED_ACTIONS.get(testCase.action); if (superseded) { - return skipVerdict(testCase, superseded); + return skipVerdict(superseded); } const unimplemented = UNIMPLEMENTED_ACTIONS.get(testCase.action); if (unimplemented) { - return skipVerdict(testCase, unimplemented); + return skipVerdict(unimplemented); } let version = 'unversioned'; @@ -158,10 +153,10 @@ export function classify(testCase: TestCase) { } if (version !== 'unversioned' && !SUPPORTED_PROTOCOL_VERSIONS.has(version)) { - return skipVerdict(testCase, `Unsupported protocol version: ${version}`); + return skipVerdict(`Unsupported protocol version: ${version}`); } if (!SUPPORTED_FORMATS.has(format)) { - return skipVerdict(testCase, `Unsupported format: ${format}`); + return skipVerdict(`Unsupported format: ${format}`); } return {runnable: true, reason: ''}; diff --git a/typescript/agents/a2ui_agent/tests/conformance/manifest.ts b/typescript/agents/a2ui_agent/tests/conformance/manifest.ts index 24226928d4..7ec04315a9 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/manifest.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/manifest.ts @@ -33,8 +33,6 @@ export function setActiveCase(current: ActiveCase | undefined): void { activeCase = current; } -const recordedSkipped = new Set(); - /** * Appends one executed-case manifest entry for the TypeScript agent harness when * `A2UI_CONFORMANCE_MANIFEST_DIR` is set. @@ -49,13 +47,6 @@ export function recordCase( if (!manifestDir) { return; } - if (status === 'skipped') { - const key = `${suite}::${testCase}`; - if (recordedSkipped.has(key)) { - return; - } - recordedSkipped.add(key); - } mkdirSync(manifestDir, {recursive: true}); const record: Record = { harness: 'ts-agent', From 2ae0ebc6fa56d252c21526f0cedb9fbf82f91b86 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 06:28:15 +0000 Subject: [PATCH 23/48] test(conformance): strict dispatch and executed-case manifest in renderer runners --- .github/workflows/flutter_packages_test.yml | 12 +- .github/workflows/swift_build_and_test.yml | 6 + .github/workflows/web_ci.yml | 13 + .../test/adapter_conformance_test.dart | 758 +--------- dart/a2ui_flutter/test/adapter_harness.dart | 1047 ++++++++++++++ .../AdapterConformanceTests.swift | 783 ++--------- .../A2UISwiftUITests/AdapterHarness.swift | 1222 +++++++++++++++++ .../accessibility_conformance.test.ts | 193 ++- typescript/renderers/angular/karma.conf.js | 69 +- .../angular/src/adapter_conformance.spec.ts | 873 ++---------- .../renderers/angular/src/adapter_harness.ts | 1017 ++++++++++++++ .../renderers/angular/tsconfig.lib.json | 2 +- .../lit/src/tests/adapter_conformance.test.ts | 735 ++-------- .../lit/src/tests/adapter_harness.ts | 958 +++++++++++++ .../react/tests/adapter_conformance.test.tsx | 827 ++--------- .../renderers/react/tests/adapter_harness.tsx | 980 +++++++++++++ 16 files changed, 5977 insertions(+), 3518 deletions(-) create mode 100644 dart/a2ui_flutter/test/adapter_harness.dart create mode 100644 swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift create mode 100644 typescript/renderers/angular/src/adapter_harness.ts create mode 100644 typescript/renderers/lit/src/tests/adapter_harness.ts create mode 100644 typescript/renderers/react/tests/adapter_harness.tsx diff --git a/.github/workflows/flutter_packages_test.yml b/.github/workflows/flutter_packages_test.yml index ad74816826..6ddd27c918 100644 --- a/.github/workflows/flutter_packages_test.yml +++ b/.github/workflows/flutter_packages_test.yml @@ -131,7 +131,7 @@ jobs: - name: Run tests working-directory: ${{ matrix.package.path }} env: - A2UI_CONFORMANCE_MANIFEST_DIR: ${{ matrix.package.path == 'dart/a2ui_core' && format('{0}/a2ui-manifest', runner.temp) || '' }} + A2UI_CONFORMANCE_MANIFEST_DIR: ${{ (matrix.package.path == 'dart/a2ui_core' || matrix.package.path == 'dart/a2ui_flutter') && format('{0}/a2ui-manifest', runner.temp) || '' }} run: | if [ -d "test" ]; then flutter test --test-randomize-ordering-seed=random @@ -149,6 +149,16 @@ jobs: --harness dart-core \ --corpus core \ --exclude "core/message_processor_scaling.yaml=performance: runs with -P performance" + - name: Set up Python for adapter conformance manifest check + if: matrix.package.path == 'dart/a2ui_flutter' + uses: ./.github/actions/setup-python + - name: Check executed adapter conformance cases + if: matrix.package.path == 'dart/a2ui_flutter' + run: | + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --harness flutter \ + --corpus adapter # The e2e_test packages are excluded from the matrix above because their # tests call a live Gemini model, which needs credentials and quota that are diff --git a/.github/workflows/swift_build_and_test.yml b/.github/workflows/swift_build_and_test.yml index d7ec4efceb..a7a06682c0 100644 --- a/.github/workflows/swift_build_and_test.yml +++ b/.github/workflows/swift_build_and_test.yml @@ -66,6 +66,12 @@ jobs: --harness swift-core \ --corpus core \ --exclude "core/message_processor_scaling.yaml=performance: runs in the test (performance) job with A2UI_RUN_TIMING=1" + - name: Check executed adapter conformance cases + run: | + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --harness swiftui \ + --corpus adapter ios-build: # `swift test` above builds for macOS only, so `#if os(iOS)` and UIKit code diff --git a/.github/workflows/web_ci.yml b/.github/workflows/web_ci.yml index e71af26873..d42d4add1a 100644 --- a/.github/workflows/web_ci.yml +++ b/.github/workflows/web_ci.yml @@ -122,6 +122,19 @@ jobs: - uses: ./.github/actions/setup-python - name: Check web_core executed cases run: uv run --project conformance python conformance/scripts/check_executed_cases.py --manifest-dir "${{ runner.temp }}/a2ui-manifest" --harness web_core --corpus core + - name: Check adapter executed cases + run: | + for h in lit react angular; do + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --corpus adapter \ + --harness "$h" + done + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --corpus adapter \ + --harness web_core_universal \ + --only core/accessibility.yaml - name: Test React 18 Compatibility run: yarn workspace @a2ui/react run test:react18 - name: Verify NPM Publish Script diff --git a/dart/a2ui_flutter/test/adapter_conformance_test.dart b/dart/a2ui_flutter/test/adapter_conformance_test.dart index 0be6bd6580..80b04c56a5 100644 --- a/dart/a2ui_flutter/test/adapter_conformance_test.dart +++ b/dart/a2ui_flutter/test/adapter_conformance_test.dart @@ -12,35 +12,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'dart:io'; -import 'dart:ui' show CheckedState; - -import 'package:a2ui_core/a2ui_core.dart'; -import 'package:a2ui_flutter/a2ui_flutter.dart'; -import 'package:a2ui_flutter/basic_catalog.dart'; -import 'package:a2ui_flutter/v0_9/basic_catalog.dart' as v0_9; -// ignore: deprecated_member_use, see https://github.com/a2ui-project/a2ui/issues/3235 -import 'package:flutter/material.dart'; -import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:json_schema_builder/json_schema_builder.dart' - hide ValidationResult; -import 'package:yaml/yaml.dart'; + +import 'adapter_harness.dart'; void main() { - final List accessibilityTests = _loadYamlTests( + final List accessibilityTests = loadYamlTests( '../../conformance/core/accessibility.yaml', ); - final List adapterTests = _loadYamlTests( + final List adapterTests = loadYamlTests( '../../conformance/core/adapter_conformance.yaml', ); + tearDownAll(assertAllUnsupportedMatched); + group('Accessibility Conformance (accessibility.yaml)', () { for (final entry in accessibilityTests) { final testCase = entry! as Map; final name = testCase['name']! as String; testWidgets(name, (tester) async { - await _runAccessibilityCase(tester, testCase); + var status = 'failed'; + String? reason; + addTearDown(() { + writeManifestRecord({ + 'harness': 'flutter', + 'suite': 'core/accessibility.yaml', + 'case': name, + 'status': status, + 'reason': ?reason, + }); + }); + try { + final String? skipReason = findUnsupportedReason( + name, + (testCase['steps'] as List?) ?? const [], + ); + if (skipReason != null) { + status = 'skipped'; + reason = skipReason; + return; + } + await runAccessibilityCase(tester, testCase); + status = 'passed'; + } catch (error) { + status = 'failed'; + reason = error.toString(); + rethrow; + } }); } }); @@ -50,687 +68,35 @@ void main() { final testCase = entry! as Map; final name = testCase['name']! as String; testWidgets(name, (tester) async { - await _runAdapterCase(tester, testCase); + var status = 'failed'; + String? reason; + addTearDown(() { + writeManifestRecord({ + 'harness': 'flutter', + 'suite': 'core/adapter_conformance.yaml', + 'case': name, + 'status': status, + 'reason': ?reason, + }); + }); + try { + final String? skipReason = findUnsupportedReason( + name, + (testCase['steps'] as List?) ?? const [], + ); + if (skipReason != null) { + status = 'skipped'; + reason = skipReason; + return; + } + await runAdapterCase(tester, testCase); + status = 'passed'; + } catch (error) { + status = 'failed'; + reason = error.toString(); + rethrow; + } }); } }); } - -List _loadYamlTests(String relativePath) { - final String source = File(relativePath).readAsStringSync(); - final Object? parsed = _yamlToDart(loadYaml(source)); - if (parsed is List) return parsed; - return (parsed! as Map)['tests']! as List; -} - -Object? _yamlToDart(Object? node) { - if (node is YamlMap) { - return { - for (final MapEntry entry in node.entries) - entry.key.toString(): _yamlToDart(entry.value), - }; - } - if (node is YamlList) { - return [for (final Object? item in node) _yamlToDart(item)]; - } - return node; -} - -CatalogImplementation _buildV10BasicCatalog({ - required Map> renderedNodes, - List? openedUrls, -}) { - final Catalog v10Api = - BasicCatalog.v1_0(openUrl: (uri) => openedUrls?.add(uri.toString())); - final Map v09ByName = { - for (final ComponentImplementation impl in BasicComponents.all) - impl.name: impl, - }; - final components = [ - for (final MapEntry entry - in v10Api.components.entries) - if (v09ByName[entry.key] case final ComponentImplementation impl) - ComponentImplementation( - name: entry.key, - schema: entry.value.schema, - builder: (context, node, props, buildChild) { - renderedNodes[node.instanceId] = node; - return impl.builder(context, node, props, buildChild); - }, - ), - ]; - return CatalogImplementation( - id: BasicCatalog.v1_0Id, - components: components, - functions: [...v10Api.functions.values, _IncrementCounterFunction()], - themeSchema: v10Api.themeSchema, - schemaId: v10Api.schemaId, - title: v10Api.title, - description: v10Api.description, - protocolVersion: v10Api.protocolVersion, - instructions: v10Api.instructions, - definitions: v10Api.definitions, - ); -} - -CatalogImplementation _buildCustomStatusBadgeCatalog( - Map> renderedNodes, -) { - final statusBadge = ComponentImplementation( - name: 'StatusBadge', - schema: Schema.fromMap({ - 'type': 'object', - 'properties': { - 'component': {'const': 'StatusBadge'}, - 'badgeText': { - 'oneOf': [ - {'type': 'string'}, - { - 'type': 'object', - 'properties': { - '@path': {'type': 'string'}, - }, - 'required': ['@path'], - }, - ], - }, - }, - 'required': ['component', 'badgeText'], - }), - builder: (context, node, props, buildChild) { - renderedNodes[node.instanceId] = node; - return Text(props.string('badgeText') ?? ''); - }, - ); - return CatalogImplementation( - id: 'https://example.com/catalogs/badges/v1/catalog.json', - components: [statusBadge], - protocolVersion: A2uiProtocolVersion.v1_0, - ); -} - -class _IncrementCounterFunction extends FunctionImplementation { - _IncrementCounterFunction() - : super( - name: 'incrementCounter', - argumentSchema: Schema.fromMap({'type': 'object'}), - ); - - @override - Object? execute( - Map args, - DataContext context, [ - CancellationSignal? cancellationSignal, - ]) { - final path = args['path']! as String; - final num step = (args['step'] as num?) ?? 1; - final num current = (context.dataModel.get(path) as num?) ?? 0; - context.dataModel.set(path, current + step); - return null; - } -} - -class _ConformanceHarness { - _ConformanceHarness() { - final CatalogImplementation v10Catalog = _buildV10BasicCatalog( - renderedNodes: renderedNodes, - openedUrls: openedUrls, - ); - final CatalogImplementation customCatalog = _buildCustomStatusBadgeCatalog( - renderedNodes, - ); - processor = MessageProcessor( - catalogs: [v0_9.basicCatalog(), v10Catalog, customCatalog], - defaultVersion: A2uiProtocolVersion.v1_0, - ); - processor.model.onAction.subscribe(actions.add); - processor.model.onSurfaceCreated.subscribe((created) { - surface = created; - created.onError.subscribe(errors.add); - }); - } - - late final MessageProcessor processor; - SurfaceModel? surface; - final Map> renderedNodes = {}; - final List actions = []; - final List errors = []; - final List openedUrls = []; - final Map savedElements = {}; - final Map> savedNodes = {}; - bool mounted = false; - - ComponentNode? get rootNode => - renderedNodes[surface?.rootId ?? 'root']; - - Future processMessages( - WidgetTester tester, - List rawMessages, - ) async { - final messages = >[ - for (final Object? raw in rawMessages) - Map.from(raw! as Map), - ]; - processor.processMessages( - AgentToRendererMessagePayload.fromJson( - messages, - protocolVersion: A2uiProtocolVersion.v1_0, - ), - ); - if (!mounted && surface != null) { - mounted = true; - await tester.pumpWidget( - MaterialApp( - home: Scaffold( - body: SingleChildScrollView(child: A2uiSurface(surface: surface!)), - ), - ), - ); - } else { - await tester.pump(); - } - } - - void dispose() { - processor.model.dispose(); - } -} - -Future _runAccessibilityCase( - WidgetTester tester, - Map testCase, -) async { - final SemanticsHandle semantics = tester.ensureSemantics(); - final harness = _ConformanceHarness(); - try { - final steps = testCase['steps'] as List?; - if (steps != null) { - for (final Object? rawStep in steps) { - final step = rawStep! as Map; - final stepMessages = step['messages']! as List; - await harness.processMessages(tester, stepMessages); - final stepAssertions = step['assertions']! as Map; - final stepTree = - stepAssertions['accessibilityTree']! as Map; - _assertAccessibilityTree(tester, stepTree); - } - } else { - final messages = testCase['messages']! as List; - await harness.processMessages(tester, messages); - final assertions = testCase['assertions']! as Map; - final expectTree = - assertions['accessibilityTree']! as Map; - _assertAccessibilityTree(tester, expectTree); - } - } finally { - harness.dispose(); - semantics.dispose(); - } -} - -Future _runAdapterCase( - WidgetTester tester, - Map testCase, -) async { - final SemanticsHandle semantics = tester.ensureSemantics(); - final harness = _ConformanceHarness(); - try { - final steps = testCase['steps']! as List; - for (final rawStep in steps) { - final step = rawStep! as Map; - await _executeStep(tester, harness, step); - _assertStepExpectations(tester, harness, step); - _snapshotIdentities(tester, harness); - } - } finally { - harness.dispose(); - semantics.dispose(); - } -} - -Future _executeStep( - WidgetTester tester, - _ConformanceHarness harness, - Map step, -) async { - harness.actions.clear(); - harness.errors.clear(); - final op = step['op']! as String; - switch (op) { - case 'process_messages': - await harness.processMessages(tester, step['messages']! as List); - case 'user_input': - await _executeUserInput(tester, step); - case 'user_gesture': - await _executeUserGesture(tester, step); - case 'programmatic_invoke': - await _executeProgrammaticInvoke(tester, harness, step); - case 'unmount_surface': - harness.mounted = false; - await tester.pumpWidget(const SizedBox()); - default: - fail('Unsupported adapter step op: $op'); - } -} - -Finder _findNodeView(String instanceId, [Finder? ancestor]) { - final Finder base = find.byWidgetPredicate( - (w) => w.key == NodeKey(instanceId) && w is! KeyedSubtree && w is! Flexible, - ); - return ancestor == null - ? base - : find.descendant(of: ancestor, matching: base); -} - -String _stepTargetId(Map step) => - (step['targetNode'] ?? step['targetComponentId'])! as String; - -Future _executeUserInput( - WidgetTester tester, - Map step, -) async { - final String targetId = _stepTargetId(step); - final Object? value = step['inputValue'] ?? step['value']; - final Finder scope = _findNodeView(targetId); - if (value is String) { - final Finder field = find.descendant( - of: scope, - matching: find.byType(TextField), - ); - await tester.enterText(field, value); - await tester.pump(); - } else if (value is bool) { - final Finder checkbox = find.descendant( - of: scope, - matching: find.byType(Checkbox), - ); - await tester.tap(checkbox); - await tester.pump(); - } -} - -Future _executeUserGesture( - WidgetTester tester, - Map step, -) async { - final String targetId = _stepTargetId(step); - final Finder scope = _findNodeView(targetId); - final Finder button = find.descendant( - of: scope, - matching: find.byWidgetPredicate((w) => w is ButtonStyleButton), - ); - await tester.tap(button); - await tester.pump(); -} - -Future _executeProgrammaticInvoke( - WidgetTester tester, - _ConformanceHarness harness, - Map step, -) async { - final String targetId = _stepTargetId(step); - final String propName = (step['property'] as String?) ?? 'action'; - final SurfaceModel surface = harness.surface!; - final ComponentNode node = _findNodeById( - tester, - harness, - targetId, - )!; - final Object? rawAction = node.context!.componentModel.properties[propName]; - if (rawAction is Map && rawAction['functionCall'] is Map) { - final fc = Map.from(rawAction['functionCall'] as Map); - try { - await node.context!.dataContext.evaluateFunctionCall(fc); - } on A2uiExpressionError catch (error) { - await surface.dispatchError( - ComponentContext.clientErrorFor(surface, error), - ); - } - } - await tester.pump(); -} - -void _assertStepExpectations( - WidgetTester tester, - _ConformanceHarness harness, - Map step, -) { - if (step['expectViewTree'] case final Map expectedTree) { - final ComponentNode? root = harness.rootNode; - expect(root, isNotNull); - _assertViewNode(tester, root!, expectedTree); - } - if (step['expectSameViewIdentity'] case final List keys) { - final Map, Element?)> - current = _collectIdentityMap(tester, harness.rootNode); - for (final rawKey in keys) { - final key = rawKey! as String; - expect(current[key]?.$1, same(harness.savedNodes[key])); - expect(current[key]?.$2, same(harness.savedElements[key])); - } - } - if (step['expectDataModel'] case final Map expectedData) { - expect(harness.surface!.dataModel.get('/'), equals(expectedData)); - } - if (step['expectActions'] case final List expectedActions) { - _assertActions(harness.actions, expectedActions); - } - if (step['expectErrors'] case final List expectedErrors) { - _assertErrors(harness.errors, expectedErrors); - } - if (step['expectAccessibilityTree'] - case final Map expectedA11y) { - _assertAccessibilityTree(tester, expectedA11y); - } - if (step['expectActiveSubscriptions'] case final int expectedSubs) { - final int liveViews = find - .byWidgetPredicate( - (w) => w.key is NodeKey && w is! KeyedSubtree && w is! Flexible, - ) - .evaluate() - .length; - expect(liveViews, expectedSubs); - for (final ComponentNode saved - in harness.savedNodes.values) { - expect(saved.disposed, isTrue); - } - } -} - -List> _directChildren( - ComponentNode node, -) { - final props = ComponentProps(node.props.peek()); - if (props.child('child') - case final ComponentNode c) { - return [c]; - } - return props.children('children'); -} - -Map, Element?)> -_collectIdentityMap( - WidgetTester tester, - ComponentNode? root, -) { - final result = , Element?)>{}; - void visit( - ComponentNode node, - String path, - Finder? ancestor, - ) { - final Finder finder = _findNodeView(node.instanceId, ancestor); - final Iterable elements = finder.evaluate(); - final Element? elem = elements.isEmpty ? null : elements.first; - result[node.instanceId] = (node, elem); - result[path] = (node, elem); - final List> children = - _directChildren(node); - for (var i = 0; i < children.length; i++) { - visit(children[i], '$path/children/$i', finder); - } - } - - if (root != null) visit(root, root.componentId, null); - return result; -} - -void _snapshotIdentities(WidgetTester tester, _ConformanceHarness harness) { - harness.savedElements.clear(); - harness.savedNodes.clear(); - final Map, Element?)> - identities = _collectIdentityMap(tester, harness.rootNode); - for (final MapEntry< - String, - (ComponentNode, Element?) - > - entry - in identities.entries) { - harness.savedNodes[entry.key] = entry.value.$1; - if (entry.value.$2 != null) { - harness.savedElements[entry.key] = entry.value.$2!; - } - } -} - -ComponentNode? _findNodeById( - WidgetTester tester, - _ConformanceHarness harness, - String id, -) { - final Map, Element?)> - identities = _collectIdentityMap(tester, harness.rootNode); - return identities[id]?.$1; -} - -void _assertViewNode( - WidgetTester tester, - ComponentNode node, - Map spec, [ - Finder? ancestor, -]) { - expect(node.componentId, spec['id']); - final Finder candidates = _findNodeView(node.instanceId, ancestor); - if (spec['state'] case final String expectedState) { - _assertNodeState(node, candidates, expectedState); - if (expectedState != 'resolved') return; - } - final Finder viewFinder = candidates.first; - expect(viewFinder, findsOneWidget); - if (spec['type'] case final String expectedType) { - expect(node.type, expectedType); - } - final props = ComponentProps(node.props.peek()); - for (final propKey in const ['text', 'value', 'badgeText']) { - if (spec[propKey] case final Object expectedVal) { - expect(props.value(propKey), expectedVal); - } - } - _assertNodeValidation(tester, props, viewFinder, spec); - if (spec['children'] case final List expectedChildren) { - final List> actualChildren = - _directChildren(node); - expect(actualChildren, hasLength(expectedChildren.length)); - for (var i = 0; i < expectedChildren.length; i++) { - _assertViewNode( - tester, - actualChildren[i], - expectedChildren[i]! as Map, - viewFinder, - ); - } - } -} - -void _assertNodeState( - ComponentNode node, - Finder candidates, - String expectedState, -) { - switch (expectedState) { - case 'pending': - expect(node.state, NodeState.pending); - expect( - find.descendant( - of: candidates, - matching: find.byType(CircularProgressIndicator), - ), - findsNothing, - ); - case 'resolved': - expect(node.state, NodeState.resolved); - expect(candidates, findsAtLeastNWidgets(1)); - case 'unknown-type': - expect(node.state, NodeState.unknownType); - expect( - find.descendant(of: candidates, matching: find.byType(Text)), - findsNothing, - ); - case 'cyclic': - expect(node.state, NodeState.cyclic); - expect( - find.descendant(of: candidates, matching: find.byType(Text)), - findsNothing, - ); - } -} - -void _assertNodeValidation( - WidgetTester tester, - ComponentProps props, - Finder viewFinder, - Map spec, -) { - if (spec['isValid'] case final bool expectedValid) { - expect(props.isValid, expectedValid); - } - if (spec['validationErrors'] case final List expectedErrors) { - expect(props.validationErrors, expectedErrors); - } - if (spec['validationResults'] case final List expectedResults) { - final List> actual = [ - for (final ValidationResult r in props.validationResults) - { - 'valid': r.valid, - if (r.code != null) 'code': r.code, - if (r.message != null) 'message': r.message, - 'severity': r.severity ?? 'error', - }, - ]; - expect(actual, expectedResults); - } -} - -void _assertActions( - List actual, - List expectedList, -) { - expect(actual, hasLength(expectedList.length)); - for (var i = 0; i < expectedList.length; i++) { - final expected = expectedList[i]! as Map; - expect(actual[i].name, expected['name']); - expect(actual[i].surfaceId, expected['surfaceId']); - expect(actual[i].sourceComponentId, expected['sourceComponentId']); - expect(actual[i].context, equals(expected['context'])); - } -} - -void _assertErrors(List actual, List expectedList) { - expect(actual, hasLength(expectedList.length)); - for (final rawExpected in expectedList) { - final expected = rawExpected! as Map; - final expectedCode = expected['code'] as String?; - final expectedSurfaceId = expected['surfaceId'] as String?; - final messageSub = - (expected['messageContains'] ?? expected['message']) as String?; - expect( - actual.any( - (err) => - (expectedCode == null || err.code == expectedCode) && - (expectedSurfaceId == null || err.surfaceId == expectedSurfaceId) && - (messageSub == null || err.message.contains(messageSub)), - ), - isTrue, - reason: 'Expected error matching $expected in $actual', - ); - } -} - -void _assertAccessibilityTree( - WidgetTester tester, - Map expectedTree, -) { - for (final MapEntry entry in expectedTree.entries) { - final String id = entry.key; - final spec = entry.value! as Map; - final Finder viewFinder = _findNodeView(id).first; - expect(viewFinder, findsOneWidget); - final Semantics semanticsWidget = tester.widget( - find.descendant(of: viewFinder, matching: find.byType(Semantics)).first, - ); - final List subtreeData = _collectSemanticsData( - tester, - viewFinder, - ); - _assertComponentSemantics(semanticsWidget, subtreeData, spec); - } -} - -List _collectSemanticsData( - WidgetTester tester, - Finder viewFinder, -) { - final results = []; - final visited = {}; - void addNode(SemanticsNode node) { - if (!visited.add(node.id)) return; - results.add(node.getSemanticsData()); - node.visitChildren((child) { - addNode(child); - return true; - }); - } - - for (final Element element - in find - .descendant( - of: viewFinder, - matching: find.byWidgetPredicate((_) => true), - ) - .evaluate()) { - final RenderObject? renderObject = element.renderObject; - if (renderObject != null && renderObject.debugSemantics != null) { - addNode(renderObject.debugSemantics!); - } - } - return results; -} - -void _assertComponentSemantics( - Semantics rootSemantics, - List subtreeData, - Map spec, -) { - if (spec['label'] case final String expectedLabel) { - final matchesRoot = rootSemantics.properties.label == expectedLabel; - final bool matchesSubtree = subtreeData.any( - (d) => - d.label == expectedLabel || - d.label.split('\n').contains(expectedLabel), - ); - expect(matchesRoot || matchesSubtree, isTrue); - } - if (spec['description'] case final String expectedDescription) { - expect(rootSemantics.properties.hint, expectedDescription); - } - if (spec['live'] case final String expectedLive) { - final bool shouldBeLive = - expectedLive == 'polite' || expectedLive == 'assertive'; - expect(rootSemantics.properties.liveRegion ?? false, shouldBeLive); - } - if (spec['hidden'] case final bool expectedHidden) { - expect(rootSemantics.properties.hidden ?? false, expectedHidden); - } - if (spec['role'] case final String expectedRole) { - expect(_hasRole(subtreeData, expectedRole), isTrue); - } - if (spec['checked'] case final bool expectedChecked) { - final CheckedState expectedState = expectedChecked - ? CheckedState.isTrue - : CheckedState.isFalse; - expect( - subtreeData.any((d) => d.flagsCollection.isChecked == expectedState), - isTrue, - ); - } -} - -bool _hasRole(List data, String role) => switch (role) { - 'button' => data.any((d) => d.flagsCollection.isButton), - 'textbox' => data.any((d) => d.flagsCollection.isTextField), - 'checkbox' => data.any( - (d) => d.flagsCollection.isChecked != CheckedState.none, - ), - 'radiogroup' => data.any((d) => d.flagsCollection.isInMutuallyExclusiveGroup), - _ => false, -}; diff --git a/dart/a2ui_flutter/test/adapter_harness.dart b/dart/a2ui_flutter/test/adapter_harness.dart new file mode 100644 index 0000000000..e06d67449c --- /dev/null +++ b/dart/a2ui_flutter/test/adapter_harness.dart @@ -0,0 +1,1047 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:ui' show CheckedState; + +import 'package:a2ui_core/a2ui_core.dart'; +import 'package:a2ui_flutter/a2ui_flutter.dart'; +import 'package:a2ui_flutter/basic_catalog.dart'; +import 'package:a2ui_flutter/v0_9/basic_catalog.dart' as v0_9; +// ignore: deprecated_member_use, see https://github.com/a2ui-project/a2ui/issues/3235 +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:json_schema_builder/json_schema_builder.dart' + hide ValidationResult; +import 'package:yaml/yaml.dart'; + +/// Reasoned skip entries keyed by `` or `/`. +/// Every entry in this map must match at least one case during the run. +const Map unsupported = {}; + +final Set _matchedUnsupportedKeys = {}; + +const Set adapterCaseKnownKeys = { + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPaths', + 'catalogs', + 'steps', +}; + +const Set adapterCatalogKnownKeys = { + 'catalogId', + 'protocolVersion', + 'components', +}; + +const Set adapterStepKnownKeys = { + 'description', + 'op', + 'messages', + 'targetNode', + 'inputValue', + 'userActivation', + 'expectViewTree', + 'expectAccessibilityTree', + 'expectDataModel', + 'expectActions', + 'expectErrors', + 'expectSameViewIdentity', + 'expectActiveSubscriptions', +}; + +const Set adapterStepKnownOps = { + 'process_messages', + 'user_input', + 'user_gesture', + 'programmatic_invoke', + 'unmount_surface', +}; + +const Set viewNodeKnownKeys = { + 'id', + 'type', + 'state', + 'text', + 'value', + 'badgeText', + 'isValid', + 'validationErrors', + 'validationResults', + 'children', +}; + +const Set validationResultKnownKeys = { + 'valid', + 'code', + 'message', + 'severity', +}; + +const Set expectedActionKnownKeys = { + 'name', + 'surfaceId', + 'sourceComponentId', + 'context', +}; + +const Set expectedErrorKnownKeys = {'code', 'surfaceId', 'message'}; + +const Set accessibilityCaseKnownKeys = { + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPath', + 'surface', + 'messages', + 'assertions', + 'steps', +}; + +const Set accessibilityStepKnownKeys = { + 'description', + 'messages', + 'assertions', +}; + +const Set accessibilityAssertionsKnownKeys = { + 'accessibilityTree', + 'axeCore', +}; + +const Set accessibilityNodeKnownKeys = { + 'label', + 'description', + 'live', + 'hidden', + 'role', + 'checked', +}; + +void _assertMapKeys( + Map obj, + Set allowed, + String where, +) { + final List unknown = + obj.keys.where((k) => !allowed.contains(k)).toList()..sort(); + if (unknown.isNotEmpty) { + fail('Unhandled keys at $where: $unknown'); + } +} + +void _assertKnownViewNodeKeys(Map node, String where) { + _assertMapKeys(node, viewNodeKnownKeys, where); + if (node['validationResults'] case final List results) { + for (var i = 0; i < results.length; i++) { + _assertMapKeys( + results[i]! as Map, + validationResultKnownKeys, + '$where.validationResults[$i]', + ); + } + } + if (node['children'] case final List children) { + for (var i = 0; i < children.length; i++) { + _assertKnownViewNodeKeys( + children[i]! as Map, + '$where.children[$i]', + ); + } + } +} + +void assertKnownKeys( + Map obj, [ + Set allowed = adapterStepKnownKeys, + String where = 'step', +]) { + _assertMapKeys(obj, allowed, where); + if (allowed != adapterStepKnownKeys) return; + + if (obj['op'] case final Object rawOp) { + if (rawOp is! String || !adapterStepKnownOps.contains(rawOp)) { + fail('Unsupported adapter step op at $where.op: $rawOp'); + } + } + if (obj['expectViewTree'] case final Map viewTree) { + _assertKnownViewNodeKeys(viewTree, '$where.expectViewTree'); + } + if (obj['expectAccessibilityTree'] case final Map a11yTree) { + for (final MapEntry entry in a11yTree.entries) { + _assertMapKeys( + entry.value! as Map, + accessibilityNodeKnownKeys, + '$where.expectAccessibilityTree.${entry.key}', + ); + } + } + if (obj['expectActions'] case final List actions) { + for (var i = 0; i < actions.length; i++) { + _assertMapKeys( + actions[i]! as Map, + expectedActionKnownKeys, + '$where.expectActions[$i]', + ); + } + } + if (obj['expectErrors'] case final List errors) { + for (var i = 0; i < errors.length; i++) { + _assertMapKeys( + errors[i]! as Map, + expectedErrorKnownKeys, + '$where.expectErrors[$i]', + ); + } + } +} + +void assertKnownAdapterCase(Map testCase) { + final name = testCase['name']! as String; + _assertMapKeys(testCase, adapterCaseKnownKeys, 'case($name)'); + if (testCase['action'] != 'render_surface') { + fail('Unsupported action in case($name): ${testCase['action']}'); + } + if (testCase['catalogs'] case final List catalogs) { + for (var i = 0; i < catalogs.length; i++) { + _assertMapKeys( + catalogs[i]! as Map, + adapterCatalogKnownKeys, + 'case($name).catalogs[$i]', + ); + } + } + final steps = testCase['steps']! as List; + for (var i = 0; i < steps.length; i++) { + assertKnownKeys( + steps[i]! as Map, + adapterStepKnownKeys, + 'case($name).steps[$i]', + ); + } +} + +void _assertKnownAccessibilityAssertions( + Map? assertions, + String where, +) { + if (assertions == null) return; + _assertMapKeys(assertions, accessibilityAssertionsKnownKeys, where); + if (assertions['accessibilityTree'] case final Map tree) { + for (final MapEntry entry in tree.entries) { + _assertMapKeys( + entry.value! as Map, + accessibilityNodeKnownKeys, + '$where.accessibilityTree.${entry.key}', + ); + } + } +} + +void assertKnownAccessibilityCase(Map testCase) { + final name = testCase['name']! as String; + _assertMapKeys(testCase, accessibilityCaseKnownKeys, 'case($name)'); + if (testCase['action'] != 'accessibility_check') { + fail('Unsupported action in case($name): ${testCase['action']}'); + } + _assertKnownAccessibilityAssertions( + testCase['assertions'] as Map?, + 'case($name).assertions', + ); + if (testCase['steps'] case final List steps) { + for (var i = 0; i < steps.length; i++) { + final step = steps[i]! as Map; + _assertMapKeys(step, accessibilityStepKnownKeys, 'case($name).steps[$i]'); + _assertKnownAccessibilityAssertions( + step['assertions'] as Map?, + 'case($name).steps[$i].assertions', + ); + } + } +} + +String? findUnsupportedReason( + String caseName, [ + List steps = const [], +]) { + if (unsupported.containsKey(caseName)) { + _matchedUnsupportedKeys.add(caseName); + return unsupported[caseName]; + } + for (final rawStep in steps) { + final step = rawStep! as Map; + final op = step['op'] as String?; + if (op != null) { + final opKey = '$caseName/$op'; + if (unsupported.containsKey(opKey)) { + _matchedUnsupportedKeys.add(opKey); + return unsupported[opKey]; + } + } + for (final String key in step.keys) { + final stepKey = '$caseName/$key'; + if (unsupported.containsKey(stepKey)) { + _matchedUnsupportedKeys.add(stepKey); + return unsupported[stepKey]; + } + } + } + return null; +} + +void assertAllUnsupportedMatched() { + final List unused = + unsupported.keys + .where((k) => !_matchedUnsupportedKeys.contains(k)) + .toList() + ..sort(); + if (unused.isNotEmpty) { + fail('Unused unsupported entries did not match any case: $unused'); + } +} + +void writeManifestRecord(Map rec) { + final String? dir = Platform.environment['A2UI_CONFORMANCE_MANIFEST_DIR']; + if (dir == null || dir.isEmpty) return; + Directory(dir).createSync(recursive: true); + final String harness = (rec['harness'] as String?) ?? 'flutter'; + final file = File( + '$dir${Platform.pathSeparator}' + '$harness-$pid-${Isolate.current.hashCode}.jsonl', + ); + final reason = rec['reason'] as String?; + final payload = { + 'harness': harness, + 'suite': (rec['suite']! as String).replaceAll(r'\', '/'), + 'case': rec['case']! as String, + 'status': rec['status']! as String, + if (reason != null && reason.isNotEmpty) 'reason': reason, + }; + file.writeAsStringSync( + '${jsonEncode(payload)}\n', + mode: FileMode.append, + flush: true, + ); +} + +List loadYamlTests(String relativePath) { + final String source = File(relativePath).readAsStringSync(); + final Object? parsed = _yamlToDart(loadYaml(source)); + if (parsed is List) return parsed; + return (parsed! as Map)['tests']! as List; +} + +Object? _yamlToDart(Object? node) { + if (node is YamlMap) { + return { + for (final MapEntry entry in node.entries) + entry.key.toString(): _yamlToDart(entry.value), + }; + } + if (node is YamlList) { + return [for (final Object? item in node) _yamlToDart(item)]; + } + return node; +} + +CatalogImplementation _buildV10BasicCatalog({ + required Map> renderedNodes, + List? openedUrls, +}) { + final Catalog v10Api = + BasicCatalog.v1_0(openUrl: (uri) => openedUrls?.add(uri.toString())); + final Map v09ByName = { + for (final ComponentImplementation impl in BasicComponents.all) + impl.name: impl, + }; + final components = [ + for (final MapEntry entry + in v10Api.components.entries) + if (v09ByName[entry.key] case final ComponentImplementation impl) + ComponentImplementation( + name: entry.key, + schema: entry.value.schema, + builder: (context, node, props, buildChild) { + renderedNodes[node.instanceId] = node; + return impl.builder(context, node, props, buildChild); + }, + ), + ]; + return CatalogImplementation( + id: BasicCatalog.v1_0Id, + components: components, + functions: [...v10Api.functions.values, _IncrementCounterFunction()], + themeSchema: v10Api.themeSchema, + schemaId: v10Api.schemaId, + title: v10Api.title, + description: v10Api.description, + protocolVersion: v10Api.protocolVersion, + instructions: v10Api.instructions, + definitions: v10Api.definitions, + ); +} + +CatalogImplementation _buildCustomStatusBadgeCatalog( + Map> renderedNodes, +) { + final statusBadge = ComponentImplementation( + name: 'StatusBadge', + schema: Schema.fromMap({ + 'type': 'object', + 'properties': { + 'component': {'const': 'StatusBadge'}, + 'badgeText': { + 'oneOf': [ + {'type': 'string'}, + { + 'type': 'object', + 'properties': { + '@path': {'type': 'string'}, + }, + 'required': ['@path'], + }, + ], + }, + }, + 'required': ['component', 'badgeText'], + }), + builder: (context, node, props, buildChild) { + renderedNodes[node.instanceId] = node; + return Text(props.string('badgeText') ?? ''); + }, + ); + return CatalogImplementation( + id: 'https://example.com/catalogs/badges/v1/catalog.json', + components: [statusBadge], + protocolVersion: A2uiProtocolVersion.v1_0, + ); +} + +class _IncrementCounterFunction extends FunctionImplementation { + _IncrementCounterFunction() + : super( + name: 'incrementCounter', + argumentSchema: Schema.fromMap({'type': 'object'}), + ); + + @override + Object? execute( + Map args, + DataContext context, [ + CancellationSignal? cancellationSignal, + ]) { + final path = args['path']! as String; + final num step = (args['step'] as num?) ?? 1; + final num current = (context.dataModel.get(path) as num?) ?? 0; + context.dataModel.set(path, current + step); + return null; + } +} + +class AdapterHarnessSession { + AdapterHarnessSession([this.tester]) { + final CatalogImplementation v10Catalog = _buildV10BasicCatalog( + renderedNodes: renderedNodes, + openedUrls: openedUrls, + ); + final CatalogImplementation customCatalog = _buildCustomStatusBadgeCatalog( + renderedNodes, + ); + processor = MessageProcessor( + catalogs: [v0_9.basicCatalog(), v10Catalog, customCatalog], + defaultVersion: A2uiProtocolVersion.v1_0, + ); + processor.model.onAction.subscribe(actions.add); + processor.model.onSurfaceCreated.subscribe((created) { + surface = created; + created.onError.subscribe(errors.add); + }); + } + + WidgetTester? tester; + late final MessageProcessor processor; + SurfaceModel? surface; + final Map> renderedNodes = {}; + final List actions = []; + final List errors = []; + final List openedUrls = []; + final Map savedElements = {}; + final Map> savedNodes = {}; + bool mounted = false; + + ComponentNode? get rootNode => + renderedNodes[surface?.rootId ?? 'root']; + + Future processMessages( + WidgetTester activeTester, + List rawMessages, + ) async { + final messages = >[ + for (final Object? raw in rawMessages) + Map.from(raw! as Map), + ]; + processor.processMessages( + AgentToRendererMessagePayload.fromJson( + messages, + protocolVersion: A2uiProtocolVersion.v1_0, + ), + ); + if (!mounted && surface != null) { + mounted = true; + await activeTester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SingleChildScrollView(child: A2uiSurface(surface: surface!)), + ), + ), + ); + } else { + await activeTester.pump(); + } + } + + void dispose() { + processor.model.dispose(); + } +} + +AdapterHarnessSession mountSurface([ + Map? caseSpec, + WidgetTester? tester, +]) => AdapterHarnessSession(tester); + +ComponentNode? extractViewTree( + AdapterHarnessSession session, +) => session.rootNode; + +void assertViewTree( + ComponentNode? actual, + Map expected, [ + WidgetTester? tester, + Finder? ancestor, +]) { + assertKnownKeys({'expectViewTree': expected}); + expect(actual, isNotNull); + final ComponentNode node = actual!; + expect(node.componentId, expected['id']); + final Finder candidates = _findNodeView(node.instanceId, ancestor); + if (expected['state'] case final String expectedState) { + _assertNodeState(node, candidates, expectedState); + if (expectedState != 'resolved') return; + } + final Finder viewFinder = candidates.first; + expect(viewFinder, findsOneWidget); + if (expected['type'] case final String expectedType) { + expect(node.type, expectedType); + } + final props = ComponentProps(node.props.peek()); + for (final propKey in const ['text', 'value', 'badgeText']) { + if (expected[propKey] case final Object expectedVal) { + expect(props.value(propKey), expectedVal); + } + } + _assertNodeValidation(tester, props, viewFinder, expected); + if (expected['children'] case final List expectedChildren) { + final List> actualChildren = + _directChildren(node); + expect(actualChildren, hasLength(expectedChildren.length)); + for (var i = 0; i < expectedChildren.length; i++) { + assertViewTree( + actualChildren[i], + expectedChildren[i]! as Map, + tester, + viewFinder, + ); + } + } +} + +Future runStep( + AdapterHarnessSession session, + Map step, [ + WidgetTester? tester, +]) async { + assertKnownKeys(step); + final WidgetTester activeTester = (tester ?? session.tester)!; + await _executeStep(activeTester, session, step); + _assertStepExpectations(activeTester, session, step); + _snapshotIdentities(activeTester, session); +} + +Future runAccessibilityCase( + WidgetTester tester, + Map testCase, +) async { + assertKnownAccessibilityCase(testCase); + final SemanticsHandle semantics = tester.ensureSemantics(); + final AdapterHarnessSession harness = mountSurface(testCase, tester); + try { + final steps = testCase['steps'] as List?; + if (steps != null) { + for (final Object? rawStep in steps) { + final step = rawStep! as Map; + final stepMessages = step['messages']! as List; + await harness.processMessages(tester, stepMessages); + final stepAssertions = step['assertions']! as Map; + final stepTree = + stepAssertions['accessibilityTree']! as Map; + _assertAccessibilityTree(tester, stepTree); + } + } else { + final messages = testCase['messages']! as List; + await harness.processMessages(tester, messages); + final assertions = testCase['assertions']! as Map; + final expectTree = + assertions['accessibilityTree']! as Map; + _assertAccessibilityTree(tester, expectTree); + } + } finally { + harness.dispose(); + semantics.dispose(); + } +} + +Future runAdapterCase( + WidgetTester tester, + Map testCase, +) async { + assertKnownAdapterCase(testCase); + final SemanticsHandle semantics = tester.ensureSemantics(); + final AdapterHarnessSession harness = mountSurface(testCase, tester); + try { + final steps = testCase['steps']! as List; + for (final rawStep in steps) { + final step = rawStep! as Map; + await runStep(harness, step, tester); + } + } finally { + harness.dispose(); + semantics.dispose(); + } +} + +Future _executeStep( + WidgetTester tester, + AdapterHarnessSession harness, + Map step, +) async { + harness.actions.clear(); + harness.errors.clear(); + final op = step['op']! as String; + switch (op) { + case 'process_messages': + await harness.processMessages(tester, step['messages']! as List); + case 'user_input': + await _executeUserInput(tester, step); + case 'user_gesture': + await _executeUserGesture(tester, step); + case 'programmatic_invoke': + await _executeProgrammaticInvoke(tester, harness, step); + case 'unmount_surface': + harness.mounted = false; + await tester.pumpWidget(const SizedBox()); + default: + fail('Unsupported adapter step op: $op'); + } +} + +Finder _findNodeView(String instanceId, [Finder? ancestor]) { + final Finder base = find.byWidgetPredicate( + (w) => w.key == NodeKey(instanceId) && w is! KeyedSubtree && w is! Flexible, + ); + return ancestor == null + ? base + : find.descendant(of: ancestor, matching: base); +} + +String _stepTargetId(Map step) => + (step['targetNode'] ?? step['targetComponentId'])! as String; + +Future _executeUserInput( + WidgetTester tester, + Map step, +) async { + final String targetId = _stepTargetId(step); + final Object? value = step['inputValue'] ?? step['value']; + final Finder scope = _findNodeView(targetId); + if (value is String) { + final Finder field = find.descendant( + of: scope, + matching: find.byType(TextField), + ); + await tester.enterText(field, value); + await tester.pump(); + } else if (value is bool) { + final Finder checkbox = find.descendant( + of: scope, + matching: find.byType(Checkbox), + ); + await tester.tap(checkbox); + await tester.pump(); + } +} + +Future _executeUserGesture( + WidgetTester tester, + Map step, +) async { + final String targetId = _stepTargetId(step); + final Finder scope = _findNodeView(targetId); + final Finder button = find.descendant( + of: scope, + matching: find.byWidgetPredicate((w) => w is ButtonStyleButton), + ); + await tester.tap(button); + await tester.pump(); +} + +Future _executeProgrammaticInvoke( + WidgetTester tester, + AdapterHarnessSession harness, + Map step, +) async { + final String targetId = _stepTargetId(step); + final String propName = (step['property'] as String?) ?? 'action'; + final SurfaceModel surface = harness.surface!; + final ComponentNode node = _findNodeById( + tester, + harness, + targetId, + )!; + final Object? rawAction = node.context!.componentModel.properties[propName]; + if (rawAction is Map && rawAction['functionCall'] is Map) { + final fc = Map.from(rawAction['functionCall'] as Map); + try { + await node.context!.dataContext.evaluateFunctionCall(fc); + } on A2uiExpressionError catch (error) { + await surface.dispatchError( + ComponentContext.clientErrorFor(surface, error), + ); + } + } + await tester.pump(); +} + +void _assertStepExpectations( + WidgetTester tester, + AdapterHarnessSession harness, + Map step, +) { + if (step['expectViewTree'] case final Map expectedTree) { + assertViewTree(extractViewTree(harness), expectedTree, tester); + } + if (step['expectSameViewIdentity'] case final List keys) { + final Map, Element?)> + current = _collectIdentityMap(tester, harness.rootNode); + for (final rawKey in keys) { + final key = rawKey! as String; + expect(current[key]?.$1, same(harness.savedNodes[key])); + expect(current[key]?.$2, same(harness.savedElements[key])); + } + } + if (step['expectDataModel'] case final Map expectedData) { + expect(harness.surface!.dataModel.get('/'), equals(expectedData)); + } + if (step['expectActions'] case final List expectedActions) { + _assertActions(harness.actions, expectedActions); + } + if (step['expectErrors'] case final List expectedErrors) { + _assertErrors(harness.errors, expectedErrors); + } + if (step['expectAccessibilityTree'] + case final Map expectedA11y) { + _assertAccessibilityTree(tester, expectedA11y); + } + if (step['expectActiveSubscriptions'] case final int expectedSubs) { + final int liveViews = find + .byWidgetPredicate( + (w) => w.key is NodeKey && w is! KeyedSubtree && w is! Flexible, + ) + .evaluate() + .length; + expect(liveViews, expectedSubs); + for (final ComponentNode saved + in harness.savedNodes.values) { + expect(saved.disposed, isTrue); + } + } +} + +List> _directChildren( + ComponentNode node, +) { + final props = ComponentProps(node.props.peek()); + if (props.child('child') + case final ComponentNode c) { + return [c]; + } + return props.children('children'); +} + +Map, Element?)> +_collectIdentityMap( + WidgetTester tester, + ComponentNode? root, +) { + final result = , Element?)>{}; + void visit( + ComponentNode node, + String path, + Finder? ancestor, + ) { + final Finder finder = _findNodeView(node.instanceId, ancestor); + final Iterable elements = finder.evaluate(); + final Element? elem = elements.isEmpty ? null : elements.first; + result[node.instanceId] = (node, elem); + result[path] = (node, elem); + final List> children = + _directChildren(node); + for (var i = 0; i < children.length; i++) { + visit(children[i], '$path/children/$i', finder); + } + } + + if (root != null) visit(root, root.componentId, null); + return result; +} + +void _snapshotIdentities(WidgetTester tester, AdapterHarnessSession harness) { + harness.savedElements.clear(); + harness.savedNodes.clear(); + final Map, Element?)> + identities = _collectIdentityMap(tester, harness.rootNode); + for (final MapEntry< + String, + (ComponentNode, Element?) + > + entry + in identities.entries) { + harness.savedNodes[entry.key] = entry.value.$1; + if (entry.value.$2 != null) { + harness.savedElements[entry.key] = entry.value.$2!; + } + } +} + +ComponentNode? _findNodeById( + WidgetTester tester, + AdapterHarnessSession harness, + String id, +) { + final Map, Element?)> + identities = _collectIdentityMap(tester, harness.rootNode); + return identities[id]?.$1; +} + +void _assertNodeState( + ComponentNode node, + Finder candidates, + String expectedState, +) { + switch (expectedState) { + case 'pending': + expect(node.state, NodeState.pending); + expect( + find.descendant( + of: candidates, + matching: find.byType(CircularProgressIndicator), + ), + findsNothing, + ); + case 'resolved': + expect(node.state, NodeState.resolved); + expect(candidates, findsAtLeastNWidgets(1)); + case 'unknown-type': + expect(node.state, NodeState.unknownType); + expect( + find.descendant(of: candidates, matching: find.byType(Text)), + findsNothing, + ); + case 'cyclic': + expect(node.state, NodeState.cyclic); + expect( + find.descendant(of: candidates, matching: find.byType(Text)), + findsNothing, + ); + } +} + +void _assertNodeValidation( + WidgetTester? tester, + ComponentProps props, + Finder viewFinder, + Map spec, +) { + if (spec['isValid'] case final bool expectedValid) { + expect(props.isValid, expectedValid); + } + if (spec['validationErrors'] case final List expectedErrors) { + expect(props.validationErrors, expectedErrors); + } + if (spec['validationResults'] case final List expectedResults) { + final List> actual = [ + for (final ValidationResult r in props.validationResults) + { + 'valid': r.valid, + if (r.code != null) 'code': r.code, + if (r.message != null) 'message': r.message, + 'severity': r.severity ?? 'error', + }, + ]; + expect(actual, expectedResults); + } +} + +void _assertActions( + List actual, + List expectedList, +) { + expect(actual, hasLength(expectedList.length)); + for (var i = 0; i < expectedList.length; i++) { + final expected = expectedList[i]! as Map; + expect(actual[i].name, expected['name']); + expect(actual[i].surfaceId, expected['surfaceId']); + expect(actual[i].sourceComponentId, expected['sourceComponentId']); + expect(actual[i].context, equals(expected['context'])); + } +} + +void _assertErrors(List actual, List expectedList) { + expect(actual, hasLength(expectedList.length)); + for (final rawExpected in expectedList) { + final expected = rawExpected! as Map; + final expectedCode = expected['code'] as String?; + final expectedSurfaceId = expected['surfaceId'] as String?; + final messageSub = + (expected['messageContains'] ?? expected['message']) as String?; + expect( + actual.any( + (err) => + (expectedCode == null || err.code == expectedCode) && + (expectedSurfaceId == null || err.surfaceId == expectedSurfaceId) && + (messageSub == null || err.message.contains(messageSub)), + ), + isTrue, + reason: 'Expected error matching $expected in $actual', + ); + } +} + +void _assertAccessibilityTree( + WidgetTester tester, + Map expectedTree, +) { + for (final MapEntry entry in expectedTree.entries) { + final String id = entry.key; + final spec = entry.value! as Map; + final Finder viewFinder = _findNodeView(id).first; + expect(viewFinder, findsOneWidget); + final Semantics semanticsWidget = tester.widget( + find.descendant(of: viewFinder, matching: find.byType(Semantics)).first, + ); + final List subtreeData = _collectSemanticsData( + tester, + viewFinder, + ); + _assertComponentSemantics(semanticsWidget, subtreeData, spec); + } +} + +List _collectSemanticsData( + WidgetTester tester, + Finder viewFinder, +) { + final results = []; + final visited = {}; + void addNode(SemanticsNode node) { + if (!visited.add(node.id)) return; + results.add(node.getSemanticsData()); + node.visitChildren((child) { + addNode(child); + return true; + }); + } + + for (final Element element + in find + .descendant( + of: viewFinder, + matching: find.byWidgetPredicate((_) => true), + ) + .evaluate()) { + final RenderObject? renderObject = element.renderObject; + if (renderObject != null && renderObject.debugSemantics != null) { + addNode(renderObject.debugSemantics!); + } + } + return results; +} + +void _assertComponentSemantics( + Semantics rootSemantics, + List subtreeData, + Map spec, +) { + if (spec['label'] case final String expectedLabel) { + final matchesRoot = rootSemantics.properties.label == expectedLabel; + final bool matchesSubtree = subtreeData.any( + (d) => + d.label == expectedLabel || + d.label.split('\n').contains(expectedLabel), + ); + expect(matchesRoot || matchesSubtree, isTrue); + } + if (spec['description'] case final String expectedDescription) { + expect(rootSemantics.properties.hint, expectedDescription); + } + if (spec['live'] case final String expectedLive) { + final bool shouldBeLive = + expectedLive == 'polite' || expectedLive == 'assertive'; + expect(rootSemantics.properties.liveRegion ?? false, shouldBeLive); + } + if (spec['hidden'] case final bool expectedHidden) { + expect(rootSemantics.properties.hidden ?? false, expectedHidden); + } + if (spec['role'] case final String expectedRole) { + expect(_hasRole(subtreeData, expectedRole), isTrue); + } + if (spec['checked'] case final bool expectedChecked) { + final CheckedState expectedState = expectedChecked + ? CheckedState.isTrue + : CheckedState.isFalse; + expect( + subtreeData.any((d) => d.flagsCollection.isChecked == expectedState), + isTrue, + ); + } +} + +bool _hasRole(List data, String role) => switch (role) { + 'button' => data.any((d) => d.flagsCollection.isButton), + 'textbox' => data.any((d) => d.flagsCollection.isTextField), + 'checkbox' => data.any( + (d) => d.flagsCollection.isChecked != CheckedState.none, + ), + 'radiogroup' => data.any((d) => d.flagsCollection.isInMutuallyExclusiveGroup), + _ => false, +}; diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift index bda71e67e0..b67c27c3b3 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift @@ -13,16 +13,9 @@ // limitations under the License. import A2UICore -import A2UIJSON import A2UISwiftUI import BasicCatalog -import BasicCatalogSwiftUI -import Combine import Foundation -import JSONSchema -import OrderedCollections -import OrderedJSON -import SwiftUI import Testing import Yams @@ -30,101 +23,6 @@ import Yams /// against `A2UISwiftUI.Surface`, `SurfaceViewModel`, and `A2UIAccessibilityModifier`. @MainActor struct AdapterConformanceTests { - private typealias Node = A2UICore.Node - - private final class AdapterTestActionHandler: ActionHandling, @unchecked Sendable { - var actions: [(action: ResolvedAction, surfaceID: String)] = [] - var errors: [RendererError] = [] - - func handle(action: ResolvedAction, from surfaceID: String) { - actions.append((action: action, surfaceID: surfaceID)) - } - - func handle(error: RendererError, from surfaceID: String) { - errors.append(error) - } - - func clearStepEvents() { - actions.removeAll() - errors.removeAll() - } - } - - private final class IncrementCounterFunction: FunctionImplementation, Sendable { - let api: FunctionAPI - - init() throws { - self.api = FunctionAPI( - name: "incrementCounter", - returnType: .number, - schema: try Schema(instance: "{}") - ) - } - - @MainActor - func evaluate(arguments: [String: JSONValue], context: DataContext) throws -> JSONValue { - guard let path = arguments["path"]?.stringValue else { return .null } - let step = arguments["step"]?.intValue ?? 1 - let current = context.dataModel.get(path)?.intValue ?? 0 - let updated = current + step - context.set(path, value: .integer(updated)) - return .integer(updated) - } - } - - private struct ViewNode { - let id: String - let instanceID: String - let type: String? - let state: String - let node: Node? - let children: [ViewNode] - - init( - id: String, - instanceID: String, - type: String? = nil, - state: String = "resolved", - node: Node? = nil, - children: [ViewNode] = [] - ) { - self.id = id - self.instanceID = instanceID - self.type = type - self.state = state - self.node = node - self.children = children - } - } - - @MainActor - private final class MountedSurfaceHarness { - let surface: Surface - private var subscriptions: [AnyCancellable] = [] - private(set) var isMounted = true - private(set) var emissionCount = 0 - - var activeSubscriptionCount: Int { - isMounted ? subscriptions.count : 0 - } - - init(surface: Surface) { - self.surface = surface - surface.viewModel.objectWillChange - .sink { [weak self] _ in self?.emissionCount += 1 } - .store(in: &subscriptions) - surface.viewModel.componentsModel.objectWillChange - .sink { [weak self] _ in self?.emissionCount += 1 } - .store(in: &subscriptions) - } - - func unmount() { - isMounted = false - subscriptions.removeAll() - surface.viewModel.dispose() - } - } - /// Adapter conformance cases that depend on runtime features not yet implemented in `A2UICore` / /// `A2UISwiftUI`, recorded as expected failures with their root-cause reason. private static let expectedAdapterFailures: [String: String] = [ @@ -161,31 +59,96 @@ struct AdapterConformanceTests { } @Test func accessibilityConformanceThroughModifier() throws { - let cases = try Self.loadYAMLCases("core/accessibility.yaml") + let suite = "core/accessibility.yaml" + let cases = try Self.loadYAMLCases(suite) #expect(!cases.isEmpty, "core/accessibility.yaml should contain test cases") var checkedCount = 0 for testCase in cases { let name = testCase["name"] as? String ?? "" - checkedCount += try runAccessibilityCase(testCase, name: name) + do { + try AdapterHarness.assertKnownAccessibilityCase(testCase) + checkedCount += try runAccessibilityCase(testCase, name: name) + writeManifestRecord( + AdapterManifestRecord(suite: suite, caseName: name, status: "passed") + ) + } catch { + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error + } } #expect(checkedCount > 0, "accessibility.yaml should verify at least one component") } @Test func adapterConformanceSuite() throws { - let cases = try Self.loadYAMLCases("core/adapter_conformance.yaml") + let suite = "core/adapter_conformance.yaml" + let cases = try Self.loadYAMLCases(suite) #expect(cases.count == 13, "core/adapter_conformance.yaml should contain 13 test cases") + var matchedExpectedFailures: Set = [] for testCase in cases { let name = testCase["name"] as? String ?? "" + do { + try AdapterHarness.assertKnownAdapterCase(testCase) + } catch { + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error + } + if let reason = Self.expectedAdapterFailures[name] { + matchedExpectedFailures.insert(name) withKnownIssue("\(name): \(reason)") { try runAdapterCase(testCase, name: name) } + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "xfailed", + reason: reason + ) + ) } else { - try runAdapterCase(testCase, name: name) + do { + try runAdapterCase(testCase, name: name) + writeManifestRecord( + AdapterManifestRecord(suite: suite, caseName: name, status: "passed") + ) + } catch { + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error + } } } + + let unusedExpectedFailures = Self.expectedAdapterFailures.keys + .filter { !matchedExpectedFailures.contains($0) } + .sorted() + #expect( + unusedExpectedFailures.isEmpty, + "Unused expectedAdapterFailures entries: \(unusedExpectedFailures)" + ) } // MARK: - Accessibility Suite Runner @@ -197,7 +160,7 @@ struct AdapterConformanceTests { actionHandler: handler, validationConfig: .relaxed ) - let surfaceID = testCase["surfaceId"] as? String ?? "a11y_surface" + let surfaceID = "a11y_surface" if let steps = testCase["steps"] as? [[String: Any]] { return try runAccessibilitySteps( steps, @@ -207,13 +170,13 @@ struct AdapterConformanceTests { ) } if let messages = testCase["messages"] { - try processor.processMessages(toJSONValue(messages)) + try processor.processMessages(AdapterHarness.toJSONValue(messages)) } let assertions = (testCase["assertions"] as? [String: Any]) ?? testCase guard let expected = assertions["accessibilityTree"] as? [String: [String: Any]] else { return 0 } - return assertAccessibilityNodes( + return try assertAccessibilityNodes( expected, processor: processor, surfaceID: surfaceID, @@ -230,11 +193,11 @@ struct AdapterConformanceTests { var checked = 0 for (idx, step) in steps.enumerated() { if let stepMessages = step["messages"] { - try processor.processMessages(toJSONValue(stepMessages)) + try processor.processMessages(AdapterHarness.toJSONValue(stepMessages)) } let assertions = (step["assertions"] as? [String: Any]) ?? step if let expected = assertions["accessibilityTree"] as? [String: [String: Any]] { - checked += assertAccessibilityNodes( + checked += try assertAccessibilityNodes( expected, processor: processor, surfaceID: surfaceID, @@ -250,26 +213,26 @@ struct AdapterConformanceTests { processor: MessageProcessor, surfaceID: String, context: String - ) -> Int { + ) throws -> Int { guard let viewModel = processor.surfaceGroupModel.surfacesMap[surfaceID] ?? processor.surfaceGroupModel.surfacesMap.values.first, let rootNode = viewModel.rootNode else { Issue.record("\(context): missing resolved root node for \(surfaceID)") - return 0 + throw AdapterConformanceError("\(context): missing resolved root node for \(surfaceID)") } - var nodesByID: [String: Node] = [:] - collectNodes(from: rootNode, into: &nodesByID) + var nodesByID: [String: A2UICore.Node] = [:] + AdapterHarness.collectNodes(from: rootNode, into: &nodesByID) var checked = 0 for (componentID, expectedProps) in expected { guard let node = nodesByID[componentID] else { Issue.record("\(context): node '\(componentID)' not found") - continue + throw AdapterConformanceError("\(context): node '\(componentID)' not found") } let modifier = A2UIAccessibilityModifier(node: node) - assertModifierMatches( + try AdapterHarness.assertModifierMatches( modifier, node: node, expected: expectedProps, @@ -280,596 +243,14 @@ struct AdapterConformanceTests { return checked } - private func assertModifierMatches( - _ modifier: A2UIAccessibilityModifier, - node: Node, - expected: [String: Any], - context: String - ) { - if let expLabel = expected["label"] as? String { - #expect(effectiveLabel(for: node, modifier: modifier) == expLabel, "\(context): label") - } - if let expDesc = expected["description"] as? String { - #expect(modifier.hint == expDesc, "\(context): description mismatch") - } - if let expLive = expected["live"] as? String { - #expect(node.explicitAccessibilityAttributes?.live == expLive, "\(context): live mismatch") - } - if let expHidden = expected["hidden"] as? Bool { - #expect(modifier.isHidden == expHidden, "\(context): hidden mismatch") - } - if let expRole = expected["role"] as? String { - #expect(semanticRole(for: node) == expRole, "\(context): role mismatch") - } - if let expChecked = expected["checked"] as? Bool { - #expect(checkedState(for: node) == expChecked, "\(context): checked mismatch") - } - } - - private func effectiveLabel(for node: Node, modifier: A2UIAccessibilityModifier) -> String? { - if let explicit = modifier.label { - return explicit - } - if let inferred = node.accessibilityAttributes?.label, !inferred.isEmpty { - return inferred - } - if let child = node.child(for: "child") { - return child.accessibilityAttributes?.label ?? child.string(for: "text") - } - return nil - } - - private func semanticRole(for node: Node) -> String { - switch node.type { - case "Button": - return "button" - case "TextField": - return "textbox" - case "CheckBox": - return "checkbox" - case "ChoicePicker": - let variant = node.string(for: "variant") ?? "mutuallyExclusive" - return variant == "mutuallyExclusive" ? "radiogroup" : "group" - default: - return node.type.lowercased() - } - } - - private func checkedState(for node: Node) -> Bool? { - guard node.type == "CheckBox" else { return nil } - return node.bool(for: "value") ?? false - } - // MARK: - Adapter Conformance Suite Runner private func runAdapterCase(_ testCase: [String: Any], name: String) throws { - let handler = AdapterTestActionHandler() - let (coreCatalogs, uiCatalogs) = try buildCatalogs(for: testCase) - let processor = MessageProcessor( - catalogs: coreCatalogs, - actionHandler: handler, - validationConfig: .relaxed - ) + let session = try mountSurface(testCase) let steps = (testCase["steps"] as? [[String: Any]]) ?? [] - var harness: MountedSurfaceHarness? - var previousTree: ViewNode? - for (idx, step) in steps.enumerated() { let stepCtx = "\(name) [step \(idx)]" - handler.clearStepEvents() - let currentTree = try executeAdapterStep( - step, - processor: processor, - uiCatalogs: uiCatalogs, - handler: handler, - harness: &harness - ) - verifyAdapterStep( - step, - harness: harness, - currentTree: currentTree, - previousTree: previousTree, - handler: handler, - context: stepCtx - ) - if let currentTree { - previousTree = currentTree - } - } - } - - private func buildCatalogs( - for testCase: [String: Any] - ) throws -> ([AnyCatalog], [Catalog]) { - let incrementFn = try IncrementCounterFunction() - let v10Functions = V10BasicCatalog.functions + [incrementFn] - var coreCatalogs: [AnyCatalog] = BasicCatalog.allCatalogs.map { cat in - Catalog( - id: cat.id, - protocolVersion: cat.protocolVersion, - components: Array(cat.components.values), - functions: Array(cat.functions.values) + [incrementFn], - themeSchema: cat.themeSchema - ).eraseToAnyCatalog() - } - var uiCatalogs: [Catalog] = BasicCatalogImplementation.allCatalogs - let customDefs = (testCase["catalogs"] as? [[String: Any]]) ?? [] - for custom in customDefs { - if let (coreCat, uiCat) = try buildCustomCatalog(custom, functions: v10Functions) { - coreCatalogs.append(coreCat) - uiCatalogs.append(uiCat) - } - } - return (coreCatalogs, uiCatalogs) - } - - private func buildCustomCatalog( - _ custom: [String: Any], - functions: [any FunctionImplementation] - ) throws -> (AnyCatalog, Catalog)? { - guard let catalogID = custom["catalogId"] as? String, - let compDict = custom["components"] as? [String: Any] - else { - return nil - } - let emptySchema = try Schema(instance: "{}") - var coreApis: [AnyComponentAPI] = [] - var uiImpls: [ComponentImplementation] = [] - for compName in compDict.keys { - let api = AnyComponentAPI(name: compName, schema: emptySchema) - coreApis.append(api) - uiImpls.append( - ComponentImplementation(api: api) { node in - Text(node.string(for: "badgeText") ?? node.string(for: "text") ?? "") - } - ) - } - let coreCat = Catalog( - id: catalogID, - protocolVersion: .v10, - components: coreApis, - functions: functions - ).eraseToAnyCatalog() - let uiCat = Catalog( - id: catalogID, - protocolVersion: .v10, - components: uiImpls, - functions: functions - ) - return (coreCat, uiCat) - } - - private func executeAdapterStep( - _ step: [String: Any], - processor: MessageProcessor, - uiCatalogs: [Catalog], - handler: AdapterTestActionHandler, - harness: inout MountedSurfaceHarness? - ) throws -> ViewNode? { - let op = step["op"] as? String ?? "process_messages" - switch op { - case "process_messages": - let messages = (step["messages"] as? [[String: Any]]) ?? [] - try processAdapterMessages( - messages, - processor: processor, - uiCatalogs: uiCatalogs, - harness: &harness - ) - case "user_input": - applyUserInput(step, surface: harness?.surface) - case "user_gesture", "programmatic_invoke": - invokeStepAction(step, surface: harness?.surface) - case "unmount_surface": - #expect((harness?.activeSubscriptionCount ?? 0) > 0, "Expected active subscriptions") - harness?.unmount() - return nil - default: - break - } - guard let surface = harness?.surface else { return nil } - return resolveViewTree(for: surface) - } - - private func processAdapterMessages( - _ messages: [[String: Any]], - processor: MessageProcessor, - uiCatalogs: [Catalog], - harness: inout MountedSurfaceHarness? - ) throws { - try processor.processMessages(toJSONValue(messages)) - if harness == nil, - let vm = processor.surfaceGroupModel.surfacesMap["adapter_surface"] - ?? processor.surfaceGroupModel.surfacesMap.values.first - { - let surface = Surface( - viewModel: vm, - catalogs: uiCatalogs, - defaultCatalogID: BasicCatalog.catalogURI(version: .v10) - ) - harness = MountedSurfaceHarness(surface: surface) - } - } - - private func applyUserInput(_ step: [String: Any], surface: Surface?) { - guard let surface, - let targetID = step["targetNode"] as? String, - let inputValue = step["inputValue"] as? String, - let rootNode = surface.viewModel.rootNode - else { - return - } - var nodes: [String: Node] = [:] - collectNodes(from: rootNode, into: &nodes) - guard let targetNode = nodes[targetID] else { return } - let textField = A2UITextField(node: targetNode) - textField.valueBinding.wrappedValue = inputValue - } - - private func invokeStepAction(_ step: [String: Any], surface: Surface?) { - guard let surface, - let targetID = step["targetNode"] as? String, - let rootNode = surface.viewModel.rootNode - else { - return - } - var nodes: [String: Node] = [:] - collectNodes(from: rootNode, into: &nodes) - guard let targetNode = nodes[targetID] else { return } - invokeAction(on: targetNode) - } - - private func invokeAction( - on node: Node, - property: String = "action" - ) { - if node.type == "Button" { - let button = A2UIButton(node: node) - guard !button.isDisabled else { return } - } else { - guard node.isValid else { return } - } - node.action(for: property)?() - } - - // MARK: - View Tree Resolution Helpers - - private func resolveViewTree(for surface: Surface) -> ViewNode? { - guard let rootNode = surface.viewModel.rootNode else { return nil } - return buildViewNode(in: surface, node: rootNode) - } - - private func buildViewNode(in surface: Surface, node: Node) -> ViewNode { - let rendered = Surface.render( - node: node, - using: surface.catalogs, - defaultCatalogID: surface.defaultCatalogID - ) - #expect(rendered != nil, "Expected Surface.render builder for '\(node.type)'") - let children = resolveChildViewNodes(in: surface, node: node) - let baseID = baseComponentID(for: node, in: surface) - return ViewNode( - id: baseID, - instanceID: node.id, - type: node.type, - state: "resolved", - node: node, - children: children - ) - } - - private func baseComponentID(for node: Node, in surface: Surface) -> String { - if surface.viewModel.componentsModel.get(node.id) != nil { - return node.id - } - if let colonPrefix = node.id.split(separator: ":").first.map(String.init), - surface.viewModel.componentsModel.get(colonPrefix) != nil - { - return colonPrefix - } - var candidate = node.id - while let underscoreIdx = candidate.lastIndex(of: "_") { - let suffix = candidate[candidate.index(after: underscoreIdx)...] - guard !suffix.isEmpty, suffix.allSatisfy(\.isNumber) else { break } - candidate = String(candidate[.. [ViewNode] { - if let singleChild = node.child(for: "child") { - return [buildViewNode(in: surface, node: singleChild)] - } - return node.children(for: "children").map { childNode in - buildViewNode(in: surface, node: childNode) - } - } - - // MARK: - Step Verification Helpers - - private func verifyAdapterStep( - _ step: [String: Any], - harness: MountedSurfaceHarness?, - currentTree: ViewNode?, - previousTree: ViewNode?, - handler: AdapterTestActionHandler, - context: String - ) { - if let expTree = step["expectViewTree"] as? [String: Any] { - assertViewNodeMatches(currentTree, expected: expTree, context: context) - } - if let expData = step["expectDataModel"], let surface = harness?.surface { - let expectedJSON = toJSONValue(expData) - #expect(surface.viewModel.dataModel.data == expectedJSON, "\(context): dataModel mismatch") - } - if let expSameIDs = step["expectSameViewIdentity"] as? [String] { - assertSameViewIdentities( - expSameIDs, - previousTree: previousTree, - currentTree: currentTree, - context: context - ) - } - if let expActions = step["expectActions"] as? [[String: Any]] { - assertActionsMatch(handler.actions, expected: expActions, context: context) - } - if let expErrors = step["expectErrors"] as? [[String: Any]] { - assertErrorsMatch(handler.errors, expected: expErrors, context: context) - } - if let expA11y = step["expectAccessibilityTree"] as? [String: [String: Any]], let currentTree { - assertAccessibilityTreeMatches(currentTree, expected: expA11y, context: context) - } - if let expSubs = step["expectActiveSubscriptions"] as? Int, let harness { - assertActiveSubscriptions(harness: harness, expected: expSubs, context: context) - } - } - - private func assertActiveSubscriptions( - harness: MountedSurfaceHarness, - expected: Int, - context: String - ) { - let emissionsBefore = harness.emissionCount - var nodeBoxEmissions = 0 - var boxCancellables: [AnyCancellable] = [] - if let rootNode = harness.surface.viewModel.rootNode { - var nodes: [String: Node] = [:] - collectNodes(from: rootNode, into: &nodes) - for node in nodes.values { - node.box.objectWillChange - .sink { _ in nodeBoxEmissions += 1 } - .store(in: &boxCancellables) - } - } - harness.surface.viewModel.dataModel.set("/status", value: .string("Unmounted")) - let postWriteEmissions = (harness.emissionCount - emissionsBefore) + nodeBoxEmissions - withExtendedLifetime(boxCancellables) {} - #expect( - harness.activeSubscriptionCount + postWriteEmissions == expected, - "\(context): subscription count mismatch" - ) - } - - private func assertViewNodeMatches( - _ viewNode: ViewNode?, - expected: [String: Any], - context: String - ) { - guard let viewNode else { - Issue.record("\(context): expected ViewNode but got nil") - return - } - if let expID = expected["id"] as? String { - #expect(viewNode.id == expID, "\(context): id mismatch") - } - if let expType = expected["type"] as? String { - #expect(viewNode.type == expType, "\(context): type mismatch") - } - if let expState = expected["state"] as? String { - #expect(viewNode.state == expState, "\(context): state mismatch") - } - assertViewNodeProperties(viewNode, expected: expected, context: context) - if let expChildren = expected["children"] as? [[String: Any]] { - #expect(viewNode.children.count == expChildren.count, "\(context): children count mismatch") - for (idx, expChild) in expChildren.enumerated() where idx < viewNode.children.count { - assertViewNodeMatches( - viewNode.children[idx], - expected: expChild, - context: "\(context)/children/\(idx)" - ) - } - } - } - - private func assertViewNodeProperties( - _ viewNode: ViewNode, - expected: [String: Any], - context: String - ) { - guard let node = viewNode.node else { return } - if let expText = expected["text"] as? String { - #expect(node.string(for: "text") == expText, "\(context): text mismatch") - } - if let expValue = expected["value"] as? String { - #expect(node.string(for: "value") == expValue, "\(context): value mismatch") - } - if let expBadge = expected["badgeText"] as? String { - #expect(node.string(for: "badgeText") == expBadge, "\(context): badgeText mismatch") - } - assertValidationProperties(node, expected: expected, context: context) - } - - private func assertValidationProperties( - _ node: Node, - expected: [String: Any], - context: String - ) { - let textField = A2UITextField(node: node) - if let expValid = expected["isValid"] as? Bool { - #expect(textField.isValid == expValid, "\(context): isValid mismatch") - } - if let expErrors = expected["validationErrors"] as? [String] { - #expect(textField.validationErrors == expErrors, "\(context): validationErrors mismatch") - } - guard let expResults = expected["validationResults"] as? [[String: Any]] else { return } - #expect(textField.validationResults.count == expResults.count, "\(context): results count") - for (idx, expRes) in expResults.enumerated() where idx < textField.validationResults.count { - let actual = textField.validationResults[idx] - if let expCode = expRes["code"] as? String { - #expect(actual.code == expCode, "\(context): result code mismatch") - } - if let expMsg = expRes["message"] as? String { - #expect(actual.message == expMsg, "\(context): result message mismatch") - } - if let expSev = expRes["severity"] as? String { - let actualSeverity = (actual.severity ?? ValidationSeverity.error).rawValue - #expect(actualSeverity == expSev, "\(context): result severity mismatch") - } - } - } - - private func assertSameViewIdentities( - _ paths: [String], - previousTree: ViewNode?, - currentTree: ViewNode?, - context: String - ) { - for path in paths { - let prevNode = findViewNode(atPath: path, in: previousTree) - let currNode = findViewNode(atPath: path, in: currentTree) - #expect(prevNode != nil && currNode != nil, "\(context): missing node at \(path)") - #expect( - prevNode?.instanceID == currNode?.instanceID, - "\(context): identity changed at \(path)" - ) - } - } - - private func findViewNode(atPath path: String, in root: ViewNode?) -> ViewNode? { - guard let root else { return nil } - if path == "root" || path == root.id { - return root - } - let parts = path.split(separator: "/").map(String.init) - if parts.count == 3, parts[0] == "root", parts[1] == "children", let idx = Int(parts[2]), - idx < root.children.count - { - return root.children[idx] - } - return nil - } - - private func assertActionsMatch( - _ actual: [(action: ResolvedAction, surfaceID: String)], - expected: [[String: Any]], - context: String - ) { - #expect(actual.count == expected.count, "\(context): action count mismatch") - for (idx, exp) in expected.enumerated() where idx < actual.count { - let entry = actual[idx] - if let sid = exp["surfaceId"] as? String { - #expect(entry.surfaceID == sid, "\(context): action surfaceId mismatch") - } - if let src = exp["sourceComponentId"] as? String { - #expect(entry.action.sourceComponentID == src, "\(context): sourceComponentId mismatch") - } - guard case .event(let actName, let actCtx, _) = entry.action.identity else { - Issue.record("\(context): expected .event action identity") - continue - } - if let name = exp["name"] as? String { - #expect(actName == name, "\(context): action name mismatch") - } - if let expCtx = exp["context"] as? [String: Any] { - for (k, v) in expCtx { - #expect(actCtx?[k] == toJSONValue(v), "\(context): action context[\(k)] mismatch") - } - } - } - } - - private func assertErrorsMatch( - _ actual: [RendererError], - expected: [[String: Any]], - context: String - ) { - #expect(actual.count >= expected.count, "\(context): expected \(expected.count) errors") - for exp in expected { - let expCode = exp["code"] as? String - let expSurface = exp["surfaceId"] as? String - let expMessage = (exp["messageContains"] ?? exp["message"]) as? String - let matched = actual.contains { err in - let fields = rendererErrorFields(err) - let codeMatch = expCode == nil || fields.code == expCode - let surfMatch = expSurface == nil || fields.surfaceID == expSurface - let msgMatch = expMessage == nil || fields.message.contains(expMessage ?? "") - return codeMatch && surfMatch && msgMatch - } - #expect(matched, "\(context): expected error matching \(exp) in \(actual)") - } - } - - private func rendererErrorFields( - _ error: RendererError - ) -> (code: String, surfaceID: String?, message: String) { - switch error { - case .validationFailed(let valError): - return (valError.code.rawValue, valError.surfaceID, valError.message) - case .generic(let genError): - return (genError.code, genError.surfaceID, genError.message) - } - } - - private func assertAccessibilityTreeMatches( - _ root: ViewNode, - expected: [String: [String: Any]], - context: String - ) { - var nodesByID: [String: Node] = [:] - if let rootNode = root.node { - collectNodes(from: rootNode, into: &nodesByID) - } - for (componentID, expectedProps) in expected { - guard let node = nodesByID[componentID] else { - Issue.record("\(context): missing accessibility node '\(componentID)'") - continue - } - let modifier = A2UIAccessibilityModifier(node: node) - assertModifierMatches( - modifier, - node: node, - expected: expectedProps, - context: "\(context) [\(componentID)]" - ) - } - } - - private func collectNodes(from node: Node, into map: inout [String: Node]) { - map[node.id] = node - let baseID = String(node.id.split(separator: ":").first ?? Substring(node.id)) - map[baseID] = node - for child in node.allChildNodes { - collectNodes(from: child, into: &map) - } - } - - private func toJSONValue(_ value: Any) -> JSONValue { - if value is NSNull { return .null } - if let boolVal = value as? Bool { return .boolean(boolVal) } - if let intVal = value as? Int { return .integer(intVal) } - if let doubleVal = value as? Double { return .number(doubleVal) } - if let strVal = value as? String { return .string(strVal) } - if let arrVal = value as? [Any] { return .array(arrVal.map { toJSONValue($0) }) } - if let dictVal = value as? [String: Any] { - var obj = OrderedDictionary() - for key in dictVal.keys.sorted() { - obj[key] = toJSONValue(dictVal[key] as Any) - } - return .object(obj) + try AdapterHarness.runStep(session, step, context: stepCtx) } - return .null } } diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift new file mode 100644 index 0000000000..b0d3abd018 --- /dev/null +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift @@ -0,0 +1,1222 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import A2UICore +import A2UIJSON +import A2UISwiftUI +import BasicCatalog +import BasicCatalogSwiftUI +import Combine +import Foundation +import JSONSchema +import OrderedCollections +import OrderedJSON +import SwiftUI +import Testing + +/// Error thrown when an adapter conformance schema or assertion check fails. +struct AdapterConformanceError: Error, CustomStringConvertible, Sendable { + let description: String + + init(_ description: String) { + self.description = description + } +} + +/// Execution record written to the JSON Lines conformance manifest. +struct AdapterManifestRecord: Sendable { + let harness: String + let suite: String + let caseName: String + let status: String + let reason: String? + + init( + harness: String = "swiftui", + suite: String, + caseName: String, + status: String, + reason: String? = nil + ) { + self.harness = harness + self.suite = suite + self.caseName = caseName + self.status = status + self.reason = reason + } +} + +/// Captures actions and errors emitted by `MessageProcessor` during a conformance step. +final class AdapterTestActionHandler: ActionHandling, @unchecked Sendable { + var actions: [(action: ResolvedAction, surfaceID: String)] = [] + var errors: [RendererError] = [] + + func handle(action: ResolvedAction, from surfaceID: String) { + actions.append((action: action, surfaceID: surfaceID)) + } + + func handle(error: RendererError, from surfaceID: String) { + errors.append(error) + } + + func clearStepEvents() { + actions.removeAll() + errors.removeAll() + } +} + +/// Local `incrementCounter` function implementation used by adapter conformance tests. +final class IncrementCounterFunction: FunctionImplementation, Sendable { + let api: FunctionAPI + + init() throws { + self.api = FunctionAPI( + name: "incrementCounter", + returnType: .number, + schema: try Schema(instance: "{}") + ) + } + + @MainActor + func evaluate(arguments: [String: JSONValue], context: DataContext) throws -> JSONValue { + guard let path = arguments["path"]?.stringValue else { return .null } + let step = arguments["step"]?.intValue ?? 1 + let current = context.dataModel.get(path)?.intValue ?? 0 + let updated = current + step + context.set(path, value: .integer(updated)) + return .integer(updated) + } +} + +/// Snapshot of a rendered view node in the SwiftUI adapter tree. +struct AdapterViewNode { + let id: String + let instanceID: String + let type: String? + let state: String + let node: A2UICore.Node? + let children: [AdapterViewNode] + + init( + id: String, + instanceID: String, + type: String? = nil, + state: String = "resolved", + node: A2UICore.Node? = nil, + children: [AdapterViewNode] = [] + ) { + self.id = id + self.instanceID = instanceID + self.type = type + self.state = state + self.node = node + self.children = children + } +} + +/// Holds a mounted `A2UISwiftUI.Surface` and tracks its Combine subscriptions. +@MainActor +final class MountedSurfaceHarness { + let surface: Surface + private var subscriptions: [AnyCancellable] = [] + private(set) var isMounted = true + private(set) var emissionCount = 0 + + var activeSubscriptionCount: Int { + isMounted ? subscriptions.count : 0 + } + + init(surface: Surface) { + self.surface = surface + surface.viewModel.objectWillChange + .sink { [weak self] _ in self?.emissionCount += 1 } + .store(in: &subscriptions) + surface.viewModel.componentsModel.objectWillChange + .sink { [weak self] _ in self?.emissionCount += 1 } + .store(in: &subscriptions) + } + + func unmount() { + isMounted = false + subscriptions.removeAll() + surface.viewModel.dispose() + } +} + +/// Session state for an adapter conformance test case. +@MainActor +final class AdapterSession { + let caseName: String + let handler: AdapterTestActionHandler + let processor: MessageProcessor + let uiCatalogs: [Catalog] + var mountedHarness: MountedSurfaceHarness? + var currentTree: AdapterViewNode? + var previousTree: AdapterViewNode? + + init( + caseName: String, + handler: AdapterTestActionHandler, + processor: MessageProcessor, + uiCatalogs: [Catalog] + ) { + self.caseName = caseName + self.handler = handler + self.processor = processor + self.uiCatalogs = uiCatalogs + } +} + +/// Shared SwiftUI adapter conformance harness helpers. +@MainActor +enum AdapterHarness { + private static let manifestLock = NSLock() + + static let adapterCaseKnownKeys: Set = [ + "name", + "description", + "features", + "action", + "protocolVersion", + "catalogPaths", + "catalogs", + "steps", + ] + + static let adapterCatalogKnownKeys: Set = [ + "catalogId", + "protocolVersion", + "components", + ] + + static let adapterStepKnownKeys: Set = [ + "description", + "op", + "messages", + "targetNode", + "inputValue", + "userActivation", + "expectViewTree", + "expectAccessibilityTree", + "expectDataModel", + "expectActions", + "expectErrors", + "expectSameViewIdentity", + "expectActiveSubscriptions", + ] + + static let adapterStepKnownOps: Set = [ + "process_messages", + "user_input", + "user_gesture", + "programmatic_invoke", + "unmount_surface", + ] + + static let viewNodeKnownKeys: Set = [ + "id", + "type", + "state", + "text", + "value", + "badgeText", + "isValid", + "validationErrors", + "validationResults", + "children", + ] + + static let validationResultKnownKeys: Set = [ + "valid", + "code", + "message", + "severity", + ] + + static let expectedActionKnownKeys: Set = [ + "name", + "surfaceId", + "sourceComponentId", + "context", + ] + + static let expectedErrorKnownKeys: Set = [ + "code", + "surfaceId", + "message", + ] + + static let accessibilityCaseKnownKeys: Set = [ + "name", + "description", + "features", + "action", + "protocolVersion", + "catalogPath", + "surface", + "messages", + "assertions", + "steps", + ] + + static let accessibilityStepKnownKeys: Set = [ + "description", + "messages", + "assertions", + ] + + static let accessibilityAssertionsKnownKeys: Set = [ + "accessibilityTree", + "axeCore", + ] + + static let accessibilityNodeKnownKeys: Set = [ + "label", + "description", + "live", + "hidden", + "role", + "checked", + ] + + private static func assertDictionaryKeys( + _ dict: [String: Any], + allowed: Set, + where location: String + ) throws { + let unknown = dict.keys.filter { !allowed.contains($0) }.sorted() + if !unknown.isEmpty { + let msg = "\(location): unknown keys \(unknown)" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + } + + private static func assertKnownViewNodeKeys( + _ node: [String: Any], + where location: String + ) throws { + try assertDictionaryKeys(node, allowed: viewNodeKnownKeys, where: location) + if let results = node["validationResults"] as? [[String: Any]] { + for (idx, item) in results.enumerated() { + try assertDictionaryKeys( + item, + allowed: validationResultKnownKeys, + where: "\(location).validationResults[\(idx)]" + ) + } + } + if let children = node["children"] as? [[String: Any]] { + for (idx, child) in children.enumerated() { + try assertKnownViewNodeKeys(child, where: "\(location).children[\(idx)]") + } + } + } + + /// Validates that `step` (or `obj`) only uses keys in `allowed` and recursively checks + /// nested expectation structures when `allowed == adapterStepKnownKeys`. + static func assertKnownKeys( + _ obj: [String: Any], + allowed: Set = adapterStepKnownKeys, + where location: String = "step" + ) throws { + try assertDictionaryKeys(obj, allowed: allowed, where: location) + guard allowed == adapterStepKnownKeys else { return } + + if let rawOp = obj["op"] { + guard let op = rawOp as? String, adapterStepKnownOps.contains(op) else { + let msg = "\(location).op: unsupported op \(rawOp)" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + } + if let viewTree = obj["expectViewTree"] as? [String: Any] { + try assertKnownViewNodeKeys(viewTree, where: "\(location).expectViewTree") + } + if let a11yTree = obj["expectAccessibilityTree"] as? [String: [String: Any]] { + for (compID, nodeSpec) in a11yTree { + try assertDictionaryKeys( + nodeSpec, + allowed: accessibilityNodeKnownKeys, + where: "\(location).expectAccessibilityTree.\(compID)" + ) + } + } + if let actions = obj["expectActions"] as? [[String: Any]] { + for (idx, item) in actions.enumerated() { + try assertDictionaryKeys( + item, + allowed: expectedActionKnownKeys, + where: "\(location).expectActions[\(idx)]" + ) + } + } + if let errors = obj["expectErrors"] as? [[String: Any]] { + for (idx, item) in errors.enumerated() { + try assertDictionaryKeys( + item, + allowed: expectedErrorKnownKeys, + where: "\(location).expectErrors[\(idx)]" + ) + } + } + } + + static func assertKnownAdapterCase(_ testCase: [String: Any]) throws { + let name = testCase["name"] as? String ?? "" + try assertDictionaryKeys(testCase, allowed: adapterCaseKnownKeys, where: "case(\(name))") + if let action = testCase["action"] as? String, action != "render_surface" { + let msg = "case(\(name)): unsupported action \(action)" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + if let catalogs = testCase["catalogs"] as? [[String: Any]] { + for (idx, cat) in catalogs.enumerated() { + try assertDictionaryKeys( + cat, + allowed: adapterCatalogKnownKeys, + where: "case(\(name)).catalogs[\(idx)]" + ) + } + } + let steps = (testCase["steps"] as? [[String: Any]]) ?? [] + for (idx, step) in steps.enumerated() { + try assertKnownKeys( + step, + allowed: adapterStepKnownKeys, + where: "case(\(name)).steps[\(idx)]" + ) + } + } + + static func assertKnownAccessibilityAssertions( + _ assertions: [String: Any]?, + where location: String + ) throws { + guard let assertions else { return } + try assertDictionaryKeys( + assertions, + allowed: accessibilityAssertionsKnownKeys, + where: location + ) + if let tree = assertions["accessibilityTree"] as? [String: [String: Any]] { + for (compID, nodeSpec) in tree { + try assertDictionaryKeys( + nodeSpec, + allowed: accessibilityNodeKnownKeys, + where: "\(location).accessibilityTree.\(compID)" + ) + } + } + } + + static func assertKnownAccessibilityCase(_ testCase: [String: Any]) throws { + let name = testCase["name"] as? String ?? "" + try assertDictionaryKeys( + testCase, + allowed: accessibilityCaseKnownKeys, + where: "case(\(name))" + ) + if let action = testCase["action"] as? String, action != "accessibility_check" { + let msg = "case(\(name)): unsupported action \(action)" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + try assertKnownAccessibilityAssertions( + testCase["assertions"] as? [String: Any], + where: "case(\(name)).assertions" + ) + if let steps = testCase["steps"] as? [[String: Any]] { + for (idx, step) in steps.enumerated() { + try assertDictionaryKeys( + step, + allowed: accessibilityStepKnownKeys, + where: "case(\(name)).steps[\(idx)]" + ) + try assertKnownAccessibilityAssertions( + step["assertions"] as? [String: Any], + where: "case(\(name)).steps[\(idx)].assertions" + ) + } + } + } + + /// Appends a manifest record to `$A2UI_CONFORMANCE_MANIFEST_DIR/-.jsonl`. + nonisolated static func writeManifestRecord(_ rec: AdapterManifestRecord) { + guard + let manifestDir = ProcessInfo.processInfo.environment["A2UI_CONFORMANCE_MANIFEST_DIR"], + !manifestDir.isEmpty + else { + return + } + + var entry: [String: String] = [ + "harness": rec.harness, + "suite": rec.suite, + "case": rec.caseName, + "status": rec.status, + ] + if let reason = rec.reason, !reason.isEmpty { + entry["reason"] = reason + } + + guard + var lineData = try? JSONSerialization.data(withJSONObject: entry, options: [.sortedKeys]) + else { + return + } + lineData.append(0x0A) + + manifestLock.lock() + defer { manifestLock.unlock() } + + let dirURL = URL(fileURLWithPath: manifestDir, isDirectory: true) + try? FileManager.default.createDirectory(at: dirURL, withIntermediateDirectories: true) + let pid = ProcessInfo.processInfo.processIdentifier + let fileURL = dirURL.appendingPathComponent("\(rec.harness)-\(pid).jsonl") + + if FileManager.default.fileExists(atPath: fileURL.path) { + if let handle = try? FileHandle(forWritingTo: fileURL) { + defer { try? handle.close() } + _ = try? handle.seekToEnd() + try? handle.write(contentsOf: lineData) + } + } else { + try? lineData.write(to: fileURL, options: .atomic) + } + } + + /// Creates an `AdapterSession` configured for `caseSpec`. + static func mountSurface(_ caseSpec: [String: Any]) throws -> AdapterSession { + let caseName = caseSpec["name"] as? String ?? "" + let handler = AdapterTestActionHandler() + let (coreCatalogs, uiCatalogs) = try buildCatalogs(for: caseSpec) + let processor = MessageProcessor( + catalogs: coreCatalogs, + actionHandler: handler, + validationConfig: .relaxed + ) + return AdapterSession( + caseName: caseName, + handler: handler, + processor: processor, + uiCatalogs: uiCatalogs + ) + } + + /// Extracts the current resolved view tree from `session`. + static func extractViewTree(_ session: AdapterSession) -> AdapterViewNode? { + guard let surface = session.mountedHarness?.surface else { return nil } + return resolveViewTree(for: surface) + } + + /// Executes a single adapter conformance `step` against `session` and verifies expectations. + static func runStep( + _ session: AdapterSession, + _ step: [String: Any], + context: String? = nil + ) throws { + try assertKnownKeys(step) + let stepCtx = context ?? session.caseName + session.handler.clearStepEvents() + let op = step["op"] as? String ?? "process_messages" + switch op { + case "process_messages": + let messages = (step["messages"] as? [[String: Any]]) ?? [] + try processAdapterMessages(messages, session: session) + session.currentTree = extractViewTree(session) + case "user_input": + applyUserInput(step, surface: session.mountedHarness?.surface) + session.currentTree = extractViewTree(session) + case "user_gesture", "programmatic_invoke": + invokeStepAction(step, surface: session.mountedHarness?.surface) + session.currentTree = extractViewTree(session) + case "unmount_surface": + #expect( + (session.mountedHarness?.activeSubscriptionCount ?? 0) > 0, + "Expected active subscriptions" + ) + session.mountedHarness?.unmount() + session.currentTree = nil + default: + Issue.record("Unsupported op \(op)") + throw AdapterConformanceError("Unsupported op \(op)") + } + + try verifyAdapterStep(step, session: session, context: stepCtx) + if let currentTree = session.currentTree { + session.previousTree = currentTree + } + } + + /// Verifies that `actual` matches `expected`. + static func assertViewTree( + _ actual: AdapterViewNode?, + _ expected: [String: Any], + context: String = "root" + ) throws { + try assertKnownViewNodeKeys(expected, where: "\(context).expectViewTree") + guard let viewNode = actual else { + let msg = "\(context): expected ViewNode but got nil" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + if let expID = expected["id"] as? String { + #expect(viewNode.id == expID, "\(context): id mismatch") + if viewNode.id != expID { + throw AdapterConformanceError("\(context): id mismatch") + } + } + if let expType = expected["type"] as? String { + #expect(viewNode.type == expType, "\(context): type mismatch") + if viewNode.type != expType { + throw AdapterConformanceError("\(context): type mismatch") + } + } + if let expState = expected["state"] as? String { + #expect(viewNode.state == expState, "\(context): state mismatch") + if viewNode.state != expState { + throw AdapterConformanceError("\(context): state mismatch") + } + } + try assertViewNodeProperties(viewNode, expected: expected, context: context) + if let expChildren = expected["children"] as? [[String: Any]] { + #expect(viewNode.children.count == expChildren.count, "\(context): children count mismatch") + if viewNode.children.count != expChildren.count { + throw AdapterConformanceError("\(context): children count mismatch") + } + for (idx, expChild) in expChildren.enumerated() where idx < viewNode.children.count { + try assertViewTree( + viewNode.children[idx], + expChild, + context: "\(context)/children/\(idx)" + ) + } + } + } + + // MARK: - Internal Helpers + + private static func buildCatalogs( + for testCase: [String: Any] + ) throws -> ([AnyCatalog], [Catalog]) { + let incrementFn = try IncrementCounterFunction() + let v10Functions = V10BasicCatalog.functions + [incrementFn] + var coreCatalogs: [AnyCatalog] = BasicCatalog.allCatalogs.map { cat in + Catalog( + id: cat.id, + protocolVersion: cat.protocolVersion, + components: Array(cat.components.values), + functions: Array(cat.functions.values) + [incrementFn], + themeSchema: cat.themeSchema + ).eraseToAnyCatalog() + } + var uiCatalogs: [Catalog] = BasicCatalogImplementation.allCatalogs + let customDefs = (testCase["catalogs"] as? [[String: Any]]) ?? [] + for custom in customDefs { + if let (coreCat, uiCat) = try buildCustomCatalog(custom, functions: v10Functions) { + coreCatalogs.append(coreCat) + uiCatalogs.append(uiCat) + } + } + return (coreCatalogs, uiCatalogs) + } + + private static func buildCustomCatalog( + _ custom: [String: Any], + functions: [any FunctionImplementation] + ) throws -> (AnyCatalog, Catalog)? { + guard let catalogID = custom["catalogId"] as? String, + let compDict = custom["components"] as? [String: Any] + else { + return nil + } + let emptySchema = try Schema(instance: "{}") + var coreApis: [AnyComponentAPI] = [] + var uiImpls: [ComponentImplementation] = [] + for compName in compDict.keys { + let api = AnyComponentAPI(name: compName, schema: emptySchema) + coreApis.append(api) + uiImpls.append( + ComponentImplementation(api: api) { node in + Text(node.string(for: "badgeText") ?? node.string(for: "text") ?? "") + } + ) + } + let coreCat = Catalog( + id: catalogID, + protocolVersion: .v10, + components: coreApis, + functions: functions + ).eraseToAnyCatalog() + let uiCat = Catalog( + id: catalogID, + protocolVersion: .v10, + components: uiImpls, + functions: functions + ) + return (coreCat, uiCat) + } + + private static func processAdapterMessages( + _ messages: [[String: Any]], + session: AdapterSession + ) throws { + try session.processor.processMessages(toJSONValue(messages)) + if session.mountedHarness == nil, + let vm = session.processor.surfaceGroupModel.surfacesMap["adapter_surface"] + ?? session.processor.surfaceGroupModel.surfacesMap.values.first + { + let surface = Surface( + viewModel: vm, + catalogs: session.uiCatalogs, + defaultCatalogID: BasicCatalog.catalogURI(version: .v10) + ) + session.mountedHarness = MountedSurfaceHarness(surface: surface) + } + } + + private static func applyUserInput(_ step: [String: Any], surface: Surface?) { + guard let surface, + let targetID = step["targetNode"] as? String, + let inputValue = step["inputValue"] as? String, + let rootNode = surface.viewModel.rootNode + else { + return + } + var nodes: [String: A2UICore.Node] = [:] + collectNodes(from: rootNode, into: &nodes) + guard let targetNode = nodes[targetID] else { return } + let textField = A2UITextField(node: targetNode) + textField.valueBinding.wrappedValue = inputValue + } + + private static func invokeStepAction(_ step: [String: Any], surface: Surface?) { + guard let surface, + let targetID = step["targetNode"] as? String, + let rootNode = surface.viewModel.rootNode + else { + return + } + var nodes: [String: A2UICore.Node] = [:] + collectNodes(from: rootNode, into: &nodes) + guard let targetNode = nodes[targetID] else { return } + invokeAction(on: targetNode) + } + + private static func invokeAction( + on node: A2UICore.Node, + property: String = "action" + ) { + if node.type == "Button" { + let button = A2UIButton(node: node) + guard !button.isDisabled else { return } + } else { + guard node.isValid else { return } + } + node.action(for: property)?() + } + + static func resolveViewTree(for surface: Surface) -> AdapterViewNode? { + guard let rootNode = surface.viewModel.rootNode else { return nil } + return buildViewNode(in: surface, node: rootNode) + } + + private static func buildViewNode( + in surface: Surface, + node: A2UICore.Node + ) -> AdapterViewNode { + let rendered = Surface.render( + node: node, + using: surface.catalogs, + defaultCatalogID: surface.defaultCatalogID + ) + #expect(rendered != nil, "Expected Surface.render builder for '\(node.type)'") + let children = resolveChildViewNodes(in: surface, node: node) + let baseID = baseComponentID(for: node, in: surface) + return AdapterViewNode( + id: baseID, + instanceID: node.id, + type: node.type, + state: "resolved", + node: node, + children: children + ) + } + + private static func baseComponentID(for node: A2UICore.Node, in surface: Surface) -> String { + if surface.viewModel.componentsModel.get(node.id) != nil { + return node.id + } + if let colonPrefix = node.id.split(separator: ":").first.map(String.init), + surface.viewModel.componentsModel.get(colonPrefix) != nil + { + return colonPrefix + } + var candidate = node.id + while let underscoreIdx = candidate.lastIndex(of: "_") { + let suffix = candidate[candidate.index(after: underscoreIdx)...] + guard !suffix.isEmpty, suffix.allSatisfy(\.isNumber) else { break } + candidate = String(candidate[.. [AdapterViewNode] { + if let singleChild = node.child(for: "child") { + return [buildViewNode(in: surface, node: singleChild)] + } + return node.children(for: "children").map { childNode in + buildViewNode(in: surface, node: childNode) + } + } + + private static func verifyAdapterStep( + _ step: [String: Any], + session: AdapterSession, + context: String + ) throws { + if let expTree = step["expectViewTree"] as? [String: Any] { + try assertViewTree(session.currentTree, expTree, context: context) + } + if let expData = step["expectDataModel"], let surface = session.mountedHarness?.surface { + let expectedJSON = toJSONValue(expData) + #expect(surface.viewModel.dataModel.data == expectedJSON, "\(context): dataModel mismatch") + if surface.viewModel.dataModel.data != expectedJSON { + throw AdapterConformanceError("\(context): dataModel mismatch") + } + } + if let expSameIDs = step["expectSameViewIdentity"] as? [String] { + try assertSameViewIdentities( + expSameIDs, + previousTree: session.previousTree, + currentTree: session.currentTree, + context: context + ) + } + if let expActions = step["expectActions"] as? [[String: Any]] { + try assertActionsMatch(session.handler.actions, expected: expActions, context: context) + } + if let expErrors = step["expectErrors"] as? [[String: Any]] { + try assertErrorsMatch(session.handler.errors, expected: expErrors, context: context) + } + if let expA11y = step["expectAccessibilityTree"] as? [String: [String: Any]], + let currentTree = session.currentTree + { + try assertAccessibilityTreeMatches(currentTree, expected: expA11y, context: context) + } + if let expSubs = step["expectActiveSubscriptions"] as? Int, + let harness = session.mountedHarness + { + try assertActiveSubscriptions(harness: harness, expected: expSubs, context: context) + } + } + + private static func assertActiveSubscriptions( + harness: MountedSurfaceHarness, + expected: Int, + context: String + ) throws { + let emissionsBefore = harness.emissionCount + var nodeBoxEmissions = 0 + var boxCancellables: [AnyCancellable] = [] + if let rootNode = harness.surface.viewModel.rootNode { + var nodes: [String: A2UICore.Node] = [:] + collectNodes(from: rootNode, into: &nodes) + for node in nodes.values { + node.box.objectWillChange + .sink { _ in nodeBoxEmissions += 1 } + .store(in: &boxCancellables) + } + } + harness.surface.viewModel.dataModel.set("/status", value: .string("Unmounted")) + let postWriteEmissions = (harness.emissionCount - emissionsBefore) + nodeBoxEmissions + withExtendedLifetime(boxCancellables) {} + let total = harness.activeSubscriptionCount + postWriteEmissions + #expect(total == expected, "\(context): subscription count mismatch") + if total != expected { + throw AdapterConformanceError("\(context): subscription count mismatch") + } + } + + private static func assertViewNodeProperties( + _ viewNode: AdapterViewNode, + expected: [String: Any], + context: String + ) throws { + guard let node = viewNode.node else { return } + if let expText = expected["text"] as? String { + #expect(node.string(for: "text") == expText, "\(context): text mismatch") + if node.string(for: "text") != expText { + throw AdapterConformanceError("\(context): text mismatch") + } + } + if let expValue = expected["value"] as? String { + #expect(node.string(for: "value") == expValue, "\(context): value mismatch") + if node.string(for: "value") != expValue { + throw AdapterConformanceError("\(context): value mismatch") + } + } + if let expBadge = expected["badgeText"] as? String { + #expect(node.string(for: "badgeText") == expBadge, "\(context): badgeText mismatch") + if node.string(for: "badgeText") != expBadge { + throw AdapterConformanceError("\(context): badgeText mismatch") + } + } + try assertValidationProperties(node, expected: expected, context: context) + } + + private static func assertValidationProperties( + _ node: A2UICore.Node, + expected: [String: Any], + context: String + ) throws { + let textField = A2UITextField(node: node) + if let expValid = expected["isValid"] as? Bool { + #expect(textField.isValid == expValid, "\(context): isValid mismatch") + if textField.isValid != expValid { + throw AdapterConformanceError("\(context): isValid mismatch") + } + } + if let expErrors = expected["validationErrors"] as? [String] { + #expect(textField.validationErrors == expErrors, "\(context): validationErrors mismatch") + if textField.validationErrors != expErrors { + throw AdapterConformanceError("\(context): validationErrors mismatch") + } + } + guard let expResults = expected["validationResults"] as? [[String: Any]] else { return } + #expect(textField.validationResults.count == expResults.count, "\(context): results count") + if textField.validationResults.count != expResults.count { + throw AdapterConformanceError("\(context): results count mismatch") + } + for (idx, expRes) in expResults.enumerated() where idx < textField.validationResults.count { + let actual = textField.validationResults[idx] + if let expCode = expRes["code"] as? String { + #expect(actual.code == expCode, "\(context): result code mismatch") + if actual.code != expCode { + throw AdapterConformanceError("\(context): result code mismatch") + } + } + if let expMsg = expRes["message"] as? String { + #expect(actual.message == expMsg, "\(context): result message mismatch") + if actual.message != expMsg { + throw AdapterConformanceError("\(context): result message mismatch") + } + } + if let expSev = expRes["severity"] as? String { + let actualSeverity = (actual.severity ?? ValidationSeverity.error).rawValue + #expect(actualSeverity == expSev, "\(context): result severity mismatch") + if actualSeverity != expSev { + throw AdapterConformanceError("\(context): result severity mismatch") + } + } + } + } + + private static func assertSameViewIdentities( + _ paths: [String], + previousTree: AdapterViewNode?, + currentTree: AdapterViewNode?, + context: String + ) throws { + for path in paths { + let prevNode = findViewNode(atPath: path, in: previousTree) + let currNode = findViewNode(atPath: path, in: currentTree) + #expect(prevNode != nil && currNode != nil, "\(context): missing node at \(path)") + #expect( + prevNode?.instanceID == currNode?.instanceID, + "\(context): identity changed at \(path)" + ) + if prevNode == nil || currNode == nil || prevNode?.instanceID != currNode?.instanceID { + throw AdapterConformanceError("\(context): identity mismatch at \(path)") + } + } + } + + private static func findViewNode( + atPath path: String, + in root: AdapterViewNode? + ) -> AdapterViewNode? { + guard let root else { return nil } + if path == "root" || path == root.id { + return root + } + let parts = path.split(separator: "/").map(String.init) + if parts.count == 3, parts[0] == "root", parts[1] == "children", let idx = Int(parts[2]), + idx < root.children.count + { + return root.children[idx] + } + return nil + } + + private static func assertActionsMatch( + _ actual: [(action: ResolvedAction, surfaceID: String)], + expected: [[String: Any]], + context: String + ) throws { + #expect(actual.count == expected.count, "\(context): action count mismatch") + if actual.count != expected.count { + throw AdapterConformanceError("\(context): action count mismatch") + } + for (idx, exp) in expected.enumerated() where idx < actual.count { + let entry = actual[idx] + if let sid = exp["surfaceId"] as? String { + #expect(entry.surfaceID == sid, "\(context): action surfaceId mismatch") + if entry.surfaceID != sid { + throw AdapterConformanceError("\(context): action surfaceId mismatch") + } + } + if let src = exp["sourceComponentId"] as? String { + #expect(entry.action.sourceComponentID == src, "\(context): sourceComponentId mismatch") + if entry.action.sourceComponentID != src { + throw AdapterConformanceError("\(context): sourceComponentId mismatch") + } + } + guard case .event(let actName, let actCtx, _) = entry.action.identity else { + Issue.record("\(context): expected .event action identity") + throw AdapterConformanceError("\(context): expected .event action identity") + } + if let name = exp["name"] as? String { + #expect(actName == name, "\(context): action name mismatch") + if actName != name { + throw AdapterConformanceError("\(context): action name mismatch") + } + } + if let expCtx = exp["context"] as? [String: Any] { + for (k, v) in expCtx { + #expect(actCtx?[k] == toJSONValue(v), "\(context): action context[\(k)] mismatch") + if actCtx?[k] != toJSONValue(v) { + throw AdapterConformanceError("\(context): action context[\(k)] mismatch") + } + } + } + } + } + + private static func assertErrorsMatch( + _ actual: [RendererError], + expected: [[String: Any]], + context: String + ) throws { + #expect(actual.count >= expected.count, "\(context): expected \(expected.count) errors") + if actual.count < expected.count { + throw AdapterConformanceError("\(context): expected \(expected.count) errors") + } + for exp in expected { + let expCode = exp["code"] as? String + let expSurface = exp["surfaceId"] as? String + let expMessage = (exp["messageContains"] ?? exp["message"]) as? String + let matched = actual.contains { err in + let fields = rendererErrorFields(err) + let codeMatch = expCode == nil || fields.code == expCode + let surfMatch = expSurface == nil || fields.surfaceID == expSurface + let msgMatch = expMessage == nil || fields.message.contains(expMessage ?? "") + return codeMatch && surfMatch && msgMatch + } + #expect(matched, "\(context): expected error matching \(exp) in \(actual)") + if !matched { + throw AdapterConformanceError("\(context): expected error matching \(exp)") + } + } + } + + private static func rendererErrorFields( + _ error: RendererError + ) -> (code: String, surfaceID: String?, message: String) { + switch error { + case .validationFailed(let valError): + return (valError.code.rawValue, valError.surfaceID, valError.message) + case .generic(let genError): + return (genError.code, genError.surfaceID, genError.message) + } + } + + static func assertAccessibilityTreeMatches( + _ root: AdapterViewNode, + expected: [String: [String: Any]], + context: String + ) throws { + var nodesByID: [String: A2UICore.Node] = [:] + if let rootNode = root.node { + collectNodes(from: rootNode, into: &nodesByID) + } + for (componentID, expectedProps) in expected { + guard let node = nodesByID[componentID] else { + Issue.record("\(context): missing accessibility node '\(componentID)'") + throw AdapterConformanceError("\(context): missing accessibility node '\(componentID)'") + } + let modifier = A2UIAccessibilityModifier(node: node) + try assertModifierMatches( + modifier, + node: node, + expected: expectedProps, + context: "\(context) [\(componentID)]" + ) + } + } + + static func assertModifierMatches( + _ modifier: A2UIAccessibilityModifier, + node: A2UICore.Node, + expected: [String: Any], + context: String + ) throws { + if let expLabel = expected["label"] as? String { + let actual = effectiveLabel(for: node, modifier: modifier) + #expect(actual == expLabel, "\(context): label") + if actual != expLabel { + throw AdapterConformanceError("\(context): label mismatch") + } + } + if let expDesc = expected["description"] as? String { + #expect(modifier.hint == expDesc, "\(context): description mismatch") + if modifier.hint != expDesc { + throw AdapterConformanceError("\(context): description mismatch") + } + } + if let expLive = expected["live"] as? String { + let actual = node.explicitAccessibilityAttributes?.live + #expect(actual == expLive, "\(context): live mismatch") + if actual != expLive { + throw AdapterConformanceError("\(context): live mismatch") + } + } + if let expHidden = expected["hidden"] as? Bool { + #expect(modifier.isHidden == expHidden, "\(context): hidden mismatch") + if modifier.isHidden != expHidden { + throw AdapterConformanceError("\(context): hidden mismatch") + } + } + if let expRole = expected["role"] as? String { + let actual = semanticRole(for: node) + #expect(actual == expRole, "\(context): role mismatch") + if actual != expRole { + throw AdapterConformanceError("\(context): role mismatch") + } + } + if let expChecked = expected["checked"] as? Bool { + let actual = checkedState(for: node) + #expect(actual == expChecked, "\(context): checked mismatch") + if actual != expChecked { + throw AdapterConformanceError("\(context): checked mismatch") + } + } + } + + private static func effectiveLabel( + for node: A2UICore.Node, + modifier: A2UIAccessibilityModifier + ) -> String? { + if let explicit = modifier.label { + return explicit + } + if let inferred = node.accessibilityAttributes?.label, !inferred.isEmpty { + return inferred + } + if let child = node.child(for: "child") { + return child.accessibilityAttributes?.label ?? child.string(for: "text") + } + return nil + } + + private static func semanticRole(for node: A2UICore.Node) -> String { + switch node.type { + case "Button": + return "button" + case "TextField": + return "textbox" + case "CheckBox": + return "checkbox" + case "ChoicePicker": + let variant = node.string(for: "variant") ?? "mutuallyExclusive" + return variant == "mutuallyExclusive" ? "radiogroup" : "group" + default: + return node.type.lowercased() + } + } + + private static func checkedState(for node: A2UICore.Node) -> Bool? { + guard node.type == "CheckBox" else { return nil } + return node.bool(for: "value") ?? false + } + + static func collectNodes(from node: A2UICore.Node, into map: inout [String: A2UICore.Node]) { + map[node.id] = node + let baseID = String(node.id.split(separator: ":").first ?? Substring(node.id)) + map[baseID] = node + for child in node.allChildNodes { + collectNodes(from: child, into: &map) + } + } + + static func toJSONValue(_ value: Any) -> JSONValue { + if value is NSNull { return .null } + if let boolVal = value as? Bool { return .boolean(boolVal) } + if let intVal = value as? Int { return .integer(intVal) } + if let doubleVal = value as? Double { return .number(doubleVal) } + if let strVal = value as? String { return .string(strVal) } + if let arrVal = value as? [Any] { return .array(arrVal.map { toJSONValue($0) }) } + if let dictVal = value as? [String: Any] { + var obj = OrderedDictionary() + for key in dictVal.keys.sorted() { + obj[key] = toJSONValue(dictVal[key] as Any) + } + return .object(obj) + } + return .null + } +} + +/// Top-level helper creating an `AdapterSession` for a conformance test case. +@MainActor +func mountSurface(_ caseSpec: [String: Any]) throws -> AdapterSession { + try AdapterHarness.mountSurface(caseSpec) +} + +/// Top-level helper executing a single adapter conformance step. +@MainActor +func runStep(_ session: AdapterSession, _ step: [String: Any]) throws { + try AdapterHarness.runStep(session, step) +} + +/// Top-level helper extracting the current view tree from an `AdapterSession`. +@MainActor +func extractViewTree(_ session: AdapterSession) -> AdapterViewNode? { + AdapterHarness.extractViewTree(session) +} + +/// Top-level helper asserting an `AdapterViewNode` matches an expected view tree dictionary. +@MainActor +func assertViewTree(_ actual: AdapterViewNode?, _ expected: [String: Any]) throws { + try AdapterHarness.assertViewTree(actual, expected) +} + +/// Top-level helper validating that `step` only uses known adapter step and expectation keys. +@MainActor +func assertKnownKeys(_ step: [String: Any]) throws { + try AdapterHarness.assertKnownKeys(step) +} + +/// Top-level helper appending a conformance manifest record when `A2UI_CONFORMANCE_MANIFEST_DIR` +/// is set. +func writeManifestRecord(_ rec: AdapterManifestRecord) { + AdapterHarness.writeManifestRecord(rec) +} diff --git a/typescript/core/web_core/tests/conformance/accessibility_conformance.test.ts b/typescript/core/web_core/tests/conformance/accessibility_conformance.test.ts index d5b022293c..5a53899f63 100644 --- a/typescript/core/web_core/tests/conformance/accessibility_conformance.test.ts +++ b/typescript/core/web_core/tests/conformance/accessibility_conformance.test.ts @@ -17,11 +17,11 @@ import assert from 'node:assert/strict'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import {describe, it, before, after, afterEach} from 'node:test'; +import {after, afterEach, before, describe, it} from 'node:test'; import * as yaml from 'js-yaml'; -import {setupTestDom, teardownTestDom, asyncUpdate} from '../../src/test/dom-setup.js'; +import {asyncUpdate, setupTestDom, teardownTestDom} from '../../src/test/dom-setup.js'; import {render} from 'lit'; import {basicCatalog} from '../../src/catalogs/basic/v1/index.js'; @@ -58,6 +58,7 @@ interface AccessibilityAssertions { } interface AccessibilityStep { + description?: string; messages: ProcessableMessage[]; assertions: AccessibilityAssertions; } @@ -65,8 +66,11 @@ interface AccessibilityStep { interface AccessibilityTestCase { name: string; description?: string; + features?: string[]; protocolVersion: string; action: string; + catalogPath?: string; + surface?: Record; messages?: ProcessableMessage[]; assertions?: AccessibilityAssertions; steps?: AccessibilityStep[]; @@ -79,6 +83,131 @@ interface MountedHarness { dispose: () => void; } +interface ManifestRecord { + harness: string; + suite: string; + case: string; + status: 'passed' | 'failed' | 'skipped' | 'xfailed'; + reason?: string; +} + +/** + * Reasoned skip entries keyed by `""` or `"/"`. + * Every entry in this map must match at least one case during the run. + */ +const UNSUPPORTED: Record = {}; + +const matchedUnsupportedKeys = new Set(); + +const KNOWN_KEYS = { + accessibilityCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPath', + 'surface', + 'messages', + 'assertions', + 'steps', + ]), + accessibilityStep: new Set(['description', 'messages', 'assertions']), + accessibilityAssertions: new Set(['accessibilityTree', 'axeCore']), + accessibilityNode: new Set(['label', 'description', 'live', 'hidden', 'role', 'checked']), +} as const; + +function assertKnownKeys(obj: unknown, allowed: ReadonlySet, where: string): void { + if (obj === undefined || obj === null) { + return; + } + assert.ok(typeof obj === 'object' && !Array.isArray(obj), `${where}: expected object`); + const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); + assert.deepEqual( + unknown, + [], + `${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`, + ); +} + +function assertKnownAccessibilityAssertions( + assertions: AccessibilityAssertions | undefined, + where: string, +): void { + if (!assertions) { + return; + } + assertKnownKeys(assertions, KNOWN_KEYS.accessibilityAssertions, where); + for (const [compId, node] of Object.entries(assertions.accessibilityTree ?? {})) { + assertKnownKeys(node, KNOWN_KEYS.accessibilityNode, `${where}.accessibilityTree.${compId}`); + } +} + +function assertKnownAccessibilityCase(testCase: AccessibilityTestCase): void { + assertKnownKeys(testCase, KNOWN_KEYS.accessibilityCase, `case(${testCase.name})`); + assert.equal( + testCase.action, + 'accessibility_check', + `case(${testCase.name}): unsupported action "${testCase.action}"`, + ); + assertKnownAccessibilityAssertions(testCase.assertions, `case(${testCase.name}).assertions`); + for (const [idx, step] of (testCase.steps ?? []).entries()) { + assertKnownKeys(step, KNOWN_KEYS.accessibilityStep, `case(${testCase.name}).steps[${idx}]`); + assertKnownAccessibilityAssertions( + step.assertions, + `case(${testCase.name}).steps[${idx}].assertions`, + ); + } +} + +function findUnsupportedReason( + caseName: string, + steps: AccessibilityStep[] = [], +): string | undefined { + if (Object.hasOwn(UNSUPPORTED, caseName)) { + matchedUnsupportedKeys.add(caseName); + return UNSUPPORTED[caseName]; + } + for (const step of steps) { + for (const key of Object.keys(step.assertions ?? {})) { + const stepKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, stepKey)) { + matchedUnsupportedKeys.add(stepKey); + return UNSUPPORTED[stepKey]; + } + } + } + return undefined; +} + +function assertAllUnsupportedMatched(): void { + const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); + assert.deepEqual( + unused, + [], + `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, + ); +} + +function writeManifestRecord(rec: ManifestRecord): void { + const manifestDir = process.env.A2UI_CONFORMANCE_MANIFEST_DIR; + if (!manifestDir) { + return; + } + fs.mkdirSync(manifestDir, {recursive: true}); + const filePath = path.join(manifestDir, `${rec.harness}-${process.pid}.jsonl`); + const payload: Record = { + harness: rec.harness, + suite: rec.suite.replaceAll('\\', '/'), + case: rec.case, + status: rec.status, + }; + if (rec.reason) { + payload.reason = rec.reason; + } + fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, 'utf8'); +} + function registerBasicCatalogElements(): void { for (const comp of basicCatalog.components.values()) { registerUniversalElement(comp); @@ -284,6 +413,7 @@ describe('Web Universal Custom Elements Accessibility Conformance', () => { const testCases = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as AccessibilityTestCase[]; let activeHarness: MountedHarness | undefined; + let pendingManifest: ManifestRecord | undefined; before(() => { setupTestDom(); @@ -293,30 +423,57 @@ describe('Web Universal Custom Elements Accessibility Conformance', () => { afterEach(() => { activeHarness?.dispose(); activeHarness = undefined; + if (pendingManifest) { + writeManifestRecord(pendingManifest); + pendingManifest = undefined; + } }); after(() => { + assertAllUnsupportedMatched(); teardownTestDom(); }); for (const testCase of testCases) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - const processor = new MessageProcessor([basicCatalog]); - const steps: AccessibilityStep[] = testCase.steps ?? [ - { - messages: testCase.messages ?? [], - assertions: testCase.assertions ?? {}, - }, - ]; - - for (const step of steps) { - processor.processMessages(step.messages); - if (!activeHarness) { - const surface = processor.model.getSurface('a11y_surface'); - assert.ok(surface, 'Surface "a11y_surface" was not created'); - activeHarness = mountResolvedSurface(surface); + it(`${testCase.name}: ${testCase.description ?? ''}`, async t => { + pendingManifest = { + harness: 'web_core_universal', + suite: 'core/accessibility.yaml', + case: testCase.name, + status: 'failed', + }; + try { + assertKnownAccessibilityCase(testCase); + const steps: AccessibilityStep[] = testCase.steps ?? [ + { + messages: testCase.messages ?? [], + assertions: testCase.assertions ?? {}, + }, + ]; + const skipReason = findUnsupportedReason(testCase.name, steps); + if (skipReason) { + pendingManifest.status = 'skipped'; + pendingManifest.reason = skipReason; + t.skip(skipReason); + return; + } + + const processor = new MessageProcessor([basicCatalog]); + + for (const step of steps) { + processor.processMessages(step.messages); + if (!activeHarness) { + const surface = processor.model.getSurface('a11y_surface'); + assert.ok(surface, 'Surface "a11y_surface" was not created'); + activeHarness = mountResolvedSurface(surface); + } + await verifyAssertions(activeHarness.container, step.assertions); } - await verifyAssertions(activeHarness.container, step.assertions); + pendingManifest.status = 'passed'; + } catch (err) { + pendingManifest.status = 'failed'; + pendingManifest.reason = err instanceof Error ? err.message : String(err); + throw err; } }); } diff --git a/typescript/renderers/angular/karma.conf.js b/typescript/renderers/angular/karma.conf.js index f2e224cdc4..fb3fc8fea2 100644 --- a/typescript/renderers/angular/karma.conf.js +++ b/typescript/renderers/angular/karma.conf.js @@ -34,6 +34,72 @@ const basicCatalogJson = JSON.parse( fs.readFileSync(path.resolve(__dirname, '../../../catalogs/basic/v1/catalog.json'), 'utf8'), ); +function ConformanceManifestReporter(baseReporterDecorator) { + baseReporterDecorator(this); + const manifestDir = process.env.A2UI_CONFORMANCE_MANIFEST_DIR; + const manifestPath = manifestDir ? path.join(manifestDir, 'angular-karma.jsonl') : null; + const recordedKeys = new Set(); + + function appendRecord(rec) { + if (!manifestPath || !rec || !rec.suite || !rec.case) { + return; + } + const key = `${rec.suite}::${rec.case}`; + if (recordedKeys.has(key)) { + return; + } + recordedKeys.add(key); + fs.mkdirSync(manifestDir, {recursive: true}); + const entry = { + harness: rec.harness || 'angular', + suite: String(rec.suite).replace(/\\/g, '/'), + case: rec.case, + status: rec.status, + ...(rec.reason ? {reason: rec.reason} : {}), + }; + fs.appendFileSync(manifestPath, `${JSON.stringify(entry)}\n`, 'utf8'); + } + + this.onBrowserInfo = function (_browser, info) { + if (info && info.conformanceManifestRecord) { + appendRecord(info.conformanceManifestRecord); + } + }; + + this.onSpecComplete = function (_browser, result) { + if (!manifestPath || !result) { + return; + } + const suiteHeader = Array.isArray(result.suite) ? result.suite.join(' ') : ''; + let suite = null; + if (suiteHeader.includes('conformance/core/accessibility.yaml')) { + suite = 'core/accessibility.yaml'; + } else if (suiteHeader.includes('conformance/core/adapter_conformance.yaml')) { + suite = 'core/adapter_conformance.yaml'; + } + if (!suite) { + return; + } + const desc = String(result.description || ''); + const colonIdx = desc.indexOf(':'); + const caseName = (colonIdx >= 0 ? desc.slice(0, colonIdx) : desc).trim(); + const status = result.skipped ? 'skipped' : result.success ? 'passed' : 'failed'; + const reason = result.skipped + ? result.pendingReason || 'skipped' + : !result.success && Array.isArray(result.log) && result.log.length > 0 + ? String(result.log[0]).split('\n')[0] + : undefined; + appendRecord({ + harness: 'angular', + suite, + case: caseName, + status, + reason, + }); + }; +} +ConformanceManifestReporter.$inject = ['baseReporterDecorator']; + module.exports = function (config) { config.set({ basePath: '', @@ -43,6 +109,7 @@ module.exports = function (config) { require('karma-chrome-launcher'), require('karma-jasmine-html-reporter'), require('karma-coverage'), + {'reporter:conformance-manifest': ['type', ConformanceManifestReporter]}, ], client: { jasmine: { @@ -65,7 +132,7 @@ module.exports = function (config) { subdir: '.', reporters: [{type: 'html'}, {type: 'text-summary'}], }, - reporters: ['progress', 'kjhtml'], + reporters: ['progress', 'kjhtml', 'conformance-manifest'], browsers: ['Chrome'], restartOnFileChange: true, hostname: '127.0.0.1', diff --git a/typescript/renderers/angular/src/adapter_conformance.spec.ts b/typescript/renderers/angular/src/adapter_conformance.spec.ts index 7469ba967b..f40f8206f4 100644 --- a/typescript/renderers/angular/src/adapter_conformance.spec.ts +++ b/typescript/renderers/angular/src/adapter_conformance.spec.ts @@ -14,790 +14,167 @@ * limitations under the License. */ -import {ChangeDetectionStrategy, Component, computed} from '@angular/core'; import {ComponentFixture, TestBed} from '@angular/core/testing'; import { - CommonSchemas, - type ComponentNode, - createFunctionImplementation, - isComponentNode, - peekValue, - type ProcessableMessage, - ResolvedBinding, - type SurfaceModel, -} from '@a2ui/web_core/v0_9'; -import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; -import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; -import {z} from 'zod'; -import { - A2uiRendererService, - AngularCatalog, - type AngularComponentImplementation, - type AnyDuringSchemaAlignment, - BasicCatalogBase, - CatalogComponent, - SurfaceComponent, - createComponentImplementation, - provideA2UI, -} from './public-api'; - -interface ExpectedAccessibilityNode { - label?: string; - description?: string; - live?: string; - hidden?: boolean; - role?: string; - checked?: boolean; -} - -interface AccessibilityAssertions { - accessibilityTree?: Record; - axeCore?: string[]; -} - -interface AccessibilityStep { - messages: ProcessableMessage[]; - assertions: AccessibilityAssertions; -} - -interface AccessibilityTestCase { - name: string; - description?: string; - protocolVersion: string; - action: string; - messages?: ProcessableMessage[]; - assertions?: AccessibilityAssertions; - steps?: AccessibilityStep[]; -} - -interface ExpectedViewNode { - id?: string; - type?: string; - state?: string; - isValid?: boolean; - validationErrors?: string[]; - validationResults?: Array>; - children?: ExpectedViewNode[]; - [key: string]: unknown; -} - -interface ExpectedActionEvent { - name: string; - surfaceId?: string; - sourceComponentId?: string; - context?: Record; -} - -interface ExpectedErrorEntry { - code?: string; - surfaceId?: string; - message?: string; -} - -interface ExtraCatalogEntry { - catalogId: string; - protocolVersion?: string; - components: Record< - string, - { - type: string; - required?: string[]; - properties?: Record; - } - >; -} - -interface AdapterStep { - description?: string; - op?: - | 'process_messages' - | 'user_input' - | 'user_gesture' - | 'programmatic_invoke' - | 'unmount_surface'; - messages?: ProcessableMessage[]; - targetNode?: string; - inputValue?: unknown; - userActivation?: boolean; - expectViewTree?: ExpectedViewNode; - expectAccessibilityTree?: Record; - expectDataModel?: Record; - expectActions?: ExpectedActionEvent[]; - expectErrors?: ExpectedErrorEntry[]; - expectSameViewIdentity?: string[]; - expectActiveSubscriptions?: number; -} - -interface AdapterTestCase { - name: string; - description?: string; - protocolVersion: string; - action: string; - catalogs?: ExtraCatalogEntry[]; - steps: AdapterStep[]; -} - -interface KarmaConformanceConfig { - accessibility: AccessibilityTestCase[]; - adapterConformance: AdapterTestCase[]; - basicCatalogJson: Record; -} - -interface HarnessState { - rendererService: A2uiRendererService; - surface?: SurfaceModel; - fixture?: ComponentFixture; - unmounted: boolean; - actions: Array>; - errors: Array>; - previousNodes: Map; - previousDomElements: Map; -} - -const RESERVED_VIEW_KEYS = new Set([ - 'id', - 'type', - 'state', - 'isValid', - 'validationErrors', - 'validationResults', - 'children', -]); - -const INTERNAL_PROP_KEYS = new Set(['isValid', 'validationErrors', 'validationResults']); - -@Component({ - selector: 'a2ui-conformance-custom-widget', - standalone: true, - template: `{{ renderedText() }}`, - changeDetection: ChangeDetectionStrategy.OnPush, -}) -class ConformanceCustomWidgetComponent extends CatalogComponent { - readonly renderedText = computed(() => { - const entries = Object.entries(this.props()); - const parts: string[] = []; - for (const [key, prop] of entries) { - if (INTERNAL_PROP_KEYS.has(key)) { - continue; - } - const val = prop?.value?.(); - if (typeof val === 'string' || typeof val === 'number') { - parts.push(String(val)); - } - } - return parts.join(' '); - }); -} - -function getConformanceSuites(): KarmaConformanceConfig { - const karma = ( - window as unknown as {__karma__?: {config?: {conformanceSuites?: KarmaConformanceConfig}}} - ).__karma__; - const suites = karma?.config?.conformanceSuites; - if (!suites) { - throw new Error( - 'Conformance suites were not injected into __karma__.config.conformanceSuites.', - ); + assertAllUnsupportedMatched, + assertAxeRules, + assertKnownAccessibilityCase, + assertKnownAdapterCase, + assertNodeAccessibility, + createDefaultAdapterCatalog, + findUnsupportedReason, + getConformanceSuites, + mountSurface, + runStep, + settleSurfaceDom, + writeManifestRecord, + type AccessibilityStep, + type ManifestRecord, +} from './adapter_harness'; +import {A2uiRendererService, SurfaceComponent, provideA2UI} from './public-api'; + +const specMetadataByFullName = new Map(); +let reporterInstalled = false; + +function ensureJasmineManifestReporter(): void { + if (reporterInstalled) { + return; } - return suites; -} - -function createDefaultAdapterCatalog(): AngularCatalog { - const incrementCounter = createFunctionImplementation( - { - name: 'incrementCounter', - returnType: 'void', - schema: z.object({ - path: z.string().optional(), - step: z.number().optional(), - amount: z.number().optional(), - }), - }, - (args, context) => { - const targetPath = args.path ?? '/counter'; - const delta = args.amount ?? args.step ?? 1; - const current = Number(context.dataModel.get(targetPath) ?? 0); - context.dataModel.set(targetPath, current + delta); - }, - ); - return new BasicCatalogBase({ - functions: [...Array.from(basicCatalog.functions.values()), incrementCounter], - }); -} - -function buildExtraCatalog(entry: ExtraCatalogEntry): AngularCatalog { - const implementations = Object.entries(entry.components).map(([name, def]) => { - const shape: Record = {}; - for (const [propKey, propSchema] of Object.entries(def.properties ?? {})) { - if (propKey === 'component') { - continue; + reporterInstalled = true; + jasmine.getEnv().addReporter({ + specDone(result) { + const meta = specMetadataByFullName.get(result.fullName); + if (!meta) { + return; } - shape[propKey] = - propSchema.type === 'number' - ? CommonSchemas.DynamicNumber.optional() - : CommonSchemas.DynamicString.optional(); - } - return createComponentImplementation( - {name, schema: z.object(shape)}, - ConformanceCustomWidgetComponent, - ); - }); - return new AngularCatalog(entry.catalogId, entry.protocolVersion ?? '1.0', implementations); -} - -async function settleSurfaceDom(fixture: ComponentFixture): Promise { - const container = fixture.nativeElement as HTMLElement; - for (let pass = 0; pass < 4; pass++) { - TestBed.flushEffects(); - fixture.detectChanges(); - await new Promise(resolve => setTimeout(resolve, 0)); - const elements = Array.from(container.querySelectorAll('*')); - for (const el of elements) { - const litEl = el as unknown as {updateComplete?: Promise}; - if (litEl.updateComplete) { - await litEl.updateComplete; - } - } - } - TestBed.flushEffects(); - fixture.detectChanges(); -} - -function findComponentElement( - container: HTMLElement, - componentId: string, -): HTMLElement | undefined { - const candidates = Array.from(container.querySelectorAll('*')); - for (const candidate of candidates) { - const a2uiEl = candidate as unknown as A2uiWebComponentElement; - if ( - a2uiEl.context?.componentModel?.id === componentId || - a2uiEl.node?.componentId === componentId - ) { - return candidate as HTMLElement; - } - } - return undefined; -} - -function getRootComponentNode(container: HTMLElement): ComponentNode | undefined { - const host = container.querySelector('a2ui-v09-component-host'); - const first = host?.firstElementChild as A2uiWebComponentElement | null | undefined; - return first?.node; -} - -function extractDirectChildNodes(node: ComponentNode): ComponentNode[] { - const props = peekValue(node.props); - if (Array.isArray(props['children'])) { - return props['children'].filter(isComponentNode); - } - if (isComponentNode(props['child'])) { - return [props['child']]; - } - return []; -} - -function collectMountedNodes(root: ComponentNode | undefined): Map { - const nodes = new Map(); - if (!root) { - return nodes; - } - const visit = (node: ComponentNode, nodePath: string): void => { - nodes.set(nodePath, node); - const props = peekValue(node.props); - if (Array.isArray(props['children'])) { - props['children'].forEach((child, i) => { - if (isComponentNode(child)) { - visit(child, `${nodePath}/children/${i}`); - } + const status: ManifestRecord['status'] = + result.status === 'pending' || result.status === 'excluded' + ? 'skipped' + : result.status === 'passed' + ? 'passed' + : 'failed'; + const reason = + status === 'skipped' + ? result.pendingReason || 'skipped' + : status === 'failed' + ? result.failedExpectations[0]?.message || 'failed' + : undefined; + writeManifestRecord({ + harness: 'angular', + suite: meta.suite, + case: meta.caseName, + status, + reason, }); - } else if (isComponentNode(props['child'])) { - visit(props['child'], `${nodePath}/child`); - } - }; - visit(root, 'root'); - return nodes; -} - -function unwrapBindingValue(value: unknown): unknown { - if (value instanceof ResolvedBinding) { - return unwrapBindingValue(value.value); - } - if (Array.isArray(value)) { - return value.map(unwrapBindingValue); - } - return value; -} - -function resolveNodeType( - node: ComponentNode, - surface: SurfaceModel, -): string { - if (node.state === 'cyclic' && node.type === 'Placeholder') { - return surface.componentsModel.get(node.componentId)?.type ?? node.type; - } - return node.type; -} - -function assertViewNodeProps( - node: ComponentNode, - expected: ExpectedViewNode, - nodePath: string, -): void { - const props = peekValue(node.props); - if (expected.isValid !== undefined) { - expect(unwrapBindingValue(props['isValid'])) - .withContext(`${nodePath}.isValid`) - .toBe(expected.isValid); - } - if (expected.validationErrors !== undefined) { - expect(unwrapBindingValue(props['validationErrors'])) - .withContext(`${nodePath}.validationErrors`) - .toEqual(expected.validationErrors); - } - if (expected.validationResults !== undefined) { - expect(unwrapBindingValue(props['validationResults'])) - .withContext(`${nodePath}.validationResults`) - .toEqual(expected.validationResults); - } - for (const [key, expectedVal] of Object.entries(expected)) { - if (!RESERVED_VIEW_KEYS.has(key)) { - expect(unwrapBindingValue(props[key])).withContext(`${nodePath}.${key}`).toEqual(expectedVal); - } - } -} - -function findDomElementForNode( - container: HTMLElement, - node: ComponentNode, -): HTMLElement | undefined { - const match = Array.from(container.querySelectorAll('*')).find( - el => (el as A2uiWebComponentElement).node === node, - ); - return match as HTMLElement | undefined; -} - -function assertViewTree( - node: ComponentNode | undefined, - expected: ExpectedViewNode, - surface: SurfaceModel, - container: HTMLElement, - nodePath = 'root', -): void { - expect(node).withContext(`Missing node at ${nodePath}`).toBeDefined(); - const actual = node!; - if (expected.id !== undefined) { - expect(actual.componentId).withContext(`${nodePath}.id`).toBe(expected.id); - } - if (expected.type !== undefined) { - expect(resolveNodeType(actual, surface)).withContext(`${nodePath}.type`).toBe(expected.type); - } - if (expected.state !== undefined) { - expect(actual.state).withContext(`${nodePath}.state`).toBe(expected.state); - } - // A resolved node must be mounted in the DOM, and rendered text must be - // visible there, not only present on the in-memory node. - if (actual.state === 'resolved') { - const domEl = findDomElementForNode(container, actual); - expect(domEl).withContext(`Expected a mounted DOM element for ${nodePath}`).toBeDefined(); - for (const key of ['text', 'badgeText'] as const) { - const expectedText = expected[key]; - if (typeof expectedText === 'string') { - expect(domEl!.textContent) - .withContext(`${nodePath}.${key} rendered text`) - .toContain(expectedText); - } - } - } - assertViewNodeProps(actual, expected, nodePath); - if (expected.children !== undefined) { - const childNodes = extractDirectChildNodes(actual); - expect(childNodes.length) - .withContext(`${nodePath}.children length`) - .toBe(expected.children.length); - expected.children.forEach((expectedChild, idx) => { - assertViewTree( - childNodes[idx], - expectedChild, - surface, - container, - `${nodePath}/children/${idx}`, - ); - }); - } -} - -function resolveAccessibleLabel(el: HTMLElement): string | undefined { - const explicit = el.getAttribute('aria-label'); - if (explicit) { - return explicit; - } - const buttonText = el.querySelector('button')?.textContent?.trim(); - if (buttonText) { - return buttonText; - } - const labelText = el.querySelector('label')?.textContent?.trim(); - if (labelText) { - return labelText; - } - return el.textContent?.trim() || undefined; -} - -function resolveSemanticRole(el: HTMLElement): string | undefined { - const explicitRole = el.getAttribute('role'); - if (explicitRole) { - return explicitRole; - } - if (el.querySelector('button')) { - return 'button'; - } - if (el.querySelector('input[type="checkbox"]')) { - return 'checkbox'; - } - if (el.querySelector('input[type="radio"]')) { - return 'radiogroup'; - } - const textInput = el.querySelector( - 'input:not([type="checkbox"]):not([type="radio"]):not([type="range"]), textarea', - ); - return textInput ? 'textbox' : undefined; -} - -function resolveCheckedState(el: HTMLElement): boolean | undefined { - const ariaChecked = el.getAttribute('aria-checked'); - if (ariaChecked === 'true' || ariaChecked === 'false') { - return ariaChecked === 'true'; - } - const checkbox = el.querySelector('input[type="checkbox"]') as HTMLInputElement | null; - return checkbox ? checkbox.checked : undefined; -} - -function assertNodeAccessibility( - container: HTMLElement, - componentId: string, - expected: ExpectedAccessibilityNode, -): void { - const el = findComponentElement(container, componentId); - expect(el).withContext(`Expected rendered DOM element for "${componentId}"`).toBeDefined(); - const target = el!; - if (expected.label !== undefined) { - expect(resolveAccessibleLabel(target)).toBe(expected.label); - } - if (expected.description !== undefined) { - expect(target.getAttribute('aria-description')).toBe(expected.description); - } - if (expected.live !== undefined) { - expect(target.getAttribute('aria-live')).toBe(expected.live); - } - if (expected.hidden !== undefined) { - expect(target.getAttribute('aria-hidden')).toBe(String(expected.hidden)); - } - if (expected.role !== undefined) { - expect(resolveSemanticRole(target)).toBe(expected.role); - } - if (expected.checked !== undefined) { - expect(resolveCheckedState(target)).toBe(expected.checked); - } -} - -function assertAxeRules(container: HTMLElement, rules: string[]): void { - if (rules.includes('button-name')) { - const buttons = Array.from(container.querySelectorAll('button')); - for (const btn of buttons) { - const host = btn.closest('a2ui-basic-button') as HTMLElement | null; - const name = - btn.getAttribute('aria-label') || - host?.getAttribute('aria-label') || - btn.textContent?.trim(); - expect(Boolean(name)).toBe(true); - } - } - if (rules.includes('aria-valid-attr-value')) { - const liveNodes = Array.from(container.querySelectorAll('[aria-live]')); - for (const node of liveNodes) { - const val = node.getAttribute('aria-live') ?? ''; - expect(['off', 'polite', 'assertive']).toContain(val); - } - } -} - -function countActiveSubscriptions(surface: SurfaceModel): number { - const createdListeners = - (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? - 0; - const deletedListeners = - (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? - 0; - let componentUpdatedListeners = 0; - for (const comp of surface.componentsModel.values) { - componentUpdatedListeners += - (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; - } - const dataModelInternal = surface.dataModel as unknown as { - subscriptions?: Set; - signals?: Map; - }; - let activeSignalTargets = 0; - for (const sig of dataModelInternal.signals?.values() ?? []) { - if (sig._targets !== null && sig._targets !== undefined) { - activeSignalTargets += 1; - } - } - const directDataSubs = dataModelInternal.subscriptions?.size ?? 0; - return ( - createdListeners + - deletedListeners + - componentUpdatedListeners + - activeSignalTargets + - directDataSubs - ); -} - -async function performStepOperation(state: HarnessState, step: AdapterStep): Promise { - if (!step.op || step.op === 'process_messages') { - return; - } - if (step.op === 'unmount_surface') { - state.fixture?.destroy(); - state.unmounted = true; - return; - } - const container = state.fixture!.nativeElement as HTMLElement; - const targetId = step.targetNode!; - if (step.op === 'user_input') { - const compEl = findComponentElement(container, targetId); - const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; - expect(input).withContext(`Input element for "${targetId}"`).not.toBeNull(); - input!.value = String(step.inputValue ?? ''); - input!.dispatchEvent(new Event('input', {bubbles: true, composed: true})); - input!.dispatchEvent(new Event('change', {bubbles: true, composed: true})); - return; - } - if (step.op === 'user_gesture') { - const compEl = findComponentElement(container, targetId); - const button = compEl?.querySelector('button') as HTMLButtonElement | null; - expect(button).withContext(`Button element for "${targetId}"`).not.toBeNull(); - button!.click(); - return; - } - if (step.op === 'programmatic_invoke') { - const compModel = state.surface!.componentsModel.get(targetId); - const compEl = findComponentElement(container, targetId) as A2uiWebComponentElement | undefined; - const dataContext = compEl?.context?.dataContext; - expect(dataContext).withContext(`DataContext for "${targetId}"`).toBeDefined(); - dataContext!.resolveAction(compModel!.properties['action'] as Record, { - userActivation: step.userActivation ?? false, - }); - } -} - -function assertExpectedActions( - actualActions: Array>, - expectedActions: ExpectedActionEvent[] | undefined, -): void { - if (expectedActions === undefined) { - return; - } - expect(actualActions.length).toBe(expectedActions.length); - expectedActions.forEach((expected, i) => { - const actual = actualActions[i]; - expect(actual?.['name']).toBe(expected.name); - if (expected.surfaceId !== undefined) { - expect(actual?.['surfaceId']).toBe(expected.surfaceId); - } - if (expected.sourceComponentId !== undefined) { - expect(actual?.['sourceComponentId']).toBe(expected.sourceComponentId); - } - if (expected.context !== undefined) { - expect(actual?.['context']).toEqual(expected.context); - } - }); -} - -function assertExpectedErrors( - actualErrors: Array>, - expectedErrors: ExpectedErrorEntry[] | undefined, -): void { - if (expectedErrors === undefined) { - return; - } - expect(actualErrors.length).toBe(expectedErrors.length); - expectedErrors.forEach((expected, i) => { - const actual = actualErrors[i]; - if (expected.code !== undefined) { - expect(actual?.['code']).toBe(expected.code); - } - if (expected.surfaceId !== undefined) { - expect(actual?.['surfaceId']).toBe(expected.surfaceId); - } - if (expected.message !== undefined) { - expect(String(actual?.['message'] ?? '')).toContain(expected.message); - } + }, }); } -function assertStepIdentityAndModel( - state: HarnessState, - step: AdapterStep, - currentNodes: Map, - currentDom: Map, -): void { - if (step.expectSameViewIdentity) { - for (const nodePath of step.expectSameViewIdentity) { - const prevNode = state.previousNodes.get(nodePath); - const currNode = currentNodes.get(nodePath); - expect(currNode).withContext(`Node at ${nodePath}`).toBeDefined(); - expect(currNode).toBe(prevNode); - const prevEl = state.previousDomElements.get(nodePath); - const currEl = currentDom.get(nodePath); - expect(prevEl).withContext(`Previous DOM element for ${nodePath}`).toBeDefined(); - expect(currEl).withContext(`Current DOM element for ${nodePath}`).toBeDefined(); - expect(currEl).toBe(prevEl); - } - } - if (step.expectDataModel !== undefined) { - expect(state.surface!.dataModel.get('/')).toEqual(step.expectDataModel); - } - if (step.expectActiveSubscriptions !== undefined) { - expect(countActiveSubscriptions(state.surface!)).toBe(step.expectActiveSubscriptions); - } -} - -function mapNodesToDomElements( - container: HTMLElement, - nodes: Map, -): Map { - const result = new Map(); - const allElements = Array.from(container.querySelectorAll('*')); - for (const [nodePath, node] of nodes) { - const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); - if (match) { - result.set(nodePath, match as HTMLElement); - } - } - return result; -} - -async function runAdapterStep(state: HarnessState, step: AdapterStep): Promise { - state.actions = []; - state.errors = []; - if (step.messages && step.messages.length > 0) { - state.rendererService.processMessages(step.messages); - } - if (!state.fixture && state.surface) { - state.fixture = TestBed.createComponent(SurfaceComponent); - state.fixture.componentRef.setInput('surfaceId', state.surface.id); - } - if (state.fixture && !state.unmounted) { - await settleSurfaceDom(state.fixture); - } - await performStepOperation(state, step); - if (state.fixture && !state.unmounted) { - await settleSurfaceDom(state.fixture); - } - const container = state.fixture!.nativeElement as HTMLElement; - const rootNode = state.unmounted ? undefined : getRootComponentNode(container); - const currentNodes = collectMountedNodes(rootNode); - const currentDom = state.unmounted - ? new Map() - : mapNodesToDomElements(container, currentNodes); - - if (step.expectViewTree) { - assertViewTree(rootNode, step.expectViewTree, state.surface!, container); - } - if (step.expectAccessibilityTree) { - for (const [compId, expectedA11y] of Object.entries(step.expectAccessibilityTree)) { - assertNodeAccessibility(container, compId, expectedA11y); - } - } - assertExpectedActions(state.actions, step.expectActions); - assertExpectedErrors(state.errors, step.expectErrors); - assertStepIdentityAndModel(state, step, currentNodes, currentDom); - state.previousNodes = currentNodes; - state.previousDomElements = currentDom; -} - describe('@a2ui/angular Accessibility Conformance (conformance/core/accessibility.yaml)', () => { + ensureJasmineManifestReporter(); + const suiteTitle = + '@a2ui/angular Accessibility Conformance (conformance/core/accessibility.yaml)'; const {accessibility} = getConformanceSuites(); for (const testCase of accessibility) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - TestBed.configureTestingModule({ - imports: [SurfaceComponent], - providers: [provideA2UI({catalogs: [createDefaultAdapterCatalog()]})], - }); - const rendererService = TestBed.inject(A2uiRendererService); + const specTitle = `${testCase.name}: ${testCase.description ?? ''}`; + specMetadataByFullName.set(`${suiteTitle} ${specTitle}`, { + suite: 'core/accessibility.yaml', + caseName: testCase.name, + }); + + it(specTitle, async () => { + assertKnownAccessibilityCase(testCase); const steps: AccessibilityStep[] = testCase.steps ?? [ { messages: testCase.messages ?? [], assertions: testCase.assertions ?? {}, }, ]; + const skipReason = findUnsupportedReason( + testCase.name, + steps as unknown as Array>, + ); + if (skipReason) { + pending(skipReason); + return; + } + + TestBed.configureTestingModule({ + imports: [SurfaceComponent], + providers: [provideA2UI({catalogs: [createDefaultAdapterCatalog()]})], + }); + const rendererService = TestBed.inject(A2uiRendererService); let fixture: ComponentFixture | undefined; - for (const step of steps) { - rendererService.processMessages(step.messages); - if (!fixture) { - const surface = rendererService.surfaceGroup.getSurface('a11y_surface'); - expect(surface).toBeDefined(); - fixture = TestBed.createComponent(SurfaceComponent); - fixture.componentRef.setInput('surfaceId', surface!.id); - } - await settleSurfaceDom(fixture); - const container = fixture.nativeElement as HTMLElement; - for (const [compId, expected] of Object.entries(step.assertions.accessibilityTree ?? {})) { - assertNodeAccessibility(container, compId, expected); - } - if (step.assertions.axeCore) { - assertAxeRules(container, step.assertions.axeCore); + try { + for (const step of steps) { + rendererService.processMessages(step.messages); + if (!fixture) { + const surface = rendererService.surfaceGroup.getSurface('a11y_surface'); + expect(surface).toBeDefined(); + fixture = TestBed.createComponent(SurfaceComponent); + fixture.componentRef.setInput('surfaceId', surface!.id); + } + await settleSurfaceDom(fixture); + const container = fixture.nativeElement as HTMLElement; + for (const [compId, expected] of Object.entries( + step.assertions.accessibilityTree ?? {}, + )) { + assertNodeAccessibility(container, compId, expected); + } + if (step.assertions.axeCore) { + assertAxeRules(container, step.assertions.axeCore); + } } + } finally { + fixture?.destroy(); } - fixture?.destroy(); }); } }); describe('@a2ui/angular Adapter Conformance (conformance/core/adapter_conformance.yaml)', () => { + ensureJasmineManifestReporter(); + const suiteTitle = + '@a2ui/angular Adapter Conformance (conformance/core/adapter_conformance.yaml)'; const {adapterConformance} = getConformanceSuites(); beforeEach(() => { spyOn(window, 'open').and.returnValue(null); }); + afterAll(() => { + assertAllUnsupportedMatched(); + }); + for (const testCase of adapterConformance) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - const catalogs: AngularCatalog[] = [ - createDefaultAdapterCatalog(), - ...(testCase.catalogs ?? []).map(buildExtraCatalog), - ]; - TestBed.configureTestingModule({ - imports: [SurfaceComponent], - providers: [provideA2UI({catalogs})], - }); - const rendererService = TestBed.inject(A2uiRendererService); - const state: HarnessState = { - rendererService, - unmounted: false, - actions: [], - errors: [], - previousNodes: new Map(), - previousDomElements: new Map(), - }; - rendererService.surfaceGroup.onSurfaceCreated.subscribe(surface => { - state.surface = surface; - surface.onAction.subscribe(action => { - state.actions.push(action as unknown as Record); - }); - surface.onError.subscribe(err => { - state.errors.push(err as unknown as Record); - }); - }); + const specTitle = `${testCase.name}: ${testCase.description ?? ''}`; + specMetadataByFullName.set(`${suiteTitle} ${specTitle}`, { + suite: 'core/adapter_conformance.yaml', + caseName: testCase.name, + }); - for (const step of testCase.steps) { - await runAdapterStep(state, step); + it(specTitle, async () => { + assertKnownAdapterCase(testCase); + const skipReason = findUnsupportedReason( + testCase.name, + testCase.steps as unknown as Array>, + ); + if (skipReason) { + pending(skipReason); + return; } - if (!state.unmounted) { - state.fixture?.destroy(); + + const state = mountSurface(testCase); + try { + for (const step of testCase.steps) { + await runStep(state, step); + } + } finally { + if (!state.unmounted) { + state.fixture?.destroy(); + } } }); } diff --git a/typescript/renderers/angular/src/adapter_harness.ts b/typescript/renderers/angular/src/adapter_harness.ts new file mode 100644 index 0000000000..508477daba --- /dev/null +++ b/typescript/renderers/angular/src/adapter_harness.ts @@ -0,0 +1,1017 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {ChangeDetectionStrategy, Component, computed} from '@angular/core'; +import {ComponentFixture, TestBed} from '@angular/core/testing'; +import { + CommonSchemas, + type ComponentNode, + createFunctionImplementation, + isComponentNode, + peekValue, + type ProcessableMessage, + ResolvedBinding, + type SurfaceModel, +} from '@a2ui/web_core/v0_9'; +import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; +import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; +import {z} from 'zod'; +import { + A2uiRendererService, + AngularCatalog, + type AngularComponentImplementation, + type AnyDuringSchemaAlignment, + BasicCatalogBase, + CatalogComponent, + SurfaceComponent, + createComponentImplementation, + provideA2UI, +} from './public-api'; + +export interface ExpectedAccessibilityNode { + label?: string; + description?: string; + live?: string; + hidden?: boolean; + role?: string; + checked?: boolean; +} + +export interface AccessibilityAssertions { + accessibilityTree?: Record; + axeCore?: string[]; +} + +export interface AccessibilityStep { + description?: string; + messages: ProcessableMessage[]; + assertions: AccessibilityAssertions; +} + +export interface AccessibilityTestCase { + name: string; + description?: string; + features?: string[]; + protocolVersion: string; + action: string; + catalogPath?: string; + surface?: Record; + messages?: ProcessableMessage[]; + assertions?: AccessibilityAssertions; + steps?: AccessibilityStep[]; +} + +export interface ExpectedViewNode { + id?: string; + type?: string; + state?: string; + isValid?: boolean; + validationErrors?: string[]; + validationResults?: Array>; + children?: ExpectedViewNode[]; + [key: string]: unknown; +} + +export interface ExpectedActionEvent { + name: string; + surfaceId?: string; + sourceComponentId?: string; + context?: Record; +} + +export interface ExpectedErrorEntry { + code?: string; + surfaceId?: string; + message?: string; +} + +export interface ExtraCatalogEntry { + catalogId: string; + protocolVersion?: string; + components: Record< + string, + { + type: string; + required?: string[]; + properties?: Record; + } + >; +} + +export interface AdapterStep { + description?: string; + op?: + | 'process_messages' + | 'user_input' + | 'user_gesture' + | 'programmatic_invoke' + | 'unmount_surface'; + messages?: ProcessableMessage[]; + targetNode?: string; + inputValue?: unknown; + userActivation?: boolean; + expectViewTree?: ExpectedViewNode; + expectAccessibilityTree?: Record; + expectDataModel?: Record; + expectActions?: ExpectedActionEvent[]; + expectErrors?: ExpectedErrorEntry[]; + expectSameViewIdentity?: string[]; + expectActiveSubscriptions?: number; +} + +export interface AdapterTestCase { + name: string; + description?: string; + features?: string[]; + protocolVersion: string; + action: string; + catalogPaths?: string[]; + catalogs?: ExtraCatalogEntry[]; + steps: AdapterStep[]; +} + +export interface KarmaConformanceConfig { + accessibility: AccessibilityTestCase[]; + adapterConformance: AdapterTestCase[]; + basicCatalogJson: Record; +} + +export interface HarnessState { + rendererService: A2uiRendererService; + surface?: SurfaceModel; + fixture?: ComponentFixture; + unmounted: boolean; + actions: Array>; + errors: Array>; + previousNodes: Map; + previousDomElements: Map; +} + +export interface ManifestRecord { + harness: string; + suite: string; + case: string; + status: 'passed' | 'failed' | 'skipped' | 'xfailed'; + reason?: string; +} + +/** + * Reasoned skip entries keyed by `""` or `"/"`. + * Every entry in this map must match at least one case during the run. + */ +export const UNSUPPORTED: Record = {}; + +const matchedUnsupportedKeys = new Set(); + +export const KNOWN_KEYS = { + adapterCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPaths', + 'catalogs', + 'steps', + ]), + adapterCatalog: new Set(['catalogId', 'protocolVersion', 'components']), + adapterStep: new Set([ + 'description', + 'op', + 'messages', + 'targetNode', + 'inputValue', + 'userActivation', + 'expectViewTree', + 'expectAccessibilityTree', + 'expectDataModel', + 'expectActions', + 'expectErrors', + 'expectSameViewIdentity', + 'expectActiveSubscriptions', + ]), + adapterOps: new Set([ + 'process_messages', + 'user_input', + 'user_gesture', + 'programmatic_invoke', + 'unmount_surface', + ]), + viewNode: new Set([ + 'id', + 'type', + 'state', + 'text', + 'value', + 'badgeText', + 'isValid', + 'validationErrors', + 'validationResults', + 'children', + ]), + validationResult: new Set(['valid', 'code', 'message', 'severity']), + expectedAction: new Set(['name', 'surfaceId', 'sourceComponentId', 'context']), + expectedError: new Set(['code', 'surfaceId', 'message']), + accessibilityCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPath', + 'surface', + 'messages', + 'assertions', + 'steps', + ]), + accessibilityStep: new Set(['description', 'messages', 'assertions']), + accessibilityAssertions: new Set(['accessibilityTree', 'axeCore']), + accessibilityNode: new Set(['label', 'description', 'live', 'hidden', 'role', 'checked']), +} as const; + +function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: string): void { + if (obj === undefined || obj === null) { + return; + } + if (typeof obj !== 'object' || Array.isArray(obj)) { + throw new Error(`${where}: expected object`); + } + const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error(`${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`); + } +} + +function assertKnownViewNodeKeys(node: unknown, where: string): void { + if (node === undefined || node === null) { + return; + } + assertObjectKeys(node, KNOWN_KEYS.viewNode, where); + const record = node as Record; + if (Array.isArray(record['validationResults'])) { + record['validationResults'].forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.validationResult, `${where}.validationResults[${idx}]`); + }); + } + if (Array.isArray(record['children'])) { + record['children'].forEach((child, idx) => { + assertKnownViewNodeKeys(child, `${where}.children[${idx}]`); + }); + } +} + +export function assertKnownKeys( + obj: unknown, + allowed: ReadonlySet = KNOWN_KEYS.adapterStep, + where = 'step', +): void { + assertObjectKeys(obj, allowed, where); + if (allowed !== KNOWN_KEYS.adapterStep || !obj || typeof obj !== 'object') { + return; + } + const step = obj as Record; + if (step['op'] !== undefined) { + if (typeof step['op'] !== 'string' || !KNOWN_KEYS.adapterOps.has(step['op'])) { + throw new Error(`${where}.op: unsupported op "${String(step['op'])}"`); + } + } + if (step['expectViewTree'] !== undefined) { + assertKnownViewNodeKeys(step['expectViewTree'], `${where}.expectViewTree`); + } + if (step['expectAccessibilityTree'] !== undefined) { + if ( + typeof step['expectAccessibilityTree'] !== 'object' || + step['expectAccessibilityTree'] === null || + Array.isArray(step['expectAccessibilityTree']) + ) { + throw new Error(`${where}.expectAccessibilityTree: expected object`); + } + for (const [compId, node] of Object.entries( + step['expectAccessibilityTree'] as Record, + )) { + assertObjectKeys( + node, + KNOWN_KEYS.accessibilityNode, + `${where}.expectAccessibilityTree.${compId}`, + ); + } + } + if (step['expectActions'] !== undefined) { + if (!Array.isArray(step['expectActions'])) { + throw new Error(`${where}.expectActions: expected array`); + } + (step['expectActions'] as unknown[]).forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.expectedAction, `${where}.expectActions[${idx}]`); + }); + } + if (step['expectErrors'] !== undefined) { + if (!Array.isArray(step['expectErrors'])) { + throw new Error(`${where}.expectErrors: expected array`); + } + (step['expectErrors'] as unknown[]).forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.expectedError, `${where}.expectErrors[${idx}]`); + }); + } +} + +export function assertKnownAdapterCase(testCase: AdapterTestCase): void { + assertObjectKeys(testCase, KNOWN_KEYS.adapterCase, `case(${testCase.name})`); + if (testCase.action !== 'render_surface') { + throw new Error(`case(${testCase.name}): unsupported action "${testCase.action}"`); + } + for (const [idx, cat] of (testCase.catalogs ?? []).entries()) { + assertObjectKeys(cat, KNOWN_KEYS.adapterCatalog, `case(${testCase.name}).catalogs[${idx}]`); + } + for (const [idx, step] of testCase.steps.entries()) { + assertKnownKeys(step, KNOWN_KEYS.adapterStep, `case(${testCase.name}).steps[${idx}]`); + } +} + +export function assertKnownAccessibilityAssertions( + assertions: AccessibilityAssertions | undefined, + where: string, +): void { + if (!assertions) { + return; + } + assertObjectKeys(assertions, KNOWN_KEYS.accessibilityAssertions, where); + for (const [compId, node] of Object.entries(assertions.accessibilityTree ?? {})) { + assertObjectKeys(node, KNOWN_KEYS.accessibilityNode, `${where}.accessibilityTree.${compId}`); + } +} + +export function assertKnownAccessibilityCase(testCase: AccessibilityTestCase): void { + assertObjectKeys(testCase, KNOWN_KEYS.accessibilityCase, `case(${testCase.name})`); + if (testCase.action !== 'accessibility_check') { + throw new Error(`case(${testCase.name}): unsupported action "${testCase.action}"`); + } + assertKnownAccessibilityAssertions(testCase.assertions, `case(${testCase.name}).assertions`); + for (const [idx, step] of (testCase.steps ?? []).entries()) { + assertObjectKeys(step, KNOWN_KEYS.accessibilityStep, `case(${testCase.name}).steps[${idx}]`); + assertKnownAccessibilityAssertions( + step.assertions, + `case(${testCase.name}).steps[${idx}].assertions`, + ); + } +} + +export function findUnsupportedReason( + caseName: string, + steps: Array> = [], +): string | undefined { + if (Object.hasOwn(UNSUPPORTED, caseName)) { + matchedUnsupportedKeys.add(caseName); + return UNSUPPORTED[caseName]; + } + for (const step of steps) { + const op = typeof step['op'] === 'string' ? step['op'] : undefined; + if (op) { + const opKey = `${caseName}/${op}`; + if (Object.hasOwn(UNSUPPORTED, opKey)) { + matchedUnsupportedKeys.add(opKey); + return UNSUPPORTED[opKey]; + } + } + for (const key of Object.keys(step)) { + const stepKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, stepKey)) { + matchedUnsupportedKeys.add(stepKey); + return UNSUPPORTED[stepKey]; + } + } + } + return undefined; +} + +export function assertAllUnsupportedMatched(): void { + const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); + if (unused.length > 0) { + throw new Error( + `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, + ); + } +} + +export function writeManifestRecord(rec: ManifestRecord): void { + const karma = ( + window as unknown as { + __karma__?: {info?: (payload: {conformanceManifestRecord: ManifestRecord}) => void}; + } + ).__karma__; + karma?.info?.({conformanceManifestRecord: rec}); +} + +const RESERVED_VIEW_KEYS = new Set([ + 'id', + 'type', + 'state', + 'isValid', + 'validationErrors', + 'validationResults', + 'children', +]); + +const INTERNAL_PROP_KEYS = new Set(['isValid', 'validationErrors', 'validationResults']); + +@Component({ + selector: 'a2ui-conformance-custom-widget', + standalone: true, + template: `{{ renderedText() }}`, + changeDetection: ChangeDetectionStrategy.OnPush, +}) +class ConformanceCustomWidgetComponent extends CatalogComponent { + readonly renderedText = computed(() => { + const entries = Object.entries(this.props()); + const parts: string[] = []; + for (const [key, prop] of entries) { + if (INTERNAL_PROP_KEYS.has(key)) { + continue; + } + const val = prop?.value?.(); + if (typeof val === 'string' || typeof val === 'number') { + parts.push(String(val)); + } + } + return parts.join(' '); + }); +} + +export function getConformanceSuites(): KarmaConformanceConfig { + const karma = ( + window as unknown as {__karma__?: {config?: {conformanceSuites?: KarmaConformanceConfig}}} + ).__karma__; + const suites = karma?.config?.conformanceSuites; + if (!suites) { + throw new Error( + 'Conformance suites were not injected into __karma__.config.conformanceSuites.', + ); + } + return suites; +} + +export function createDefaultAdapterCatalog(): AngularCatalog { + const incrementCounter = createFunctionImplementation( + { + name: 'incrementCounter', + returnType: 'void', + schema: z.object({ + path: z.string().optional(), + step: z.number().optional(), + amount: z.number().optional(), + }), + }, + (args, context) => { + const targetPath = args.path ?? '/counter'; + const delta = args.amount ?? args.step ?? 1; + const current = Number(context.dataModel.get(targetPath) ?? 0); + context.dataModel.set(targetPath, current + delta); + }, + ); + return new BasicCatalogBase({ + functions: [...Array.from(basicCatalog.functions.values()), incrementCounter], + }); +} + +export function buildExtraCatalog(entry: ExtraCatalogEntry): AngularCatalog { + const implementations = Object.entries(entry.components).map(([name, def]) => { + const shape: Record = {}; + for (const [propKey, propSchema] of Object.entries(def.properties ?? {})) { + if (propKey === 'component') { + continue; + } + shape[propKey] = + propSchema.type === 'number' + ? CommonSchemas.DynamicNumber.optional() + : CommonSchemas.DynamicString.optional(); + } + return createComponentImplementation( + {name, schema: z.object(shape)}, + ConformanceCustomWidgetComponent, + ); + }); + return new AngularCatalog(entry.catalogId, entry.protocolVersion ?? '1.0', implementations); +} + +export async function settleSurfaceDom(fixture: ComponentFixture): Promise { + const container = fixture.nativeElement as HTMLElement; + for (let pass = 0; pass < 4; pass++) { + TestBed.flushEffects(); + fixture.detectChanges(); + await new Promise(resolve => setTimeout(resolve, 0)); + const elements = Array.from(container.querySelectorAll('*')); + for (const el of elements) { + const litEl = el as unknown as {updateComplete?: Promise}; + if (litEl.updateComplete) { + await litEl.updateComplete; + } + } + } + TestBed.flushEffects(); + fixture.detectChanges(); +} + +function findComponentElement( + container: HTMLElement, + componentId: string, +): HTMLElement | undefined { + const candidates = Array.from(container.querySelectorAll('*')); + for (const candidate of candidates) { + const a2uiEl = candidate as unknown as A2uiWebComponentElement; + if ( + a2uiEl.context?.componentModel?.id === componentId || + a2uiEl.node?.componentId === componentId + ) { + return candidate as HTMLElement; + } + } + return undefined; +} + +function getRootComponentNode(container: HTMLElement): ComponentNode | undefined { + const host = container.querySelector('a2ui-v09-component-host'); + const first = host?.firstElementChild as A2uiWebComponentElement | null | undefined; + return first?.node; +} + +function extractDirectChildNodes(node: ComponentNode): ComponentNode[] { + const props = peekValue(node.props); + if (Array.isArray(props['children'])) { + return props['children'].filter(isComponentNode); + } + if (isComponentNode(props['child'])) { + return [props['child']]; + } + return []; +} + +function collectMountedNodes(root: ComponentNode | undefined): Map { + const nodes = new Map(); + if (!root) { + return nodes; + } + const visit = (node: ComponentNode, nodePath: string): void => { + nodes.set(nodePath, node); + const props = peekValue(node.props); + if (Array.isArray(props['children'])) { + props['children'].forEach((child, i) => { + if (isComponentNode(child)) { + visit(child, `${nodePath}/children/${i}`); + } + }); + } else if (isComponentNode(props['child'])) { + visit(props['child'], `${nodePath}/child`); + } + }; + visit(root, 'root'); + return nodes; +} + +function unwrapBindingValue(value: unknown): unknown { + if (value instanceof ResolvedBinding) { + return unwrapBindingValue(value.value); + } + if (Array.isArray(value)) { + return value.map(unwrapBindingValue); + } + return value; +} + +function resolveNodeType( + node: ComponentNode, + surface?: SurfaceModel, +): string { + if (node.state === 'cyclic' && node.type === 'Placeholder' && surface) { + return surface.componentsModel.get(node.componentId)?.type ?? node.type; + } + return node.type; +} + +function assertViewNodeProps( + node: ComponentNode, + expected: ExpectedViewNode, + nodePath: string, +): void { + const props = peekValue(node.props); + if (expected.isValid !== undefined) { + expect(unwrapBindingValue(props['isValid'])) + .withContext(`${nodePath}.isValid`) + .toBe(expected.isValid); + } + if (expected.validationErrors !== undefined) { + expect(unwrapBindingValue(props['validationErrors'])) + .withContext(`${nodePath}.validationErrors`) + .toEqual(expected.validationErrors); + } + if (expected.validationResults !== undefined) { + expect(unwrapBindingValue(props['validationResults'])) + .withContext(`${nodePath}.validationResults`) + .toEqual(expected.validationResults); + } + for (const [key, expectedVal] of Object.entries(expected)) { + if (!RESERVED_VIEW_KEYS.has(key)) { + expect(unwrapBindingValue(props[key])).withContext(`${nodePath}.${key}`).toEqual(expectedVal); + } + } +} + +function findDomElementForNode( + container: HTMLElement, + node: ComponentNode, +): HTMLElement | undefined { + const match = Array.from(container.querySelectorAll('*')).find( + el => (el as A2uiWebComponentElement).node === node, + ); + return match as HTMLElement | undefined; +} + +export function extractViewTree(state: HarnessState): ComponentNode | undefined { + if (!state.fixture || state.unmounted) { + return undefined; + } + const container = state.fixture.nativeElement as HTMLElement; + return getRootComponentNode(container); +} + +export function assertViewTree( + node: ComponentNode | undefined, + expected: ExpectedViewNode, + surface?: SurfaceModel, + container?: HTMLElement, + nodePath = 'root', +): void { + expect(node).withContext(`Missing node at ${nodePath}`).toBeDefined(); + const actual = node!; + if (expected.id !== undefined) { + expect(actual.componentId).withContext(`${nodePath}.id`).toBe(expected.id); + } + if (expected.type !== undefined) { + expect(resolveNodeType(actual, surface)).withContext(`${nodePath}.type`).toBe(expected.type); + } + if (expected.state !== undefined) { + expect(actual.state).withContext(`${nodePath}.state`).toBe(expected.state); + } + if (actual.state === 'resolved' && container) { + const domEl = findDomElementForNode(container, actual); + expect(domEl).withContext(`Expected a mounted DOM element for ${nodePath}`).toBeDefined(); + for (const key of ['text', 'badgeText'] as const) { + const expectedText = expected[key]; + if (typeof expectedText === 'string') { + expect(domEl!.textContent) + .withContext(`${nodePath}.${key} rendered text`) + .toContain(expectedText); + } + } + } + assertViewNodeProps(actual, expected, nodePath); + if (expected.children !== undefined) { + const childNodes = extractDirectChildNodes(actual); + expect(childNodes.length) + .withContext(`${nodePath}.children length`) + .toBe(expected.children.length); + expected.children.forEach((expectedChild, idx) => { + assertViewTree( + childNodes[idx], + expectedChild, + surface, + container, + `${nodePath}/children/${idx}`, + ); + }); + } +} + +function resolveAccessibleLabel(el: HTMLElement): string | undefined { + const explicit = el.getAttribute('aria-label'); + if (explicit) { + return explicit; + } + const buttonText = el.querySelector('button')?.textContent?.trim(); + if (buttonText) { + return buttonText; + } + const labelText = el.querySelector('label')?.textContent?.trim(); + if (labelText) { + return labelText; + } + return el.textContent?.trim() || undefined; +} + +function resolveSemanticRole(el: HTMLElement): string | undefined { + const explicitRole = el.getAttribute('role'); + if (explicitRole) { + return explicitRole; + } + if (el.querySelector('button')) { + return 'button'; + } + if (el.querySelector('input[type="checkbox"]')) { + return 'checkbox'; + } + if (el.querySelector('input[type="radio"]')) { + return 'radiogroup'; + } + const textInput = el.querySelector( + 'input:not([type="checkbox"]):not([type="radio"]):not([type="range"]), textarea', + ); + return textInput ? 'textbox' : undefined; +} + +function resolveCheckedState(el: HTMLElement): boolean | undefined { + const ariaChecked = el.getAttribute('aria-checked'); + if (ariaChecked === 'true' || ariaChecked === 'false') { + return ariaChecked === 'true'; + } + const checkbox = el.querySelector('input[type="checkbox"]') as HTMLInputElement | null; + return checkbox ? checkbox.checked : undefined; +} + +export function assertNodeAccessibility( + container: HTMLElement, + componentId: string, + expected: ExpectedAccessibilityNode, +): void { + const el = findComponentElement(container, componentId); + expect(el).withContext(`Expected rendered DOM element for "${componentId}"`).toBeDefined(); + const target = el!; + if (expected.label !== undefined) { + expect(resolveAccessibleLabel(target)).toBe(expected.label); + } + if (expected.description !== undefined) { + expect(target.getAttribute('aria-description')).toBe(expected.description); + } + if (expected.live !== undefined) { + expect(target.getAttribute('aria-live')).toBe(expected.live); + } + if (expected.hidden !== undefined) { + expect(target.getAttribute('aria-hidden')).toBe(String(expected.hidden)); + } + if (expected.role !== undefined) { + expect(resolveSemanticRole(target)).toBe(expected.role); + } + if (expected.checked !== undefined) { + expect(resolveCheckedState(target)).toBe(expected.checked); + } +} + +export function assertAxeRules(container: HTMLElement, rules: string[]): void { + if (rules.includes('button-name')) { + const buttons = Array.from(container.querySelectorAll('button')); + for (const btn of buttons) { + const host = btn.closest('a2ui-basic-button') as HTMLElement | null; + const name = + btn.getAttribute('aria-label') || + host?.getAttribute('aria-label') || + btn.textContent?.trim(); + expect(Boolean(name)).toBe(true); + } + } + if (rules.includes('aria-valid-attr-value')) { + const liveNodes = Array.from(container.querySelectorAll('[aria-live]')); + for (const node of liveNodes) { + const val = node.getAttribute('aria-live') ?? ''; + expect(['off', 'polite', 'assertive']).toContain(val); + } + } +} + +function countActiveSubscriptions(surface: SurfaceModel): number { + const createdListeners = + (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? + 0; + const deletedListeners = + (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? + 0; + let componentUpdatedListeners = 0; + for (const comp of surface.componentsModel.values) { + componentUpdatedListeners += + (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; + } + const dataModelInternal = surface.dataModel as unknown as { + subscriptions?: Set; + signals?: Map; + }; + let activeSignalTargets = 0; + for (const sig of dataModelInternal.signals?.values() ?? []) { + if (sig._targets !== null && sig._targets !== undefined) { + activeSignalTargets += 1; + } + } + const directDataSubs = dataModelInternal.subscriptions?.size ?? 0; + return ( + createdListeners + + deletedListeners + + componentUpdatedListeners + + activeSignalTargets + + directDataSubs + ); +} + +async function performStepOperation(state: HarnessState, step: AdapterStep): Promise { + if (!step.op || step.op === 'process_messages') { + return; + } + if (step.op === 'unmount_surface') { + state.fixture?.destroy(); + state.unmounted = true; + return; + } + const container = state.fixture!.nativeElement as HTMLElement; + const targetId = step.targetNode!; + if (step.op === 'user_input') { + const compEl = findComponentElement(container, targetId); + const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; + expect(input).withContext(`Input element for "${targetId}"`).not.toBeNull(); + input!.value = String(step.inputValue ?? ''); + input!.dispatchEvent(new Event('input', {bubbles: true, composed: true})); + input!.dispatchEvent(new Event('change', {bubbles: true, composed: true})); + return; + } + if (step.op === 'user_gesture') { + const compEl = findComponentElement(container, targetId); + const button = compEl?.querySelector('button') as HTMLButtonElement | null; + expect(button).withContext(`Button element for "${targetId}"`).not.toBeNull(); + button!.click(); + return; + } + if (step.op === 'programmatic_invoke') { + const compModel = state.surface!.componentsModel.get(targetId); + const compEl = findComponentElement(container, targetId) as A2uiWebComponentElement | undefined; + const dataContext = compEl?.context?.dataContext; + expect(dataContext).withContext(`DataContext for "${targetId}"`).toBeDefined(); + dataContext!.resolveAction(compModel!.properties['action'] as Record, { + userActivation: step.userActivation ?? false, + }); + return; + } + throw new Error(`Unsupported adapter step op: ${String(step.op)}`); +} + +function assertExpectedActions( + actualActions: Array>, + expectedActions: ExpectedActionEvent[] | undefined, +): void { + if (expectedActions === undefined) { + return; + } + expect(actualActions.length).toBe(expectedActions.length); + expectedActions.forEach((expected, i) => { + const actual = actualActions[i]; + expect(actual?.['name']).toBe(expected.name); + if (expected.surfaceId !== undefined) { + expect(actual?.['surfaceId']).toBe(expected.surfaceId); + } + if (expected.sourceComponentId !== undefined) { + expect(actual?.['sourceComponentId']).toBe(expected.sourceComponentId); + } + if (expected.context !== undefined) { + expect(actual?.['context']).toEqual(expected.context); + } + }); +} + +function assertExpectedErrors( + actualErrors: Array>, + expectedErrors: ExpectedErrorEntry[] | undefined, +): void { + if (expectedErrors === undefined) { + return; + } + expect(actualErrors.length).toBe(expectedErrors.length); + expectedErrors.forEach((expected, i) => { + const actual = actualErrors[i]; + if (expected.code !== undefined) { + expect(actual?.['code']).toBe(expected.code); + } + if (expected.surfaceId !== undefined) { + expect(actual?.['surfaceId']).toBe(expected.surfaceId); + } + if (expected.message !== undefined) { + expect(String(actual?.['message'] ?? '')).toContain(expected.message); + } + }); +} + +function assertStepIdentityAndModel( + state: HarnessState, + step: AdapterStep, + currentNodes: Map, + currentDom: Map, +): void { + if (step.expectSameViewIdentity) { + for (const nodePath of step.expectSameViewIdentity) { + const prevNode = state.previousNodes.get(nodePath); + const currNode = currentNodes.get(nodePath); + expect(currNode).withContext(`Node at ${nodePath}`).toBeDefined(); + expect(currNode).toBe(prevNode); + const prevEl = state.previousDomElements.get(nodePath); + const currEl = currentDom.get(nodePath); + expect(prevEl).withContext(`Previous DOM element for ${nodePath}`).toBeDefined(); + expect(currEl).withContext(`Current DOM element for ${nodePath}`).toBeDefined(); + expect(currEl).toBe(prevEl); + } + } + if (step.expectDataModel !== undefined) { + expect(state.surface!.dataModel.get('/')).toEqual(step.expectDataModel); + } + if (step.expectActiveSubscriptions !== undefined) { + expect(countActiveSubscriptions(state.surface!)).toBe(step.expectActiveSubscriptions); + } +} + +function mapNodesToDomElements( + container: HTMLElement, + nodes: Map, +): Map { + const result = new Map(); + const allElements = Array.from(container.querySelectorAll('*')); + for (const [nodePath, node] of nodes) { + const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); + if (match) { + result.set(nodePath, match as HTMLElement); + } + } + return result; +} + +export function mountSurface(caseSpec?: Partial): HarnessState { + const catalogs: AngularCatalog[] = [ + createDefaultAdapterCatalog(), + ...(caseSpec?.catalogs ?? []).map(buildExtraCatalog), + ]; + TestBed.configureTestingModule({ + imports: [SurfaceComponent], + providers: [provideA2UI({catalogs})], + }); + const rendererService = TestBed.inject(A2uiRendererService); + const state: HarnessState = { + rendererService, + unmounted: false, + actions: [], + errors: [], + previousNodes: new Map(), + previousDomElements: new Map(), + }; + rendererService.surfaceGroup.onSurfaceCreated.subscribe(surface => { + state.surface = surface; + surface.onAction.subscribe(action => { + state.actions.push(action as unknown as Record); + }); + surface.onError.subscribe(err => { + state.errors.push(err as unknown as Record); + }); + }); + return state; +} + +export async function runStep(state: HarnessState, step: AdapterStep): Promise { + assertKnownKeys(step); + state.actions = []; + state.errors = []; + if (step.messages && step.messages.length > 0) { + state.rendererService.processMessages(step.messages); + } + if (!state.fixture && state.surface) { + state.fixture = TestBed.createComponent(SurfaceComponent); + state.fixture.componentRef.setInput('surfaceId', state.surface.id); + } + if (state.fixture && !state.unmounted) { + await settleSurfaceDom(state.fixture); + } + await performStepOperation(state, step); + if (state.fixture && !state.unmounted) { + await settleSurfaceDom(state.fixture); + } + const container = state.fixture!.nativeElement as HTMLElement; + const rootNode = extractViewTree(state); + const currentNodes = collectMountedNodes(rootNode); + const currentDom = state.unmounted + ? new Map() + : mapNodesToDomElements(container, currentNodes); + + if (step.expectViewTree) { + assertViewTree(rootNode, step.expectViewTree, state.surface!, container); + } + if (step.expectAccessibilityTree) { + for (const [compId, expectedA11y] of Object.entries(step.expectAccessibilityTree)) { + assertNodeAccessibility(container, compId, expectedA11y); + } + } + assertExpectedActions(state.actions, step.expectActions); + assertExpectedErrors(state.errors, step.expectErrors); + assertStepIdentityAndModel(state, step, currentNodes, currentDom); + state.previousNodes = currentNodes; + state.previousDomElements = currentDom; +} diff --git a/typescript/renderers/angular/tsconfig.lib.json b/typescript/renderers/angular/tsconfig.lib.json index 7b51673f40..d1d17425c7 100644 --- a/typescript/renderers/angular/tsconfig.lib.json +++ b/typescript/renderers/angular/tsconfig.lib.json @@ -9,5 +9,5 @@ "types": [] }, "include": ["src/**/*.ts"], - "exclude": ["**/*.spec.ts"] + "exclude": ["**/*.spec.ts", "src/adapter_harness.ts"] } diff --git a/typescript/renderers/lit/src/tests/adapter_conformance.test.ts b/typescript/renderers/lit/src/tests/adapter_conformance.test.ts index 3259ede906..9aae29f96c 100644 --- a/typescript/renderers/lit/src/tests/adapter_conformance.test.ts +++ b/typescript/renderers/lit/src/tests/adapter_conformance.test.ts @@ -14,606 +14,33 @@ * limitations under the License. */ -import assert from 'node:assert/strict'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import {describe, it, before, after, afterEach} from 'node:test'; +import {after, afterEach, before, describe, it} from 'node:test'; import * as yaml from 'js-yaml'; -import {z} from 'zod'; - -import {setupTestDom, teardownTestDom, asyncUpdate} from './dom-setup.js'; -import {html} from 'lit'; +import {setupTestDom, teardownTestDom} from './dom-setup.js'; import { - Catalog, - MessageProcessor, - ResolvedBinding, - getValue, - isComponentNode, - type ComponentNode, - type FunctionImplementation, - type ProcessableMessage, - type SurfaceModel, -} from '@a2ui/web_core'; -import {basicCatalog as v1BasicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; -import {A2uiLitElement} from '@a2ui/web_core/universal'; - -import type {A2uiSurface} from '../index.js'; -import type {LitComponentApi} from '../types.js'; + assertAccessibilityNode, + assertAllUnsupportedMatched, + assertAxeRules, + assertKnownAccessibilityCase, + assertKnownAdapterCase, + ensureSurfaceMounted, + findUnsupportedReason, + mountSurface, + runStep, + settleSurface, + writeManifestRecord, + type AccessibilityStep, + type AccessibilityTestCase, + type AdapterTestCase, + type ManifestRecord, +} from './adapter_harness.js'; const CONFORMANCE_ROOT = path.resolve(import.meta.dirname, '../../../../../../conformance/core'); -interface ExpectedAccessibilityNode { - label?: string; - description?: string; - live?: string; - hidden?: boolean; - role?: string; - checked?: boolean; -} - -interface AccessibilityAssertions { - accessibilityTree?: Record; - axeCore?: string[]; -} - -interface AccessibilityStep { - messages: ProcessableMessage[]; - assertions: AccessibilityAssertions; -} - -interface AccessibilityTestCase { - name: string; - description?: string; - messages?: ProcessableMessage[]; - assertions?: AccessibilityAssertions; - steps?: AccessibilityStep[]; -} - -interface ExpectedViewNode { - id: string; - type?: string; - state?: string; - text?: string; - value?: unknown; - badgeText?: string; - isValid?: boolean; - validationErrors?: string[]; - validationResults?: unknown[]; - children?: ExpectedViewNode[]; -} - -interface ExpectedAction { - name: string; - surfaceId: string; - sourceComponentId: string; - context?: Record; -} - -interface ExpectedError { - code?: string; - surfaceId?: string; - message?: string; -} - -interface AdapterStep { - op: - | 'process_messages' - | 'user_input' - | 'user_gesture' - | 'programmatic_invoke' - | 'unmount_surface'; - messages?: ProcessableMessage[]; - targetNode?: string; - inputValue?: unknown; - userActivation?: boolean; - expectViewTree?: ExpectedViewNode; - expectDataModel?: Record; - expectSameViewIdentity?: string[]; - expectActions?: ExpectedAction[]; - expectErrors?: ExpectedError[]; - expectAccessibilityTree?: Record; - expectActiveSubscriptions?: number; -} - -interface SecondaryCatalogSpec { - catalogId: string; - protocolVersion?: string; - components: Record; -} - -interface AdapterTestCase { - name: string; - description?: string; - catalogs?: SecondaryCatalogSpec[]; - steps: AdapterStep[]; -} - -interface AdapterSession { - processor: MessageProcessor; - surface?: SurfaceModel; - surfaceEl?: A2uiSurface; - recordedActions: ExpectedAction[]; - recordedErrors: Array<{code: string; message: string; surfaceId: string}>; - priorIdentities: Map; -} - -async function settleSurface(root: HTMLElement): Promise { - for (let pass = 0; pass < 3; pass++) { - await asyncUpdate(root, () => {}); - const elements = [root, ...Array.from(root.querySelectorAll('*'))]; - for (const el of elements) { - const litEl = el as unknown as {updateComplete?: Promise}; - if (litEl.updateComplete) { - await litEl.updateComplete; - } - } - } -} - -function createIncrementCounterFn(): FunctionImplementation { - return { - name: 'incrementCounter', - returnType: 'void', - schema: z.object({ - path: z.string(), - step: z.number(), - }), - execute: (args: {path: string; step: number}, context) => { - const current = Number(context.dataModel.get(args.path) ?? 0); - context.dataModel.set(args.path, current + args.step); - }, - }; -} - -function createAugmentedBasicCatalog(): Catalog { - const base = v1BasicCatalog as unknown as Catalog; - const components = Array.from(base.components.values()); - const functions = [...Array.from(base.functions?.values() ?? []), createIncrementCounterFn()]; - return new Catalog( - base.id, - base.protocolVersion ?? '1.0', - components, - functions, - ); -} - -function buildSecondaryCatalog(spec: SecondaryCatalogSpec): Catalog { - const components: LitComponentApi[] = []; - const badgeSchema = z.object({ - badgeText: z.string().optional(), - }); - for (const compName of Object.keys(spec.components)) { - const tagName = `a2ui-conformance-${compName.toLowerCase()}`; - let element = customElements.get(tagName) as CustomElementConstructor | undefined; - if (!element) { - class CustomBadgeElement extends A2uiLitElement { - override createRenderRoot() { - return this; - } - protected override render() { - const props = this.controller?.props as Record | undefined; - return html`${String(props?.badgeText ?? '')}`; - } - } - element = CustomBadgeElement; - } - components.push({ - name: compName, - tagName, - element, - schema: badgeSchema, - }); - } - return new Catalog( - spec.catalogId, - (spec.protocolVersion ?? 'v1.0') as 'v1.0', - components, - [], - ); -} - -function findDomElementByComponentId( - container: HTMLElement, - componentId: string, -): HTMLElement | undefined { - const elements = Array.from(container.querySelectorAll('*')); - for (const el of elements) { - const a2uiEl = el as unknown as A2uiLitElement; - if (a2uiEl.context?.componentModel.id === componentId) { - return el as HTMLElement; - } - } - return undefined; -} - -function getDirectChildNodes( - node: ComponentNode, -): Array> { - const props = getValue(node.props); - const childrenProp = props.children; - if (Array.isArray(childrenProp)) { - return childrenProp.filter(isComponentNode) as Array>; - } - const childProp = props.child; - if (isComponentNode(childProp)) { - return [childProp as ComponentNode]; - } - return []; -} - -function findDomElementForNode( - container: HTMLElement, - target: ComponentNode, -): HTMLElement | undefined { - const elements = Array.from(container.querySelectorAll('*')); - for (const el of elements) { - const a2uiEl = el as unknown as { - node?: ComponentNode; - context?: {componentModel: {id: string}; dataContext: {path: string}}; - }; - if (a2uiEl.node === target) { - return el as HTMLElement; - } - if ( - a2uiEl.context?.componentModel.id === target.componentId && - a2uiEl.context?.dataContext.path === target.dataPath - ) { - return el as HTMLElement; - } - } - return undefined; -} - -function resolveIdentityElement( - surfaceEl: A2uiSurface, - identityPath: string, -): HTMLElement | undefined { - if (identityPath === 'root') { - return findDomElementByComponentId(surfaceEl, 'root'); - } - const match = /^root\/children\/(\d+)$/.exec(identityPath); - if (match && surfaceEl.rootNode) { - const idx = Number(match[1]); - const children = getDirectChildNodes(surfaceEl.rootNode); - const targetNode = children[idx]; - return targetNode ? findDomElementForNode(surfaceEl, targetNode) : undefined; - } - return findDomElementByComponentId(surfaceEl, identityPath); -} - -function snapshotViewIdentities(surfaceEl: A2uiSurface | undefined): Map { - const map = new Map(); - if (!surfaceEl?.rootNode) { - return map; - } - const rootEl = resolveIdentityElement(surfaceEl, 'root'); - if (rootEl) { - map.set('root', rootEl); - } - const children = getDirectChildNodes(surfaceEl.rootNode); - children.forEach((child, index) => { - const childEl = findDomElementForNode(surfaceEl, child); - if (childEl) { - map.set(`root/children/${index}`, childEl); - } - }); - return map; -} - -function unwrapProp(val: unknown): unknown { - return val instanceof ResolvedBinding ? val.value : val; -} - -function assertNodeScalarProps( - surfaceEl: A2uiSurface, - actual: ComponentNode, - expected: ExpectedViewNode, -): void { - const props = getValue(actual.props); - if (expected.text !== undefined) { - assert.equal(unwrapProp(props.text), expected.text); - const domEl = findDomElementForNode(surfaceEl, actual); - assert.ok(domEl?.textContent?.includes(expected.text)); - } - if (expected.value !== undefined) { - assert.deepEqual(unwrapProp(props.value), expected.value); - } - if (expected.badgeText !== undefined) { - assert.equal(unwrapProp(props.badgeText), expected.badgeText); - const domEl = findDomElementForNode(surfaceEl, actual); - assert.ok(domEl?.textContent?.includes(expected.badgeText)); - } - if (expected.isValid !== undefined) { - assert.equal(props.isValid, expected.isValid); - const domEl = findDomElementForNode(surfaceEl, actual); - const inputContainer = domEl?.querySelector('.a2ui-textfield'); - if (inputContainer) { - assert.equal(inputContainer.classList.contains('a2ui-invalid'), !expected.isValid); - } - } - if (expected.validationErrors !== undefined) { - const actualErrors = props.validationErrors as string[] | undefined; - assert.deepEqual(actualErrors, expected.validationErrors); - } - if (expected.validationResults !== undefined) { - assert.deepEqual(props.validationResults, expected.validationResults); - } -} - -function assertExpectedViewTree( - surfaceEl: A2uiSurface, - actual: ComponentNode, - expected: ExpectedViewNode, -): void { - assert.equal(actual.componentId, expected.id); - if (expected.type !== undefined) { - assert.equal(actual.type, expected.type); - } - if (expected.state !== undefined) { - assert.equal(actual.state, expected.state); - } - if (actual.state === 'resolved') { - const domEl = findDomElementForNode(surfaceEl, actual); - assert.ok(domEl, `Expected mounted DOM element for "${actual.componentId}"`); - } - assertNodeScalarProps(surfaceEl, actual, expected); - - if (expected.children !== undefined) { - const actualChildren = getDirectChildNodes(actual); - assert.equal( - actualChildren.length, - expected.children.length, - `Child count mismatch on "${actual.componentId}"`, - ); - for (let i = 0; i < expected.children.length; i++) { - assertExpectedViewTree(surfaceEl, actualChildren[i], expected.children[i]); - } - } -} - -function resolveAccessibleLabel(el: HTMLElement): string | undefined { - const explicit = el.getAttribute('aria-label'); - if (explicit) { - return explicit; - } - const innerButton = el.querySelector('button'); - if (innerButton?.textContent) { - return innerButton.textContent.trim(); - } - const innerLabel = el.querySelector('label'); - if (innerLabel?.textContent) { - return innerLabel.textContent.trim(); - } - return el.textContent?.trim() || undefined; -} - -function resolveSemanticRole(el: HTMLElement): string | undefined { - const explicitRole = el.getAttribute('role'); - if (explicitRole) { - return explicitRole; - } - if (el.querySelector('button')) { - return 'button'; - } - if (el.querySelector('input[type="checkbox"]')) { - return 'checkbox'; - } - if (el.querySelector('input[type="radio"]')) { - return 'radiogroup'; - } - if ( - el.querySelector( - 'input:not([type="checkbox"]):not([type="radio"]):not([type="range"]), textarea', - ) - ) { - return 'textbox'; - } - return undefined; -} - -function assertAccessibilityNode( - container: HTMLElement, - componentId: string, - expected: ExpectedAccessibilityNode, -): void { - const el = findDomElementByComponentId(container, componentId); - assert.ok(el, `Expected DOM element for component "${componentId}"`); - - if (expected.label !== undefined) { - assert.equal(resolveAccessibleLabel(el), expected.label); - } - if (expected.description !== undefined) { - assert.equal(el.getAttribute('aria-description'), expected.description); - } - if (expected.live !== undefined) { - assert.equal(el.getAttribute('aria-live'), expected.live); - } - if (expected.hidden !== undefined) { - assert.equal(el.getAttribute('aria-hidden'), String(expected.hidden)); - } - if (expected.role !== undefined) { - assert.equal(resolveSemanticRole(el), expected.role); - } - if (expected.checked !== undefined) { - const checkbox = el.querySelector('input[type="checkbox"]') as HTMLInputElement | null; - assert.equal(checkbox?.checked, expected.checked); - } -} - -function assertAxeRules(container: HTMLElement, rules: string[]): void { - if (rules.includes('button-name')) { - const buttons = Array.from(container.querySelectorAll('button')); - for (const btn of buttons) { - const host = btn.closest('a2ui-basic-button') as HTMLElement | null; - const name = - btn.getAttribute('aria-label') || - host?.getAttribute('aria-label') || - btn.textContent?.trim(); - assert.ok(name, 'Expected accessible name on rendered button'); - } - } - if (rules.includes('aria-valid-attr-value')) { - const liveNodes = Array.from(container.querySelectorAll('[aria-live]')); - for (const node of liveNodes) { - const val = node.getAttribute('aria-live'); - assert.ok( - val !== null && ['off', 'polite', 'assertive'].includes(val), - `Invalid aria-live attribute value: ${String(val)}`, - ); - } - } -} - -function countActiveSurfaceSubscriptions(surface: SurfaceModel): number { - const onCreatedSize = (surface.componentsModel.onCreated as unknown as {listeners: Set}) - .listeners.size; - const onDeletedSize = (surface.componentsModel.onDeleted as unknown as {listeners: Set}) - .listeners.size; - const dataSubsSize = (surface.dataModel as unknown as {subscriptions: Map>}) - .subscriptions.size; - return onCreatedSize + onDeletedSize + dataSubsSize; -} - -async function ensureSurfaceMounted( - session: AdapterSession, - mountedElements: HTMLElement[], -): Promise { - if (session.surfaceEl) { - return; - } - const surface = - session.processor.model.getSurface('adapter_surface') ?? - session.processor.model.getSurface('a11y_surface'); - assert.ok(surface, 'Expected surface to exist in MessageProcessor'); - session.surface = surface; - - surface.onAction.subscribe(action => { - session.recordedActions.push({ - name: action.name, - surfaceId: action.surfaceId, - sourceComponentId: action.sourceComponentId, - context: action.context, - }); - }); - surface.onError.subscribe(err => { - session.recordedErrors.push({ - code: err.code, - message: err.message, - surfaceId: surface.id, - }); - }); - - const el = document.createElement('a2ui-surface') as unknown as A2uiSurface; - mountedElements.push(el); - document.body.appendChild(el); - el.surface = surface; - session.surfaceEl = el; -} - -function performStepOperation(session: AdapterSession, step: AdapterStep): void { - if (step.op === 'process_messages') { - session.processor.processMessages(step.messages ?? []); - return; - } - if (step.op === 'user_input' && session.surfaceEl && step.targetNode) { - const hostEl = findDomElementByComponentId(session.surfaceEl, step.targetNode); - const input = hostEl?.querySelector('input, textarea') as HTMLInputElement | undefined; - assert.ok(input, `Expected input element inside "${step.targetNode}"`); - input.value = String(step.inputValue ?? ''); - input.dispatchEvent(new Event('input', {bubbles: true})); - return; - } - if (step.op === 'user_gesture' && session.surfaceEl && step.targetNode) { - const hostEl = findDomElementByComponentId(session.surfaceEl, step.targetNode); - const btn = (hostEl?.querySelector('button') ?? hostEl) as HTMLElement | undefined; - assert.ok(btn, `Expected clickable element for "${step.targetNode}"`); - const origOpen = window.open; - window.open = () => null; - try { - btn.click(); - } finally { - window.open = origOpen; - } - return; - } - if (step.op === 'programmatic_invoke' && session.surfaceEl && step.targetNode) { - const hostEl = findDomElementByComponentId(session.surfaceEl, step.targetNode) as unknown as - | A2uiLitElement - | undefined; - const rawAction = hostEl?.context?.componentModel.properties.action as - | Record - | undefined; - assert.ok(rawAction && hostEl?.context, `Expected action on "${step.targetNode}"`); - hostEl.context.dataContext.resolveAction(rawAction, { - userActivation: Boolean(step.userActivation), - }); - return; - } - if (step.op === 'unmount_surface' && session.surfaceEl) { - session.surfaceEl.remove(); - } -} - -function assertSameViewIdentities(session: AdapterSession, identityPaths: string[]): void { - if (!session.surfaceEl) { - return; - } - for (const idPath of identityPaths) { - const beforeEl = session.priorIdentities.get(idPath); - const afterEl = resolveIdentityElement(session.surfaceEl, idPath); - assert.ok(beforeEl && afterEl, `Missing view identity for "${idPath}"`); - assert.equal(afterEl, beforeEl, `Expected view identity to be preserved for "${idPath}"`); - } -} - -function assertRecordedErrors( - actualErrors: Array<{code: string; message: string; surfaceId: string}>, - expectedErrors: ExpectedError[], -): void { - assert.equal(actualErrors.length, expectedErrors.length); - for (let i = 0; i < expectedErrors.length; i++) { - const exp = expectedErrors[i]; - const act = actualErrors[i]; - if (exp.code !== undefined) { - assert.equal(act.code, exp.code); - } - if (exp.surfaceId !== undefined) { - assert.equal(act.surfaceId, exp.surfaceId); - } - if (exp.message !== undefined) { - assert.ok(act.message.includes(exp.message)); - } - } -} - -function verifyStepExpectations(session: AdapterSession, step: AdapterStep): void { - if (step.expectViewTree && session.surfaceEl?.rootNode) { - assertExpectedViewTree(session.surfaceEl, session.surfaceEl.rootNode, step.expectViewTree); - } - if (step.expectDataModel && session.surface) { - assert.deepEqual(session.surface.dataModel.get('/'), step.expectDataModel); - } - if (step.expectSameViewIdentity) { - assertSameViewIdentities(session, step.expectSameViewIdentity); - } - if (step.expectActions !== undefined) { - assert.deepEqual(session.recordedActions, step.expectActions); - } - if (step.expectErrors !== undefined) { - assertRecordedErrors(session.recordedErrors, step.expectErrors); - } - if (step.expectAccessibilityTree && session.surfaceEl) { - for (const [compId, expected] of Object.entries(step.expectAccessibilityTree)) { - assertAccessibilityNode(session.surfaceEl, compId, expected); - } - } - if (step.expectActiveSubscriptions !== undefined && session.surface) { - assert.equal(countActiveSurfaceSubscriptions(session.surface), step.expectActiveSubscriptions); - } -} - describe('@a2ui/lit Adapter & Accessibility Conformance', () => { const accessibilityCases = yaml.load( fs.readFileSync(path.join(CONFORMANCE_ROOT, 'accessibility.yaml'), 'utf8'), @@ -623,6 +50,7 @@ describe('@a2ui/lit Adapter & Accessibility Conformance', () => { ) as AdapterTestCase[]; const mountedElements: HTMLElement[] = []; + let pendingManifest: ManifestRecord | undefined; before(async () => { setupTestDom(); @@ -633,40 +61,63 @@ describe('@a2ui/lit Adapter & Accessibility Conformance', () => { while (mountedElements.length > 0) { mountedElements.pop()?.remove(); } + if (pendingManifest) { + writeManifestRecord(pendingManifest); + pendingManifest = undefined; + } }); after(() => { + assertAllUnsupportedMatched(); teardownTestDom(); }); describe('Accessibility Conformance (accessibility.yaml)', () => { for (const testCase of accessibilityCases) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - const catalog = createAugmentedBasicCatalog(); - const session: AdapterSession = { - processor: new MessageProcessor([catalog]), - recordedActions: [], - recordedErrors: [], - priorIdentities: new Map(), + it(`${testCase.name}: ${testCase.description ?? ''}`, async t => { + pendingManifest = { + harness: 'lit', + suite: 'core/accessibility.yaml', + case: testCase.name, + status: 'failed', }; - const steps: AccessibilityStep[] = testCase.steps ?? [ - { - messages: testCase.messages ?? [], - assertions: testCase.assertions ?? {}, - }, - ]; - - for (const step of steps) { - session.processor.processMessages(step.messages); - await ensureSurfaceMounted(session, mountedElements); - await settleSurface(session.surfaceEl!); - const tree = step.assertions.accessibilityTree ?? {}; - for (const [compId, expected] of Object.entries(tree)) { - assertAccessibilityNode(session.surfaceEl!, compId, expected); + try { + assertKnownAccessibilityCase(testCase); + const steps: AccessibilityStep[] = testCase.steps ?? [ + { + messages: testCase.messages ?? [], + assertions: testCase.assertions ?? {}, + }, + ]; + const skipReason = findUnsupportedReason( + testCase.name, + steps as unknown as Array>, + ); + if (skipReason) { + pendingManifest.status = 'skipped'; + pendingManifest.reason = skipReason; + t.skip(skipReason); + return; } - if (step.assertions.axeCore) { - assertAxeRules(session.surfaceEl!, step.assertions.axeCore); + + const session = mountSurface(undefined, mountedElements); + for (const step of steps) { + session.processor.processMessages(step.messages); + await ensureSurfaceMounted(session, mountedElements); + await settleSurface(session.surfaceEl!); + const tree = step.assertions.accessibilityTree ?? {}; + for (const [compId, expected] of Object.entries(tree)) { + assertAccessibilityNode(session.surfaceEl!, compId, expected); + } + if (step.assertions.axeCore) { + assertAxeRules(session.surfaceEl!, step.assertions.axeCore); + } } + pendingManifest.status = 'passed'; + } catch (err) { + pendingManifest.status = 'failed'; + pendingManifest.reason = err instanceof Error ? err.message : String(err); + throw err; } }); } @@ -674,41 +125,35 @@ describe('@a2ui/lit Adapter & Accessibility Conformance', () => { describe('Framework Adapter Conformance (adapter_conformance.yaml)', () => { for (const testCase of adapterCases) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - const catalogs: Array> = [ - createAugmentedBasicCatalog(), - ...(testCase.catalogs ?? []).map(buildSecondaryCatalog), - ]; - const session: AdapterSession = { - processor: new MessageProcessor(catalogs), - recordedActions: [], - recordedErrors: [], - priorIdentities: new Map(), + it(`${testCase.name}: ${testCase.description ?? ''}`, async t => { + pendingManifest = { + harness: 'lit', + suite: 'core/adapter_conformance.yaml', + case: testCase.name, + status: 'failed', }; - - for (const step of testCase.steps) { - session.recordedActions = []; - session.recordedErrors = []; - session.priorIdentities = snapshotViewIdentities(session.surfaceEl); - - if (step.op === 'process_messages' && !session.surfaceEl) { - for (const msg of step.messages ?? []) { - const record = msg as Record; - if ('createSurface' in record) { - session.processor.processMessages([msg]); - await ensureSurfaceMounted(session, mountedElements); - } else { - session.processor.processMessages([msg]); - } - } - } else { - performStepOperation(session, step); + try { + assertKnownAdapterCase(testCase); + const skipReason = findUnsupportedReason( + testCase.name, + testCase.steps as unknown as Array>, + ); + if (skipReason) { + pendingManifest.status = 'skipped'; + pendingManifest.reason = skipReason; + t.skip(skipReason); + return; } - if (session.surfaceEl && step.op !== 'unmount_surface') { - await settleSurface(session.surfaceEl); + const session = mountSurface(testCase, mountedElements); + for (const step of testCase.steps) { + await runStep(session, step); } - verifyStepExpectations(session, step); + pendingManifest.status = 'passed'; + } catch (err) { + pendingManifest.status = 'failed'; + pendingManifest.reason = err instanceof Error ? err.message : String(err); + throw err; } }); } diff --git a/typescript/renderers/lit/src/tests/adapter_harness.ts b/typescript/renderers/lit/src/tests/adapter_harness.ts new file mode 100644 index 0000000000..6d80a3065f --- /dev/null +++ b/typescript/renderers/lit/src/tests/adapter_harness.ts @@ -0,0 +1,958 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import {asyncUpdate} from './dom-setup.js'; +import {html} from 'lit'; +import {z} from 'zod'; + +import { + Catalog, + MessageProcessor, + ResolvedBinding, + getValue, + isComponentNode, + type ComponentNode, + type FunctionImplementation, + type ProcessableMessage, + type SurfaceModel, +} from '@a2ui/web_core'; +import {basicCatalog as v1BasicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; +import {A2uiLitElement} from '@a2ui/web_core/universal'; + +import type {A2uiSurface} from '../index.js'; +import type {LitComponentApi} from '../types.js'; + +export interface ExpectedAccessibilityNode { + label?: string; + description?: string; + live?: string; + hidden?: boolean; + role?: string; + checked?: boolean; +} + +export interface AccessibilityAssertions { + accessibilityTree?: Record; + axeCore?: string[]; +} + +export interface AccessibilityStep { + description?: string; + messages: ProcessableMessage[]; + assertions: AccessibilityAssertions; +} + +export interface AccessibilityTestCase { + name: string; + description?: string; + features?: string[]; + protocolVersion?: string; + action?: string; + catalogPath?: string; + surface?: Record; + messages?: ProcessableMessage[]; + assertions?: AccessibilityAssertions; + steps?: AccessibilityStep[]; +} + +export interface ExpectedViewNode { + id: string; + type?: string; + state?: string; + text?: string; + value?: unknown; + badgeText?: string; + isValid?: boolean; + validationErrors?: string[]; + validationResults?: unknown[]; + children?: ExpectedViewNode[]; +} + +export interface ExpectedAction { + name: string; + surfaceId: string; + sourceComponentId: string; + context?: Record; +} + +export interface ExpectedError { + code?: string; + surfaceId?: string; + message?: string; +} + +export interface AdapterStep { + description?: string; + op: + | 'process_messages' + | 'user_input' + | 'user_gesture' + | 'programmatic_invoke' + | 'unmount_surface'; + messages?: ProcessableMessage[]; + targetNode?: string; + inputValue?: unknown; + userActivation?: boolean; + expectViewTree?: ExpectedViewNode; + expectDataModel?: Record; + expectSameViewIdentity?: string[]; + expectActions?: ExpectedAction[]; + expectErrors?: ExpectedError[]; + expectAccessibilityTree?: Record; + expectActiveSubscriptions?: number; +} + +export interface SecondaryCatalogSpec { + catalogId: string; + protocolVersion?: string; + components: Record; +} + +export interface AdapterTestCase { + name: string; + description?: string; + features?: string[]; + protocolVersion?: string; + action?: string; + catalogPaths?: string[]; + catalogs?: SecondaryCatalogSpec[]; + steps: AdapterStep[]; +} + +export interface AdapterSession { + processor: MessageProcessor; + surface?: SurfaceModel; + surfaceEl?: A2uiSurface; + mountedElements: HTMLElement[]; + recordedActions: ExpectedAction[]; + recordedErrors: Array<{code: string; message: string; surfaceId: string}>; + priorIdentities: Map; +} + +export interface ManifestRecord { + harness: string; + suite: string; + case: string; + status: 'passed' | 'failed' | 'skipped' | 'xfailed'; + reason?: string; +} + +/** + * Reasoned skip entries keyed by `""` or `"/"`. + * Every entry in this map must match at least one case during the run. + */ +export const UNSUPPORTED: Record = {}; + +const matchedUnsupportedKeys = new Set(); + +export const KNOWN_KEYS = { + adapterCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPaths', + 'catalogs', + 'steps', + ]), + adapterCatalog: new Set(['catalogId', 'protocolVersion', 'components']), + adapterStep: new Set([ + 'description', + 'op', + 'messages', + 'targetNode', + 'inputValue', + 'userActivation', + 'expectViewTree', + 'expectAccessibilityTree', + 'expectDataModel', + 'expectActions', + 'expectErrors', + 'expectSameViewIdentity', + 'expectActiveSubscriptions', + ]), + adapterOps: new Set([ + 'process_messages', + 'user_input', + 'user_gesture', + 'programmatic_invoke', + 'unmount_surface', + ]), + viewNode: new Set([ + 'id', + 'type', + 'state', + 'text', + 'value', + 'badgeText', + 'isValid', + 'validationErrors', + 'validationResults', + 'children', + ]), + validationResult: new Set(['valid', 'code', 'message', 'severity']), + expectedAction: new Set(['name', 'surfaceId', 'sourceComponentId', 'context']), + expectedError: new Set(['code', 'surfaceId', 'message']), + accessibilityCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPath', + 'surface', + 'messages', + 'assertions', + 'steps', + ]), + accessibilityStep: new Set(['description', 'messages', 'assertions']), + accessibilityAssertions: new Set(['accessibilityTree', 'axeCore']), + accessibilityNode: new Set(['label', 'description', 'live', 'hidden', 'role', 'checked']), +} as const; + +function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: string): void { + if (obj === undefined || obj === null) { + return; + } + assert.ok(typeof obj === 'object' && !Array.isArray(obj), `${where}: expected object`); + const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); + assert.deepEqual( + unknown, + [], + `${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`, + ); +} + +function assertKnownViewNodeKeys(node: unknown, where: string): void { + if (node === undefined || node === null) { + return; + } + assertObjectKeys(node, KNOWN_KEYS.viewNode, where); + const record = node as Record; + if (Array.isArray(record.validationResults)) { + record.validationResults.forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.validationResult, `${where}.validationResults[${idx}]`); + }); + } + if (Array.isArray(record.children)) { + record.children.forEach((child, idx) => { + assertKnownViewNodeKeys(child, `${where}.children[${idx}]`); + }); + } +} + +export function assertKnownKeys( + obj: unknown, + allowed: ReadonlySet = KNOWN_KEYS.adapterStep, + where = 'step', +): void { + assertObjectKeys(obj, allowed, where); + if (allowed !== KNOWN_KEYS.adapterStep || !obj || typeof obj !== 'object') { + return; + } + const step = obj as Record; + if (step.op !== undefined) { + assert.ok( + typeof step.op === 'string' && KNOWN_KEYS.adapterOps.has(step.op), + `${where}.op: unsupported op "${String(step.op)}"`, + ); + } + if (step.expectViewTree !== undefined) { + assertKnownViewNodeKeys(step.expectViewTree, `${where}.expectViewTree`); + } + if (step.expectAccessibilityTree !== undefined) { + assert.ok( + typeof step.expectAccessibilityTree === 'object' && + step.expectAccessibilityTree !== null && + !Array.isArray(step.expectAccessibilityTree), + `${where}.expectAccessibilityTree: expected object`, + ); + for (const [compId, node] of Object.entries( + step.expectAccessibilityTree as Record, + )) { + assertObjectKeys( + node, + KNOWN_KEYS.accessibilityNode, + `${where}.expectAccessibilityTree.${compId}`, + ); + } + } + if (step.expectActions !== undefined) { + assert.ok(Array.isArray(step.expectActions), `${where}.expectActions: expected array`); + step.expectActions.forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.expectedAction, `${where}.expectActions[${idx}]`); + }); + } + if (step.expectErrors !== undefined) { + assert.ok(Array.isArray(step.expectErrors), `${where}.expectErrors: expected array`); + step.expectErrors.forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.expectedError, `${where}.expectErrors[${idx}]`); + }); + } +} + +export function assertKnownAdapterCase(testCase: AdapterTestCase): void { + assertObjectKeys(testCase, KNOWN_KEYS.adapterCase, `case(${testCase.name})`); + if (testCase.action !== undefined) { + assert.equal( + testCase.action, + 'render_surface', + `case(${testCase.name}): unsupported action "${testCase.action}"`, + ); + } + for (const [idx, cat] of (testCase.catalogs ?? []).entries()) { + assertObjectKeys(cat, KNOWN_KEYS.adapterCatalog, `case(${testCase.name}).catalogs[${idx}]`); + } + for (const [idx, step] of testCase.steps.entries()) { + assertKnownKeys(step, KNOWN_KEYS.adapterStep, `case(${testCase.name}).steps[${idx}]`); + } +} + +export function assertKnownAccessibilityAssertions( + assertions: AccessibilityAssertions | undefined, + where: string, +): void { + if (!assertions) { + return; + } + assertObjectKeys(assertions, KNOWN_KEYS.accessibilityAssertions, where); + for (const [compId, node] of Object.entries(assertions.accessibilityTree ?? {})) { + assertObjectKeys(node, KNOWN_KEYS.accessibilityNode, `${where}.accessibilityTree.${compId}`); + } +} + +export function assertKnownAccessibilityCase(testCase: AccessibilityTestCase): void { + assertObjectKeys(testCase, KNOWN_KEYS.accessibilityCase, `case(${testCase.name})`); + if (testCase.action !== undefined) { + assert.equal( + testCase.action, + 'accessibility_check', + `case(${testCase.name}): unsupported action "${testCase.action}"`, + ); + } + assertKnownAccessibilityAssertions(testCase.assertions, `case(${testCase.name}).assertions`); + for (const [idx, step] of (testCase.steps ?? []).entries()) { + assertObjectKeys(step, KNOWN_KEYS.accessibilityStep, `case(${testCase.name}).steps[${idx}]`); + assertKnownAccessibilityAssertions( + step.assertions, + `case(${testCase.name}).steps[${idx}].assertions`, + ); + } +} + +export function findUnsupportedReason( + caseName: string, + steps: Array> = [], +): string | undefined { + if (Object.hasOwn(UNSUPPORTED, caseName)) { + matchedUnsupportedKeys.add(caseName); + return UNSUPPORTED[caseName]; + } + for (const step of steps) { + const op = typeof step.op === 'string' ? step.op : undefined; + if (op) { + const opKey = `${caseName}/${op}`; + if (Object.hasOwn(UNSUPPORTED, opKey)) { + matchedUnsupportedKeys.add(opKey); + return UNSUPPORTED[opKey]; + } + } + for (const key of Object.keys(step)) { + const stepKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, stepKey)) { + matchedUnsupportedKeys.add(stepKey); + return UNSUPPORTED[stepKey]; + } + } + } + return undefined; +} + +export function assertAllUnsupportedMatched(): void { + const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); + assert.deepEqual( + unused, + [], + `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, + ); +} + +export function writeManifestRecord(rec: ManifestRecord): void { + const manifestDir = process.env.A2UI_CONFORMANCE_MANIFEST_DIR; + if (!manifestDir) { + return; + } + fs.mkdirSync(manifestDir, {recursive: true}); + const filePath = path.join(manifestDir, `${rec.harness}-${process.pid}.jsonl`); + const payload: Record = { + harness: rec.harness, + suite: rec.suite.replaceAll('\\', '/'), + case: rec.case, + status: rec.status, + }; + if (rec.reason) { + payload.reason = rec.reason; + } + fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, 'utf8'); +} + +export async function settleSurface(root: HTMLElement): Promise { + for (let pass = 0; pass < 3; pass++) { + await asyncUpdate(root, () => {}); + const elements = [root, ...Array.from(root.querySelectorAll('*'))]; + for (const el of elements) { + const litEl = el as unknown as {updateComplete?: Promise}; + if (litEl.updateComplete) { + await litEl.updateComplete; + } + } + } +} + +function createIncrementCounterFn(): FunctionImplementation { + return { + name: 'incrementCounter', + returnType: 'void', + schema: z.object({ + path: z.string(), + step: z.number(), + }), + execute: (args: {path: string; step: number}, context) => { + const current = Number(context.dataModel.get(args.path) ?? 0); + context.dataModel.set(args.path, current + args.step); + }, + }; +} + +export function createAugmentedBasicCatalog(): Catalog { + const base = v1BasicCatalog as unknown as Catalog; + const components = Array.from(base.components.values()); + const functions = [...Array.from(base.functions?.values() ?? []), createIncrementCounterFn()]; + return new Catalog( + base.id, + base.protocolVersion ?? '1.0', + components, + functions, + ); +} + +export function buildSecondaryCatalog(spec: SecondaryCatalogSpec): Catalog { + const components: LitComponentApi[] = []; + const badgeSchema = z.object({ + badgeText: z.string().optional(), + }); + for (const compName of Object.keys(spec.components)) { + const tagName = `a2ui-conformance-${compName.toLowerCase()}`; + let element = customElements.get(tagName) as CustomElementConstructor | undefined; + if (!element) { + class CustomBadgeElement extends A2uiLitElement { + override createRenderRoot() { + return this; + } + protected override render() { + const props = this.controller?.props as Record | undefined; + return html`${String(props?.badgeText ?? '')}`; + } + } + element = CustomBadgeElement; + } + components.push({ + name: compName, + tagName, + element, + schema: badgeSchema, + }); + } + return new Catalog( + spec.catalogId, + (spec.protocolVersion ?? 'v1.0') as 'v1.0', + components, + [], + ); +} + +function findDomElementByComponentId( + container: HTMLElement, + componentId: string, +): HTMLElement | undefined { + const elements = Array.from(container.querySelectorAll('*')); + for (const el of elements) { + const a2uiEl = el as unknown as A2uiLitElement; + if (a2uiEl.context?.componentModel.id === componentId) { + return el as HTMLElement; + } + } + return undefined; +} + +function getDirectChildNodes( + node: ComponentNode, +): Array> { + const props = getValue(node.props); + const childrenProp = props.children; + if (Array.isArray(childrenProp)) { + return childrenProp.filter(isComponentNode) as Array>; + } + const childProp = props.child; + if (isComponentNode(childProp)) { + return [childProp as ComponentNode]; + } + return []; +} + +function findDomElementForNode( + container: HTMLElement, + target: ComponentNode, +): HTMLElement | undefined { + const elements = Array.from(container.querySelectorAll('*')); + for (const el of elements) { + const a2uiEl = el as unknown as { + node?: ComponentNode; + context?: {componentModel: {id: string}; dataContext: {path: string}}; + }; + if (a2uiEl.node === target) { + return el as HTMLElement; + } + if ( + a2uiEl.context?.componentModel.id === target.componentId && + a2uiEl.context?.dataContext.path === target.dataPath + ) { + return el as HTMLElement; + } + } + return undefined; +} + +function resolveIdentityElement( + surfaceEl: A2uiSurface, + identityPath: string, +): HTMLElement | undefined { + if (identityPath === 'root') { + return findDomElementByComponentId(surfaceEl, 'root'); + } + const match = /^root\/children\/(\d+)$/.exec(identityPath); + if (match && surfaceEl.rootNode) { + const idx = Number(match[1]); + const children = getDirectChildNodes(surfaceEl.rootNode); + const targetNode = children[idx]; + return targetNode ? findDomElementForNode(surfaceEl, targetNode) : undefined; + } + return findDomElementByComponentId(surfaceEl, identityPath); +} + +function snapshotViewIdentities(surfaceEl: A2uiSurface | undefined): Map { + const map = new Map(); + if (!surfaceEl?.rootNode) { + return map; + } + const rootEl = resolveIdentityElement(surfaceEl, 'root'); + if (rootEl) { + map.set('root', rootEl); + } + const children = getDirectChildNodes(surfaceEl.rootNode); + children.forEach((child, index) => { + const childEl = findDomElementForNode(surfaceEl, child); + if (childEl) { + map.set(`root/children/${index}`, childEl); + } + }); + return map; +} + +function unwrapProp(val: unknown): unknown { + return val instanceof ResolvedBinding ? val.value : val; +} + +function assertNodeScalarProps( + surfaceEl: A2uiSurface | undefined, + actual: ComponentNode, + expected: ExpectedViewNode, +): void { + const props = getValue(actual.props); + if (expected.text !== undefined) { + assert.equal(unwrapProp(props.text), expected.text); + if (surfaceEl) { + const domEl = findDomElementForNode(surfaceEl, actual); + assert.ok(domEl?.textContent?.includes(expected.text)); + } + } + if (expected.value !== undefined) { + assert.deepEqual(unwrapProp(props.value), expected.value); + } + if (expected.badgeText !== undefined) { + assert.equal(unwrapProp(props.badgeText), expected.badgeText); + if (surfaceEl) { + const domEl = findDomElementForNode(surfaceEl, actual); + assert.ok(domEl?.textContent?.includes(expected.badgeText)); + } + } + if (expected.isValid !== undefined) { + assert.equal(props.isValid, expected.isValid); + if (surfaceEl) { + const domEl = findDomElementForNode(surfaceEl, actual); + const inputContainer = domEl?.querySelector('.a2ui-textfield'); + if (inputContainer) { + assert.equal(inputContainer.classList.contains('a2ui-invalid'), !expected.isValid); + } + } + } + if (expected.validationErrors !== undefined) { + const actualErrors = props.validationErrors as string[] | undefined; + assert.deepEqual(actualErrors, expected.validationErrors); + } + if (expected.validationResults !== undefined) { + assert.deepEqual(props.validationResults, expected.validationResults); + } +} + +export function extractViewTree( + session: AdapterSession, +): ComponentNode | undefined { + return session.surfaceEl?.rootNode; +} + +export function assertViewTree( + actual: ComponentNode | undefined, + expected: ExpectedViewNode, + surfaceEl?: A2uiSurface, +): void { + assert.ok(actual, `Expected view node for "${expected.id}"`); + const resolvedSurfaceEl = + surfaceEl ?? + (document.body.querySelector('a2ui-surface') as unknown as A2uiSurface | undefined); + assert.equal(actual.componentId, expected.id); + if (expected.type !== undefined) { + assert.equal(actual.type, expected.type); + } + if (expected.state !== undefined) { + assert.equal(actual.state, expected.state); + } + if (actual.state === 'resolved' && resolvedSurfaceEl) { + const domEl = findDomElementForNode(resolvedSurfaceEl, actual); + assert.ok(domEl, `Expected mounted DOM element for "${actual.componentId}"`); + } + assertNodeScalarProps(resolvedSurfaceEl, actual, expected); + + if (expected.children !== undefined) { + const actualChildren = getDirectChildNodes(actual); + assert.equal( + actualChildren.length, + expected.children.length, + `Child count mismatch on "${actual.componentId}"`, + ); + for (let i = 0; i < expected.children.length; i++) { + assertViewTree(actualChildren[i], expected.children[i], resolvedSurfaceEl); + } + } +} + +function resolveAccessibleLabel(el: HTMLElement): string | undefined { + const explicit = el.getAttribute('aria-label'); + if (explicit) { + return explicit; + } + const innerButton = el.querySelector('button'); + if (innerButton?.textContent) { + return innerButton.textContent.trim(); + } + const innerLabel = el.querySelector('label'); + if (innerLabel?.textContent) { + return innerLabel.textContent.trim(); + } + return el.textContent?.trim() || undefined; +} + +function resolveSemanticRole(el: HTMLElement): string | undefined { + const explicitRole = el.getAttribute('role'); + if (explicitRole) { + return explicitRole; + } + if (el.querySelector('button')) { + return 'button'; + } + if (el.querySelector('input[type="checkbox"]')) { + return 'checkbox'; + } + if (el.querySelector('input[type="radio"]')) { + return 'radiogroup'; + } + if ( + el.querySelector( + 'input:not([type="checkbox"]):not([type="radio"]):not([type="range"]), textarea', + ) + ) { + return 'textbox'; + } + return undefined; +} + +export function assertAccessibilityNode( + container: HTMLElement, + componentId: string, + expected: ExpectedAccessibilityNode, +): void { + const el = findDomElementByComponentId(container, componentId); + assert.ok(el, `Expected DOM element for component "${componentId}"`); + + if (expected.label !== undefined) { + assert.equal(resolveAccessibleLabel(el), expected.label); + } + if (expected.description !== undefined) { + assert.equal(el.getAttribute('aria-description'), expected.description); + } + if (expected.live !== undefined) { + assert.equal(el.getAttribute('aria-live'), expected.live); + } + if (expected.hidden !== undefined) { + assert.equal(el.getAttribute('aria-hidden'), String(expected.hidden)); + } + if (expected.role !== undefined) { + assert.equal(resolveSemanticRole(el), expected.role); + } + if (expected.checked !== undefined) { + const checkbox = el.querySelector('input[type="checkbox"]') as HTMLInputElement | null; + assert.equal(checkbox?.checked, expected.checked); + } +} + +export function assertAxeRules(container: HTMLElement, rules: string[]): void { + if (rules.includes('button-name')) { + const buttons = Array.from(container.querySelectorAll('button')); + for (const btn of buttons) { + const host = btn.closest('a2ui-basic-button') as HTMLElement | null; + const name = + btn.getAttribute('aria-label') || + host?.getAttribute('aria-label') || + btn.textContent?.trim(); + assert.ok(name, 'Expected accessible name on rendered button'); + } + } + if (rules.includes('aria-valid-attr-value')) { + const liveNodes = Array.from(container.querySelectorAll('[aria-live]')); + for (const node of liveNodes) { + const val = node.getAttribute('aria-live'); + assert.ok( + val !== null && ['off', 'polite', 'assertive'].includes(val), + `Invalid aria-live attribute value: ${String(val)}`, + ); + } + } +} + +function countActiveSurfaceSubscriptions(surface: SurfaceModel): number { + const onCreatedSize = (surface.componentsModel.onCreated as unknown as {listeners: Set}) + .listeners.size; + const onDeletedSize = (surface.componentsModel.onDeleted as unknown as {listeners: Set}) + .listeners.size; + const dataSubsSize = (surface.dataModel as unknown as {subscriptions: Map>}) + .subscriptions.size; + return onCreatedSize + onDeletedSize + dataSubsSize; +} + +export async function ensureSurfaceMounted( + session: AdapterSession, + mountedElements: HTMLElement[] = session.mountedElements, +): Promise { + if (session.surfaceEl) { + return; + } + const surface = + session.processor.model.getSurface('adapter_surface') ?? + session.processor.model.getSurface('a11y_surface') ?? + session.processor.model.surfacesMap.values().next().value; + assert.ok(surface, 'Expected surface to exist in MessageProcessor'); + session.surface = surface; + + surface.onAction.subscribe(action => { + session.recordedActions.push({ + name: action.name, + surfaceId: action.surfaceId, + sourceComponentId: action.sourceComponentId, + context: action.context, + }); + }); + surface.onError.subscribe(err => { + session.recordedErrors.push({ + code: err.code, + message: err.message, + surfaceId: surface.id, + }); + }); + + const el = document.createElement('a2ui-surface') as unknown as A2uiSurface; + mountedElements.push(el); + if (mountedElements !== session.mountedElements) { + session.mountedElements.push(el); + } + document.body.appendChild(el); + el.surface = surface; + session.surfaceEl = el; +} + +function performStepOperation(session: AdapterSession, step: AdapterStep): void { + switch (step.op) { + case 'process_messages': + session.processor.processMessages(step.messages ?? []); + return; + case 'user_input': { + assert.ok(session.surfaceEl && step.targetNode, 'Expected surfaceEl and targetNode'); + const hostEl = findDomElementByComponentId(session.surfaceEl, step.targetNode); + const input = hostEl?.querySelector('input, textarea') as HTMLInputElement | undefined; + assert.ok(input, `Expected input element inside "${step.targetNode}"`); + input.value = String(step.inputValue ?? ''); + input.dispatchEvent(new Event('input', {bubbles: true})); + return; + } + case 'user_gesture': { + assert.ok(session.surfaceEl && step.targetNode, 'Expected surfaceEl and targetNode'); + const hostEl = findDomElementByComponentId(session.surfaceEl, step.targetNode); + const btn = (hostEl?.querySelector('button') ?? hostEl) as HTMLElement | undefined; + assert.ok(btn, `Expected clickable element for "${step.targetNode}"`); + const origOpen = window.open; + window.open = () => null; + try { + btn.click(); + } finally { + window.open = origOpen; + } + return; + } + case 'programmatic_invoke': { + assert.ok(session.surfaceEl && step.targetNode, 'Expected surfaceEl and targetNode'); + const hostEl = findDomElementByComponentId(session.surfaceEl, step.targetNode) as unknown as + | A2uiLitElement + | undefined; + const rawAction = hostEl?.context?.componentModel.properties.action as + | Record + | undefined; + assert.ok(rawAction && hostEl?.context, `Expected action on "${step.targetNode}"`); + hostEl.context.dataContext.resolveAction(rawAction, { + userActivation: Boolean(step.userActivation), + }); + return; + } + case 'unmount_surface': + session.surfaceEl?.remove(); + return; + default: + assert.fail(`Unsupported adapter step op: ${String(step.op)}`); + } +} + +function assertSameViewIdentities(session: AdapterSession, identityPaths: string[]): void { + assert.ok(session.surfaceEl, 'Expected mounted surfaceEl for view identity check'); + for (const idPath of identityPaths) { + const beforeEl = session.priorIdentities.get(idPath); + const afterEl = resolveIdentityElement(session.surfaceEl, idPath); + assert.ok(beforeEl && afterEl, `Missing view identity for "${idPath}"`); + assert.equal(afterEl, beforeEl, `Expected view identity to be preserved for "${idPath}"`); + } +} + +function assertRecordedErrors( + actualErrors: Array<{code: string; message: string; surfaceId: string}>, + expectedErrors: ExpectedError[], +): void { + assert.equal(actualErrors.length, expectedErrors.length); + for (let i = 0; i < expectedErrors.length; i++) { + const exp = expectedErrors[i]; + const act = actualErrors[i]; + if (exp.code !== undefined) { + assert.equal(act.code, exp.code); + } + if (exp.surfaceId !== undefined) { + assert.equal(act.surfaceId, exp.surfaceId); + } + if (exp.message !== undefined) { + assert.ok(act.message.includes(exp.message)); + } + } +} + +function verifyStepExpectations(session: AdapterSession, step: AdapterStep): void { + if (step.expectViewTree) { + const actualRoot = extractViewTree(session); + assertViewTree(actualRoot, step.expectViewTree, session.surfaceEl); + } + if (step.expectDataModel && session.surface) { + assert.deepEqual(session.surface.dataModel.get('/'), step.expectDataModel); + } + if (step.expectSameViewIdentity) { + assertSameViewIdentities(session, step.expectSameViewIdentity); + } + if (step.expectActions !== undefined) { + assert.deepEqual(session.recordedActions, step.expectActions); + } + if (step.expectErrors !== undefined) { + assertRecordedErrors(session.recordedErrors, step.expectErrors); + } + if (step.expectAccessibilityTree && session.surfaceEl) { + for (const [compId, expected] of Object.entries(step.expectAccessibilityTree)) { + assertAccessibilityNode(session.surfaceEl, compId, expected); + } + } + if (step.expectActiveSubscriptions !== undefined && session.surface) { + assert.equal(countActiveSurfaceSubscriptions(session.surface), step.expectActiveSubscriptions); + } +} + +export function mountSurface( + caseSpec?: Partial, + mountedElements: HTMLElement[] = [], +): AdapterSession { + const catalogs: Array> = [ + createAugmentedBasicCatalog(), + ...(caseSpec?.catalogs ?? []).map(buildSecondaryCatalog), + ]; + return { + processor: new MessageProcessor(catalogs), + mountedElements, + recordedActions: [], + recordedErrors: [], + priorIdentities: new Map(), + }; +} + +export async function runStep(session: AdapterSession, step: AdapterStep): Promise { + assertKnownKeys(step); + session.recordedActions = []; + session.recordedErrors = []; + session.priorIdentities = snapshotViewIdentities(session.surfaceEl); + + if (step.op === 'process_messages' && !session.surfaceEl) { + for (const msg of step.messages ?? []) { + const record = msg as Record; + if ('createSurface' in record) { + session.processor.processMessages([msg]); + await ensureSurfaceMounted(session, session.mountedElements); + } else { + session.processor.processMessages([msg]); + } + } + } else { + performStepOperation(session, step); + } + + if (session.surfaceEl && step.op !== 'unmount_surface') { + await settleSurface(session.surfaceEl); + } + verifyStepExpectations(session, step); +} diff --git a/typescript/renderers/react/tests/adapter_conformance.test.tsx b/typescript/renderers/react/tests/adapter_conformance.test.tsx index 99494346ba..d9bc5ac794 100644 --- a/typescript/renderers/react/tests/adapter_conformance.test.tsx +++ b/typescript/renderers/react/tests/adapter_conformance.test.tsx @@ -18,711 +18,118 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; -import {act, fireEvent, render, type RenderResult} from '@testing-library/react'; +import {act, render, type RenderResult} from '@testing-library/react'; import * as yaml from 'js-yaml'; import React from 'react'; -import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; -import {z} from 'zod'; +import {afterAll, afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; +import {MessageProcessor} from '@a2ui/web_core'; +import {A2uiSurface, type ReactCatalogComponent} from '../src'; import { - Catalog, - type ComponentNode, - createFunctionImplementation, - isComponentNode, - MessageProcessor, - peekValue, - type ProcessableMessage, - ResolvedBinding, - type SurfaceModel, -} from '@a2ui/web_core'; -import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; -import {CommonSchemas} from '@a2ui/web_core/v0_9'; -import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; -import {A2uiSurface, createComponentImplementation, type ReactCatalogComponent} from '../src'; + assertAllUnsupportedMatched, + assertAxeRules, + assertKnownAccessibilityCase, + assertKnownAdapterCase, + assertNodeAccessibility, + createDefaultAdapterCatalog, + findUnsupportedReason, + mountSurface, + runStep, + settleSurfaceDom, + writeManifestRecord, + type AccessibilityStep, + type AccessibilityTestCase, + type AdapterTestCase, + type ManifestRecord, +} from './adapter_harness'; const CONFORMANCE_DIR = path.resolve(import.meta.dirname, '../../../../conformance/core'); -interface ExpectedAccessibilityNode { - label?: string; - description?: string; - live?: string; - hidden?: boolean; - role?: string; - checked?: boolean; -} - -interface AccessibilityAssertions { - accessibilityTree?: Record; - axeCore?: string[]; -} - -interface AccessibilityStep { - messages: ProcessableMessage[]; - assertions: AccessibilityAssertions; -} - -interface AccessibilityTestCase { - name: string; - description?: string; - protocolVersion: string; - action: string; - messages?: ProcessableMessage[]; - assertions?: AccessibilityAssertions; - steps?: AccessibilityStep[]; -} - -interface ExpectedViewNode { - id?: string; - type?: string; - state?: string; - isValid?: boolean; - validationErrors?: string[]; - validationResults?: Array>; - children?: ExpectedViewNode[]; - [key: string]: unknown; -} - -interface ExpectedActionEvent { - name: string; - surfaceId?: string; - sourceComponentId?: string; - context?: Record; -} - -interface ExpectedErrorEntry { - code?: string; - surfaceId?: string; - message?: string; -} - -interface ExtraCatalogEntry { - catalogId: string; - protocolVersion?: string; - components: Record< - string, - { - type: string; - required?: string[]; - properties?: Record; - } - >; -} - -interface AdapterStep { - description?: string; - op?: - | 'process_messages' - | 'user_input' - | 'user_gesture' - | 'programmatic_invoke' - | 'unmount_surface'; - messages?: ProcessableMessage[]; - targetNode?: string; - inputValue?: unknown; - userActivation?: boolean; - expectViewTree?: ExpectedViewNode; - expectAccessibilityTree?: Record; - expectDataModel?: Record; - expectActions?: ExpectedActionEvent[]; - expectErrors?: ExpectedErrorEntry[]; - expectSameViewIdentity?: string[]; - expectActiveSubscriptions?: number; -} - -interface AdapterTestCase { - name: string; - description?: string; - protocolVersion: string; - action: string; - catalogs?: ExtraCatalogEntry[]; - steps: AdapterStep[]; -} - -interface HarnessState { - processor: MessageProcessor; - surface?: SurfaceModel; - rendered?: RenderResult; - unmounted: boolean; - actions: Array>; - errors: Array>; - previousNodes: Map; - previousDomElements: Map; -} - function loadYamlCases(fileName: string): T[] { const fullPath = path.join(CONFORMANCE_DIR, fileName); return yaml.load(fs.readFileSync(fullPath, 'utf8')) as T[]; } -function createDefaultAdapterCatalog(): Catalog { - const incrementCounter = createFunctionImplementation( - { - name: 'incrementCounter', - returnType: 'void', - schema: z.object({ - path: z.string().optional(), - step: z.number().optional(), - amount: z.number().optional(), - }), - }, - (args, context) => { - const targetPath = args.path ?? '/counter'; - const delta = args.amount ?? args.step ?? 1; - const current = Number(context.dataModel.get(targetPath) ?? 0); - context.dataModel.set(targetPath, current + delta); - }, - ); - return new Catalog( - basicCatalog.id, - basicCatalog.protocolVersion, - Array.from(basicCatalog.components.values()), - [...Array.from(basicCatalog.functions.values()), incrementCounter], - basicCatalog.themeSchema, - basicCatalog.instructions, - ); -} - -function buildExtraCatalog(entry: ExtraCatalogEntry): Catalog { - const implementations = Object.entries(entry.components).map(([name, def]) => { - const shape: Record = {}; - for (const [propKey, propSchema] of Object.entries(def.properties ?? {})) { - if (propKey === 'component') { - continue; - } - shape[propKey] = - propSchema.type === 'number' - ? CommonSchemas.DynamicNumber.optional() - : CommonSchemas.DynamicString.optional(); - } - return createComponentImplementation({name, schema: z.object(shape)}, ({props}) => ( - - {Object.entries(props) - .filter( - ([k, v]) => !k.startsWith('set') && (typeof v === 'string' || typeof v === 'number'), - ) - .map(([, v]) => String(v)) - .join(' ')} - - )); - }); - return new Catalog( - entry.catalogId, - entry.protocolVersion ?? '1.0', - implementations, - ); -} - -async function settleSurfaceDom(container: HTMLElement): Promise { - for (let pass = 0; pass < 3; pass++) { - await act(async () => { - await new Promise(resolve => setTimeout(resolve, 0)); - const elements = Array.from(container.querySelectorAll('*')); - for (const el of elements) { - const litEl = el as unknown as {updateComplete?: Promise}; - if (litEl.updateComplete) { - await litEl.updateComplete; - } - } - }); - } -} +const pendingManifestByTestName = new Map(); -function findComponentElement( - container: HTMLElement, - componentId: string, -): HTMLElement | undefined { - const candidates = Array.from(container.querySelectorAll('*')); - for (const candidate of candidates) { - const a2uiEl = candidate as unknown as A2uiWebComponentElement; - if ( - a2uiEl.context?.componentModel.id === componentId || - a2uiEl.node?.componentId === componentId - ) { - return candidate as HTMLElement; +afterEach(() => { + const currentTestName = expect.getState().currentTestName ?? ''; + for (const [key, rec] of pendingManifestByTestName) { + if (currentTestName.endsWith(key)) { + writeManifestRecord(rec); + pendingManifestByTestName.delete(key); } } - return undefined; -} - -function getRootComponentNode(container: HTMLElement): ComponentNode | undefined { - const first = container.firstElementChild as A2uiWebComponentElement | null; - return first?.node; -} - -function extractDirectChildNodes(node: ComponentNode): ComponentNode[] { - const props = peekValue(node.props); - if (Array.isArray(props.children)) { - return props.children.filter(isComponentNode); - } - if (isComponentNode(props.child)) { - return [props.child]; - } - return []; -} - -function collectMountedNodes(root: ComponentNode | undefined): Map { - const nodes = new Map(); - if (!root) { - return nodes; - } - const visit = (node: ComponentNode, nodePath: string): void => { - nodes.set(nodePath, node); - const props = peekValue(node.props); - if (Array.isArray(props.children)) { - props.children.forEach((child, i) => { - if (isComponentNode(child)) { - visit(child, `${nodePath}/children/${i}`); - } - }); - } else if (isComponentNode(props.child)) { - visit(props.child, `${nodePath}/child`); - } - }; - visit(root, 'root'); - return nodes; -} - -function unwrapBindingValue(value: unknown): unknown { - if (value instanceof ResolvedBinding) { - return unwrapBindingValue(value.value); - } - if (Array.isArray(value)) { - return value.map(unwrapBindingValue); - } - return value; -} - -function resolveNodeType( - node: ComponentNode, - surface: SurfaceModel, -): string { - if (node.state === 'cyclic' && node.type === 'Placeholder') { - return surface.componentsModel.get(node.componentId)?.type ?? node.type; - } - return node.type; -} - -const RESERVED_VIEW_KEYS = new Set([ - 'id', - 'type', - 'state', - 'isValid', - 'validationErrors', - 'validationResults', - 'children', -]); - -function assertViewNodeProps( - node: ComponentNode, - expected: ExpectedViewNode, - nodePath: string, -): void { - const props = peekValue(node.props); - if (expected.isValid !== undefined) { - expect(unwrapBindingValue(props.isValid), `${nodePath}.isValid`).toBe(expected.isValid); - } - if (expected.validationErrors !== undefined) { - expect(unwrapBindingValue(props.validationErrors), `${nodePath}.validationErrors`).toEqual( - expected.validationErrors, - ); - } - if (expected.validationResults !== undefined) { - expect(unwrapBindingValue(props.validationResults), `${nodePath}.validationResults`).toEqual( - expected.validationResults, - ); - } - for (const [key, expectedVal] of Object.entries(expected)) { - if (!RESERVED_VIEW_KEYS.has(key)) { - expect(unwrapBindingValue(props[key]), `${nodePath}.${key}`).toEqual(expectedVal); - } - } -} - -function findDomElementForNode( - container: HTMLElement, - node: ComponentNode, -): HTMLElement | undefined { - const match = Array.from(container.querySelectorAll('*')).find( - el => (el as A2uiWebComponentElement).node === node, - ); - return match as HTMLElement | undefined; -} - -function assertViewTree( - node: ComponentNode | undefined, - expected: ExpectedViewNode, - surface: SurfaceModel, - container: HTMLElement, - nodePath = 'root', -): void { - expect(node, `Missing node at ${nodePath}`).toBeDefined(); - const actual = node!; - if (expected.id !== undefined) { - expect(actual.componentId, `${nodePath}.id`).toBe(expected.id); - } - if (expected.type !== undefined) { - expect(resolveNodeType(actual, surface), `${nodePath}.type`).toBe(expected.type); - } - if (expected.state !== undefined) { - expect(actual.state, `${nodePath}.state`).toBe(expected.state); - } - // A resolved node must be mounted in the DOM, and rendered text must be - // visible there, not only present on the in-memory node. - if (actual.state === 'resolved') { - const domEl = findDomElementForNode(container, actual); - expect(domEl, `Expected a mounted DOM element for ${nodePath}`).toBeDefined(); - for (const key of ['text', 'badgeText'] as const) { - const expectedText = expected[key]; - if (typeof expectedText === 'string') { - expect(domEl!.textContent, `${nodePath}.${key} rendered text`).toContain(expectedText); - } - } - } - assertViewNodeProps(actual, expected, nodePath); - if (expected.children !== undefined) { - const childNodes = extractDirectChildNodes(actual); - expect(childNodes, `${nodePath}.children length`).toHaveLength(expected.children.length); - expected.children.forEach((expectedChild, idx) => { - assertViewTree( - childNodes[idx], - expectedChild, - surface, - container, - `${nodePath}/children/${idx}`, - ); - }); - } -} - -function resolveAccessibleLabel(el: HTMLElement): string | undefined { - const explicit = el.getAttribute('aria-label'); - if (explicit) { - return explicit; - } - const buttonText = el.querySelector('button')?.textContent?.trim(); - if (buttonText) { - return buttonText; - } - const labelText = el.querySelector('label')?.textContent?.trim(); - if (labelText) { - return labelText; - } - return el.textContent?.trim() || undefined; -} - -function resolveSemanticRole(el: HTMLElement): string | undefined { - const explicitRole = el.getAttribute('role'); - if (explicitRole) { - return explicitRole; - } - if (el.querySelector('button')) { - return 'button'; - } - if (el.querySelector('input[type="checkbox"]')) { - return 'checkbox'; - } - if (el.querySelector('input[type="radio"]')) { - return 'radiogroup'; - } - const textInput = el.querySelector( - 'input:not([type="checkbox"]):not([type="radio"]):not([type="range"]), textarea', - ); - return textInput ? 'textbox' : undefined; -} - -function resolveCheckedState(el: HTMLElement): boolean | undefined { - const ariaChecked = el.getAttribute('aria-checked'); - if (ariaChecked === 'true' || ariaChecked === 'false') { - return ariaChecked === 'true'; - } - const checkbox = el.querySelector('input[type="checkbox"]') as HTMLInputElement | null; - return checkbox ? checkbox.checked : undefined; -} - -function assertNodeAccessibility( - container: HTMLElement, - componentId: string, - expected: ExpectedAccessibilityNode, -): void { - const el = findComponentElement(container, componentId); - expect(el, `Expected rendered DOM element for "${componentId}"`).toBeDefined(); - const target = el!; - if (expected.label !== undefined) { - expect(resolveAccessibleLabel(target)).toBe(expected.label); - } - if (expected.description !== undefined) { - expect(target.getAttribute('aria-description')).toBe(expected.description); - } - if (expected.live !== undefined) { - expect(target.getAttribute('aria-live')).toBe(expected.live); - } - if (expected.hidden !== undefined) { - expect(target.getAttribute('aria-hidden')).toBe(String(expected.hidden)); - } - if (expected.role !== undefined) { - expect(resolveSemanticRole(target)).toBe(expected.role); - } - if (expected.checked !== undefined) { - expect(resolveCheckedState(target)).toBe(expected.checked); - } -} - -function assertAxeRules(container: HTMLElement, rules: string[]): void { - if (rules.includes('button-name')) { - const buttons = Array.from(container.querySelectorAll('button')); - for (const btn of buttons) { - const host = btn.closest('a2ui-basic-button') as HTMLElement | null; - const name = - btn.getAttribute('aria-label') || - host?.getAttribute('aria-label') || - btn.textContent?.trim(); - expect(Boolean(name)).toBe(true); - } - } - if (rules.includes('aria-valid-attr-value')) { - const liveNodes = Array.from(container.querySelectorAll('[aria-live]')); - for (const node of liveNodes) { - const val = node.getAttribute('aria-live'); - expect(['off', 'polite', 'assertive']).toContain(val); - } - } -} - -function countActiveSubscriptions(surface: SurfaceModel): number { - const createdListeners = - (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? - 0; - const deletedListeners = - (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? - 0; - let componentUpdatedListeners = 0; - for (const comp of surface.componentsModel.values) { - componentUpdatedListeners += - (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; - } - const dataModelInternal = surface.dataModel as unknown as { - subscriptions?: Set; - signals?: Map; - }; - let activeSignalTargets = 0; - for (const sig of dataModelInternal.signals?.values() ?? []) { - if (sig._targets !== null && sig._targets !== undefined) { - activeSignalTargets += 1; - } - } - const directDataSubs = dataModelInternal.subscriptions?.size ?? 0; - return ( - createdListeners + - deletedListeners + - componentUpdatedListeners + - activeSignalTargets + - directDataSubs - ); -} - -async function performStepOperation(state: HarnessState, step: AdapterStep): Promise { - if (!step.op || step.op === 'process_messages') { - return; - } - if (step.op === 'unmount_surface') { - state.rendered?.unmount(); - state.unmounted = true; - return; - } - const container = state.rendered!.container; - const targetId = step.targetNode!; - if (step.op === 'user_input') { - const compEl = findComponentElement(container, targetId); - const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; - expect(input, `Input element for "${targetId}"`).not.toBeNull(); - await act(async () => { - fireEvent.input(input!, {target: {value: String(step.inputValue ?? '')}}); - }); - return; - } - if (step.op === 'user_gesture') { - const compEl = findComponentElement(container, targetId); - const button = compEl?.querySelector('button'); - expect(button, `Button element for "${targetId}"`).not.toBeNull(); - await act(async () => { - fireEvent.click(button!); - }); - return; - } - if (step.op === 'programmatic_invoke') { - const compModel = state.surface!.componentsModel.get(targetId); - const compEl = findComponentElement(container, targetId) as A2uiWebComponentElement | undefined; - const dataContext = compEl?.context?.dataContext; - expect(dataContext, `DataContext for "${targetId}"`).toBeDefined(); - await act(async () => { - dataContext!.resolveAction(compModel!.properties.action as Record, { - userActivation: step.userActivation ?? false, - }); - }); - } -} - -function assertExpectedActions( - actualActions: Array>, - expectedActions: ExpectedActionEvent[] | undefined, -): void { - if (expectedActions === undefined) { - return; - } - expect(actualActions).toHaveLength(expectedActions.length); - expectedActions.forEach((expected, i) => { - const actual = actualActions[i]; - expect(actual?.name).toBe(expected.name); - if (expected.surfaceId !== undefined) { - expect(actual?.surfaceId).toBe(expected.surfaceId); - } - if (expected.sourceComponentId !== undefined) { - expect(actual?.sourceComponentId).toBe(expected.sourceComponentId); - } - if (expected.context !== undefined) { - expect(actual?.context).toEqual(expected.context); - } - }); -} - -function assertExpectedErrors( - actualErrors: Array>, - expectedErrors: ExpectedErrorEntry[] | undefined, -): void { - if (expectedErrors === undefined) { - return; - } - expect(actualErrors).toHaveLength(expectedErrors.length); - expectedErrors.forEach((expected, i) => { - const actual = actualErrors[i]; - if (expected.code !== undefined) { - expect(actual?.code).toBe(expected.code); - } - if (expected.surfaceId !== undefined) { - expect(actual?.surfaceId).toBe(expected.surfaceId); - } - if (expected.message !== undefined) { - expect(String(actual?.message ?? '')).toContain(expected.message); - } - }); -} - -function assertStepIdentityAndModel( - state: HarnessState, - step: AdapterStep, - currentNodes: Map, - currentDom: Map, -): void { - if (step.expectSameViewIdentity) { - for (const nodePath of step.expectSameViewIdentity) { - const prevNode = state.previousNodes.get(nodePath); - const currNode = currentNodes.get(nodePath); - expect(currNode, `Node at ${nodePath}`).toBeDefined(); - expect(currNode).toBe(prevNode); - const prevEl = state.previousDomElements.get(nodePath); - const currEl = currentDom.get(nodePath); - expect(prevEl, `Previous DOM element for ${nodePath}`).toBeDefined(); - expect(currEl, `Current DOM element for ${nodePath}`).toBeDefined(); - expect(currEl).toBe(prevEl); - } - } - if (step.expectDataModel !== undefined) { - expect(state.surface!.dataModel.get('/')).toEqual(step.expectDataModel); - } - if (step.expectActiveSubscriptions !== undefined) { - expect(countActiveSubscriptions(state.surface!)).toBe(step.expectActiveSubscriptions); - } -} - -function mapNodesToDomElements( - container: HTMLElement, - nodes: Map, -): Map { - const result = new Map(); - const allElements = Array.from(container.querySelectorAll('*')); - for (const [nodePath, node] of nodes) { - const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); - if (match) { - result.set(nodePath, match as HTMLElement); - } - } - return result; -} - -async function runAdapterStep(state: HarnessState, step: AdapterStep): Promise { - state.actions = []; - state.errors = []; - if (step.messages && step.messages.length > 0) { - await act(async () => { - state.processor.processMessages(step.messages!); - }); - } - if (!state.rendered && state.surface) { - state.rendered = render(); - } - if (state.rendered && !state.unmounted) { - await settleSurfaceDom(state.rendered.container); - } - await performStepOperation(state, step); - if (state.rendered && !state.unmounted) { - await settleSurfaceDom(state.rendered.container); - } - const container = state.rendered!.container; - const rootNode = state.unmounted ? undefined : getRootComponentNode(container); - const currentNodes = collectMountedNodes(rootNode); - const currentDom = state.unmounted - ? new Map() - : mapNodesToDomElements(container, currentNodes); +}); - if (step.expectViewTree) { - assertViewTree(rootNode, step.expectViewTree, state.surface!, container); - } - if (step.expectAccessibilityTree) { - for (const [compId, expectedA11y] of Object.entries(step.expectAccessibilityTree)) { - assertNodeAccessibility(container, compId, expectedA11y); - } - } - assertExpectedActions(state.actions, step.expectActions); - assertExpectedErrors(state.errors, step.expectErrors); - assertStepIdentityAndModel(state, step, currentNodes, currentDom); - state.previousNodes = currentNodes; - state.previousDomElements = currentDom; -} +afterAll(() => { + assertAllUnsupportedMatched(); +}); describe('@a2ui/react Accessibility Conformance (conformance/core/accessibility.yaml)', () => { const accessibilityCases = loadYamlCases('accessibility.yaml'); for (const testCase of accessibilityCases) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - const processor = new MessageProcessor([ - createDefaultAdapterCatalog(), - ]); - const steps: AccessibilityStep[] = testCase.steps ?? [ - { - messages: testCase.messages ?? [], - assertions: testCase.assertions ?? {}, - }, - ]; + const testTitle = `${testCase.name}: ${testCase.description ?? ''}`; + it(testTitle, async ctx => { + const record: ManifestRecord = { + harness: 'react', + suite: 'core/accessibility.yaml', + case: testCase.name, + status: 'failed', + }; + pendingManifestByTestName.set(testTitle, record); let rendered: RenderResult | undefined; - - for (const step of steps) { - await act(async () => { - processor.processMessages(step.messages); - }); - if (!rendered) { - const surface = processor.model.getSurface('a11y_surface'); - expect(surface).toBeDefined(); - rendered = render(); - } - await settleSurfaceDom(rendered.container); - for (const [compId, expected] of Object.entries(step.assertions.accessibilityTree ?? {})) { - assertNodeAccessibility(rendered.container, compId, expected); + try { + assertKnownAccessibilityCase(testCase); + const steps: AccessibilityStep[] = testCase.steps ?? [ + { + messages: testCase.messages ?? [], + assertions: testCase.assertions ?? {}, + }, + ]; + const skipReason = findUnsupportedReason( + testCase.name, + steps as unknown as Array>, + ); + if (skipReason) { + record.status = 'skipped'; + record.reason = skipReason; + ctx.skip(); + return; } - if (step.assertions.axeCore) { - assertAxeRules(rendered.container, step.assertions.axeCore); + + const processor = new MessageProcessor([ + createDefaultAdapterCatalog(), + ]); + + for (const step of steps) { + await act(async () => { + processor.processMessages(step.messages); + }); + if (!rendered) { + const surface = processor.model.getSurface('a11y_surface'); + expect(surface).toBeDefined(); + rendered = render(); + } + await settleSurfaceDom(rendered.container); + for (const [compId, expected] of Object.entries( + step.assertions.accessibilityTree ?? {}, + )) { + assertNodeAccessibility(rendered.container, compId, expected); + } + if (step.assertions.axeCore) { + assertAxeRules(rendered.container, step.assertions.axeCore); + } } + record.status = 'passed'; + } catch (err) { + record.status = 'failed'; + record.reason = err instanceof Error ? err.message : String(err); + throw err; + } finally { + rendered?.unmount(); } - rendered?.unmount(); }); } }); @@ -739,35 +146,41 @@ describe('@a2ui/react Adapter Conformance (conformance/core/adapter_conformance. }); for (const testCase of adapterCases) { - it(`${testCase.name}: ${testCase.description ?? ''}`, async () => { - const catalogs: Array> = [ - createDefaultAdapterCatalog(), - ...(testCase.catalogs ?? []).map(buildExtraCatalog), - ]; - const processor = new MessageProcessor(catalogs); - const state: HarnessState = { - processor, - unmounted: false, - actions: [], - errors: [], - previousNodes: new Map(), - previousDomElements: new Map(), + const testTitle = `${testCase.name}: ${testCase.description ?? ''}`; + it(testTitle, async ctx => { + const record: ManifestRecord = { + harness: 'react', + suite: 'core/adapter_conformance.yaml', + case: testCase.name, + status: 'failed', }; - processor.model.onSurfaceCreated.subscribe(surface => { - state.surface = surface; - surface.onAction.subscribe(action => { - state.actions.push(action as unknown as Record); - }); - surface.onError.subscribe(err => { - state.errors.push(err as unknown as Record); - }); - }); + pendingManifestByTestName.set(testTitle, record); + const state = mountSurface(testCase); + try { + assertKnownAdapterCase(testCase); + const skipReason = findUnsupportedReason( + testCase.name, + testCase.steps as unknown as Array>, + ); + if (skipReason) { + record.status = 'skipped'; + record.reason = skipReason; + ctx.skip(); + return; + } - for (const step of testCase.steps) { - await runAdapterStep(state, step); - } - if (!state.unmounted) { - state.rendered?.unmount(); + for (const step of testCase.steps) { + await runStep(state, step); + } + record.status = 'passed'; + } catch (err) { + record.status = 'failed'; + record.reason = err instanceof Error ? err.message : String(err); + throw err; + } finally { + if (!state.unmounted) { + state.rendered?.unmount(); + } } }); } diff --git a/typescript/renderers/react/tests/adapter_harness.tsx b/typescript/renderers/react/tests/adapter_harness.tsx new file mode 100644 index 0000000000..208cd26b32 --- /dev/null +++ b/typescript/renderers/react/tests/adapter_harness.tsx @@ -0,0 +1,980 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import {act, fireEvent, render, type RenderResult} from '@testing-library/react'; +import React from 'react'; +import {expect} from 'vitest'; +import {z} from 'zod'; + +import { + Catalog, + type ComponentNode, + createFunctionImplementation, + isComponentNode, + MessageProcessor, + peekValue, + type ProcessableMessage, + ResolvedBinding, + type SurfaceModel, +} from '@a2ui/web_core'; +import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; +import {CommonSchemas} from '@a2ui/web_core/v0_9'; +import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; +import {A2uiSurface, createComponentImplementation, type ReactCatalogComponent} from '../src'; + +export interface ExpectedAccessibilityNode { + label?: string; + description?: string; + live?: string; + hidden?: boolean; + role?: string; + checked?: boolean; +} + +export interface AccessibilityAssertions { + accessibilityTree?: Record; + axeCore?: string[]; +} + +export interface AccessibilityStep { + description?: string; + messages: ProcessableMessage[]; + assertions: AccessibilityAssertions; +} + +export interface AccessibilityTestCase { + name: string; + description?: string; + features?: string[]; + protocolVersion: string; + action: string; + catalogPath?: string; + surface?: Record; + messages?: ProcessableMessage[]; + assertions?: AccessibilityAssertions; + steps?: AccessibilityStep[]; +} + +export interface ExpectedViewNode { + id?: string; + type?: string; + state?: string; + isValid?: boolean; + validationErrors?: string[]; + validationResults?: Array>; + children?: ExpectedViewNode[]; + [key: string]: unknown; +} + +export interface ExpectedActionEvent { + name: string; + surfaceId?: string; + sourceComponentId?: string; + context?: Record; +} + +export interface ExpectedErrorEntry { + code?: string; + surfaceId?: string; + message?: string; +} + +export interface ExtraCatalogEntry { + catalogId: string; + protocolVersion?: string; + components: Record< + string, + { + type: string; + required?: string[]; + properties?: Record; + } + >; +} + +export interface AdapterStep { + description?: string; + op?: + | 'process_messages' + | 'user_input' + | 'user_gesture' + | 'programmatic_invoke' + | 'unmount_surface'; + messages?: ProcessableMessage[]; + targetNode?: string; + inputValue?: unknown; + userActivation?: boolean; + expectViewTree?: ExpectedViewNode; + expectAccessibilityTree?: Record; + expectDataModel?: Record; + expectActions?: ExpectedActionEvent[]; + expectErrors?: ExpectedErrorEntry[]; + expectSameViewIdentity?: string[]; + expectActiveSubscriptions?: number; +} + +export interface AdapterTestCase { + name: string; + description?: string; + features?: string[]; + protocolVersion: string; + action: string; + catalogPaths?: string[]; + catalogs?: ExtraCatalogEntry[]; + steps: AdapterStep[]; +} + +export interface HarnessState { + processor: MessageProcessor; + surface?: SurfaceModel; + rendered?: RenderResult; + unmounted: boolean; + actions: Array>; + errors: Array>; + previousNodes: Map; + previousDomElements: Map; +} + +export interface ManifestRecord { + harness: string; + suite: string; + case: string; + status: 'passed' | 'failed' | 'skipped' | 'xfailed'; + reason?: string; +} + +/** + * Reasoned skip entries keyed by `""` or `"/"`. + * Every entry in this map must match at least one case during the run. + */ +export const UNSUPPORTED: Record = {}; + +const matchedUnsupportedKeys = new Set(); + +export const KNOWN_KEYS = { + adapterCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPaths', + 'catalogs', + 'steps', + ]), + adapterCatalog: new Set(['catalogId', 'protocolVersion', 'components']), + adapterStep: new Set([ + 'description', + 'op', + 'messages', + 'targetNode', + 'inputValue', + 'userActivation', + 'expectViewTree', + 'expectAccessibilityTree', + 'expectDataModel', + 'expectActions', + 'expectErrors', + 'expectSameViewIdentity', + 'expectActiveSubscriptions', + ]), + adapterOps: new Set([ + 'process_messages', + 'user_input', + 'user_gesture', + 'programmatic_invoke', + 'unmount_surface', + ]), + viewNode: new Set([ + 'id', + 'type', + 'state', + 'text', + 'value', + 'badgeText', + 'isValid', + 'validationErrors', + 'validationResults', + 'children', + ]), + validationResult: new Set(['valid', 'code', 'message', 'severity']), + expectedAction: new Set(['name', 'surfaceId', 'sourceComponentId', 'context']), + expectedError: new Set(['code', 'surfaceId', 'message']), + accessibilityCase: new Set([ + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPath', + 'surface', + 'messages', + 'assertions', + 'steps', + ]), + accessibilityStep: new Set(['description', 'messages', 'assertions']), + accessibilityAssertions: new Set(['accessibilityTree', 'axeCore']), + accessibilityNode: new Set(['label', 'description', 'live', 'hidden', 'role', 'checked']), +} as const; + +function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: string): void { + if (obj === undefined || obj === null) { + return; + } + expect(typeof obj === 'object' && !Array.isArray(obj), `${where}: expected object`).toBe(true); + const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); + expect( + unknown, + `${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`, + ).toEqual([]); +} + +function assertKnownViewNodeKeys(node: unknown, where: string): void { + if (node === undefined || node === null) { + return; + } + assertObjectKeys(node, KNOWN_KEYS.viewNode, where); + const record = node as Record; + if (Array.isArray(record.validationResults)) { + record.validationResults.forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.validationResult, `${where}.validationResults[${idx}]`); + }); + } + if (Array.isArray(record.children)) { + record.children.forEach((child, idx) => { + assertKnownViewNodeKeys(child, `${where}.children[${idx}]`); + }); + } +} + +export function assertKnownKeys( + obj: unknown, + allowed: ReadonlySet = KNOWN_KEYS.adapterStep, + where = 'step', +): void { + assertObjectKeys(obj, allowed, where); + if (allowed !== KNOWN_KEYS.adapterStep || !obj || typeof obj !== 'object') { + return; + } + const step = obj as Record; + if (step.op !== undefined) { + expect( + typeof step.op === 'string' && KNOWN_KEYS.adapterOps.has(step.op), + `${where}.op: unsupported op "${String(step.op)}"`, + ).toBe(true); + } + if (step.expectViewTree !== undefined) { + assertKnownViewNodeKeys(step.expectViewTree, `${where}.expectViewTree`); + } + if (step.expectAccessibilityTree !== undefined) { + expect( + typeof step.expectAccessibilityTree === 'object' && + step.expectAccessibilityTree !== null && + !Array.isArray(step.expectAccessibilityTree), + `${where}.expectAccessibilityTree: expected object`, + ).toBe(true); + for (const [compId, node] of Object.entries( + step.expectAccessibilityTree as Record, + )) { + assertObjectKeys( + node, + KNOWN_KEYS.accessibilityNode, + `${where}.expectAccessibilityTree.${compId}`, + ); + } + } + if (step.expectActions !== undefined) { + expect(Array.isArray(step.expectActions), `${where}.expectActions: expected array`).toBe(true); + (step.expectActions as unknown[]).forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.expectedAction, `${where}.expectActions[${idx}]`); + }); + } + if (step.expectErrors !== undefined) { + expect(Array.isArray(step.expectErrors), `${where}.expectErrors: expected array`).toBe(true); + (step.expectErrors as unknown[]).forEach((item, idx) => { + assertObjectKeys(item, KNOWN_KEYS.expectedError, `${where}.expectErrors[${idx}]`); + }); + } +} + +export function assertKnownAdapterCase(testCase: AdapterTestCase): void { + assertObjectKeys(testCase, KNOWN_KEYS.adapterCase, `case(${testCase.name})`); + expect(testCase.action, `case(${testCase.name}): unsupported action "${testCase.action}"`).toBe( + 'render_surface', + ); + for (const [idx, cat] of (testCase.catalogs ?? []).entries()) { + assertObjectKeys(cat, KNOWN_KEYS.adapterCatalog, `case(${testCase.name}).catalogs[${idx}]`); + } + for (const [idx, step] of testCase.steps.entries()) { + assertKnownKeys(step, KNOWN_KEYS.adapterStep, `case(${testCase.name}).steps[${idx}]`); + } +} + +export function assertKnownAccessibilityAssertions( + assertions: AccessibilityAssertions | undefined, + where: string, +): void { + if (!assertions) { + return; + } + assertObjectKeys(assertions, KNOWN_KEYS.accessibilityAssertions, where); + for (const [compId, node] of Object.entries(assertions.accessibilityTree ?? {})) { + assertObjectKeys(node, KNOWN_KEYS.accessibilityNode, `${where}.accessibilityTree.${compId}`); + } +} + +export function assertKnownAccessibilityCase(testCase: AccessibilityTestCase): void { + assertObjectKeys(testCase, KNOWN_KEYS.accessibilityCase, `case(${testCase.name})`); + expect(testCase.action, `case(${testCase.name}): unsupported action "${testCase.action}"`).toBe( + 'accessibility_check', + ); + assertKnownAccessibilityAssertions(testCase.assertions, `case(${testCase.name}).assertions`); + for (const [idx, step] of (testCase.steps ?? []).entries()) { + assertObjectKeys(step, KNOWN_KEYS.accessibilityStep, `case(${testCase.name}).steps[${idx}]`); + assertKnownAccessibilityAssertions( + step.assertions, + `case(${testCase.name}).steps[${idx}].assertions`, + ); + } +} + +export function findUnsupportedReason( + caseName: string, + steps: Array> = [], +): string | undefined { + if (Object.hasOwn(UNSUPPORTED, caseName)) { + matchedUnsupportedKeys.add(caseName); + return UNSUPPORTED[caseName]; + } + for (const step of steps) { + const op = typeof step.op === 'string' ? step.op : undefined; + if (op) { + const opKey = `${caseName}/${op}`; + if (Object.hasOwn(UNSUPPORTED, opKey)) { + matchedUnsupportedKeys.add(opKey); + return UNSUPPORTED[opKey]; + } + } + for (const key of Object.keys(step)) { + const stepKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, stepKey)) { + matchedUnsupportedKeys.add(stepKey); + return UNSUPPORTED[stepKey]; + } + } + } + return undefined; +} + +export function assertAllUnsupportedMatched(): void { + const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); + expect( + unused, + `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, + ).toEqual([]); +} + +export function writeManifestRecord(rec: ManifestRecord): void { + const manifestDir = process.env.A2UI_CONFORMANCE_MANIFEST_DIR; + if (!manifestDir) { + return; + } + fs.mkdirSync(manifestDir, {recursive: true}); + const filePath = path.join(manifestDir, `${rec.harness}-${process.pid}.jsonl`); + const payload: Record = { + harness: rec.harness, + suite: rec.suite.replaceAll('\\', '/'), + case: rec.case, + status: rec.status, + }; + if (rec.reason) { + payload.reason = rec.reason; + } + fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, 'utf8'); +} + +export function createDefaultAdapterCatalog(): Catalog { + const incrementCounter = createFunctionImplementation( + { + name: 'incrementCounter', + returnType: 'void', + schema: z.object({ + path: z.string().optional(), + step: z.number().optional(), + amount: z.number().optional(), + }), + }, + (args, context) => { + const targetPath = args.path ?? '/counter'; + const delta = args.amount ?? args.step ?? 1; + const current = Number(context.dataModel.get(targetPath) ?? 0); + context.dataModel.set(targetPath, current + delta); + }, + ); + return new Catalog( + basicCatalog.id, + basicCatalog.protocolVersion, + Array.from(basicCatalog.components.values()), + [...Array.from(basicCatalog.functions.values()), incrementCounter], + basicCatalog.themeSchema, + basicCatalog.instructions, + ); +} + +export function buildExtraCatalog(entry: ExtraCatalogEntry): Catalog { + const implementations = Object.entries(entry.components).map(([name, def]) => { + const shape: Record = {}; + for (const [propKey, propSchema] of Object.entries(def.properties ?? {})) { + if (propKey === 'component') { + continue; + } + shape[propKey] = + propSchema.type === 'number' + ? CommonSchemas.DynamicNumber.optional() + : CommonSchemas.DynamicString.optional(); + } + return createComponentImplementation({name, schema: z.object(shape)}, ({props}) => ( + + {Object.entries(props) + .filter( + ([k, v]) => !k.startsWith('set') && (typeof v === 'string' || typeof v === 'number'), + ) + .map(([, v]) => String(v)) + .join(' ')} + + )); + }); + return new Catalog( + entry.catalogId, + entry.protocolVersion ?? '1.0', + implementations, + ); +} + +export async function settleSurfaceDom(container: HTMLElement): Promise { + for (let pass = 0; pass < 3; pass++) { + await act(async () => { + await new Promise(resolve => setTimeout(resolve, 0)); + const elements = Array.from(container.querySelectorAll('*')); + for (const el of elements) { + const litEl = el as unknown as {updateComplete?: Promise}; + if (litEl.updateComplete) { + await litEl.updateComplete; + } + } + }); + } +} + +function findComponentElement( + container: HTMLElement, + componentId: string, +): HTMLElement | undefined { + const candidates = Array.from(container.querySelectorAll('*')); + for (const candidate of candidates) { + const a2uiEl = candidate as unknown as A2uiWebComponentElement; + if ( + a2uiEl.context?.componentModel.id === componentId || + a2uiEl.node?.componentId === componentId + ) { + return candidate as HTMLElement; + } + } + return undefined; +} + +function getRootComponentNode(container: HTMLElement): ComponentNode | undefined { + const first = container.firstElementChild as A2uiWebComponentElement | null; + return first?.node; +} + +function extractDirectChildNodes(node: ComponentNode): ComponentNode[] { + const props = peekValue(node.props); + if (Array.isArray(props.children)) { + return props.children.filter(isComponentNode); + } + if (isComponentNode(props.child)) { + return [props.child]; + } + return []; +} + +function collectMountedNodes(root: ComponentNode | undefined): Map { + const nodes = new Map(); + if (!root) { + return nodes; + } + const visit = (node: ComponentNode, nodePath: string): void => { + nodes.set(nodePath, node); + const props = peekValue(node.props); + if (Array.isArray(props.children)) { + props.children.forEach((child, i) => { + if (isComponentNode(child)) { + visit(child, `${nodePath}/children/${i}`); + } + }); + } else if (isComponentNode(props.child)) { + visit(props.child, `${nodePath}/child`); + } + }; + visit(root, 'root'); + return nodes; +} + +function unwrapBindingValue(value: unknown): unknown { + if (value instanceof ResolvedBinding) { + return unwrapBindingValue(value.value); + } + if (Array.isArray(value)) { + return value.map(unwrapBindingValue); + } + return value; +} + +function resolveNodeType( + node: ComponentNode, + surface?: SurfaceModel, +): string { + if (node.state === 'cyclic' && node.type === 'Placeholder' && surface) { + return surface.componentsModel.get(node.componentId)?.type ?? node.type; + } + return node.type; +} + +const RESERVED_VIEW_KEYS = new Set([ + 'id', + 'type', + 'state', + 'isValid', + 'validationErrors', + 'validationResults', + 'children', +]); + +function assertViewNodeProps( + node: ComponentNode, + expected: ExpectedViewNode, + nodePath: string, +): void { + const props = peekValue(node.props); + if (expected.isValid !== undefined) { + expect(unwrapBindingValue(props.isValid), `${nodePath}.isValid`).toBe(expected.isValid); + } + if (expected.validationErrors !== undefined) { + expect(unwrapBindingValue(props.validationErrors), `${nodePath}.validationErrors`).toEqual( + expected.validationErrors, + ); + } + if (expected.validationResults !== undefined) { + expect(unwrapBindingValue(props.validationResults), `${nodePath}.validationResults`).toEqual( + expected.validationResults, + ); + } + for (const [key, expectedVal] of Object.entries(expected)) { + if (!RESERVED_VIEW_KEYS.has(key)) { + expect(unwrapBindingValue(props[key]), `${nodePath}.${key}`).toEqual(expectedVal); + } + } +} + +function findDomElementForNode( + container: HTMLElement, + node: ComponentNode, +): HTMLElement | undefined { + const match = Array.from(container.querySelectorAll('*')).find( + el => (el as A2uiWebComponentElement).node === node, + ); + return match as HTMLElement | undefined; +} + +export function extractViewTree(state: HarnessState): ComponentNode | undefined { + if (!state.rendered || state.unmounted) { + return undefined; + } + return getRootComponentNode(state.rendered.container); +} + +export function assertViewTree( + node: ComponentNode | undefined, + expected: ExpectedViewNode, + surface?: SurfaceModel, + container?: HTMLElement, + nodePath = 'root', +): void { + expect(node, `Missing node at ${nodePath}`).toBeDefined(); + const actual = node!; + if (expected.id !== undefined) { + expect(actual.componentId, `${nodePath}.id`).toBe(expected.id); + } + if (expected.type !== undefined) { + expect(resolveNodeType(actual, surface), `${nodePath}.type`).toBe(expected.type); + } + if (expected.state !== undefined) { + expect(actual.state, `${nodePath}.state`).toBe(expected.state); + } + if (actual.state === 'resolved' && container) { + const domEl = findDomElementForNode(container, actual); + expect(domEl, `Expected a mounted DOM element for ${nodePath}`).toBeDefined(); + for (const key of ['text', 'badgeText'] as const) { + const expectedText = expected[key]; + if (typeof expectedText === 'string') { + expect(domEl!.textContent, `${nodePath}.${key} rendered text`).toContain(expectedText); + } + } + } + assertViewNodeProps(actual, expected, nodePath); + if (expected.children !== undefined) { + const childNodes = extractDirectChildNodes(actual); + expect(childNodes, `${nodePath}.children length`).toHaveLength(expected.children.length); + expected.children.forEach((expectedChild, idx) => { + assertViewTree( + childNodes[idx], + expectedChild, + surface, + container, + `${nodePath}/children/${idx}`, + ); + }); + } +} + +function resolveAccessibleLabel(el: HTMLElement): string | undefined { + const explicit = el.getAttribute('aria-label'); + if (explicit) { + return explicit; + } + const buttonText = el.querySelector('button')?.textContent?.trim(); + if (buttonText) { + return buttonText; + } + const labelText = el.querySelector('label')?.textContent?.trim(); + if (labelText) { + return labelText; + } + return el.textContent?.trim() || undefined; +} + +function resolveSemanticRole(el: HTMLElement): string | undefined { + const explicitRole = el.getAttribute('role'); + if (explicitRole) { + return explicitRole; + } + if (el.querySelector('button')) { + return 'button'; + } + if (el.querySelector('input[type="checkbox"]')) { + return 'checkbox'; + } + if (el.querySelector('input[type="radio"]')) { + return 'radiogroup'; + } + const textInput = el.querySelector( + 'input:not([type="checkbox"]):not([type="radio"]):not([type="range"]), textarea', + ); + return textInput ? 'textbox' : undefined; +} + +function resolveCheckedState(el: HTMLElement): boolean | undefined { + const ariaChecked = el.getAttribute('aria-checked'); + if (ariaChecked === 'true' || ariaChecked === 'false') { + return ariaChecked === 'true'; + } + const checkbox = el.querySelector('input[type="checkbox"]') as HTMLInputElement | null; + return checkbox ? checkbox.checked : undefined; +} + +export function assertNodeAccessibility( + container: HTMLElement, + componentId: string, + expected: ExpectedAccessibilityNode, +): void { + const el = findComponentElement(container, componentId); + expect(el, `Expected rendered DOM element for "${componentId}"`).toBeDefined(); + const target = el!; + if (expected.label !== undefined) { + expect(resolveAccessibleLabel(target)).toBe(expected.label); + } + if (expected.description !== undefined) { + expect(target.getAttribute('aria-description')).toBe(expected.description); + } + if (expected.live !== undefined) { + expect(target.getAttribute('aria-live')).toBe(expected.live); + } + if (expected.hidden !== undefined) { + expect(target.getAttribute('aria-hidden')).toBe(String(expected.hidden)); + } + if (expected.role !== undefined) { + expect(resolveSemanticRole(target)).toBe(expected.role); + } + if (expected.checked !== undefined) { + expect(resolveCheckedState(target)).toBe(expected.checked); + } +} + +export function assertAxeRules(container: HTMLElement, rules: string[]): void { + if (rules.includes('button-name')) { + const buttons = Array.from(container.querySelectorAll('button')); + for (const btn of buttons) { + const host = btn.closest('a2ui-basic-button') as HTMLElement | null; + const name = + btn.getAttribute('aria-label') || + host?.getAttribute('aria-label') || + btn.textContent?.trim(); + expect(Boolean(name)).toBe(true); + } + } + if (rules.includes('aria-valid-attr-value')) { + const liveNodes = Array.from(container.querySelectorAll('[aria-live]')); + for (const node of liveNodes) { + const val = node.getAttribute('aria-live'); + expect(['off', 'polite', 'assertive']).toContain(val); + } + } +} + +function countActiveSubscriptions(surface: SurfaceModel): number { + const createdListeners = + (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? + 0; + const deletedListeners = + (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? + 0; + let componentUpdatedListeners = 0; + for (const comp of surface.componentsModel.values) { + componentUpdatedListeners += + (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; + } + const dataModelInternal = surface.dataModel as unknown as { + subscriptions?: Set; + signals?: Map; + }; + let activeSignalTargets = 0; + for (const sig of dataModelInternal.signals?.values() ?? []) { + if (sig._targets !== null && sig._targets !== undefined) { + activeSignalTargets += 1; + } + } + const directDataSubs = dataModelInternal.subscriptions?.size ?? 0; + return ( + createdListeners + + deletedListeners + + componentUpdatedListeners + + activeSignalTargets + + directDataSubs + ); +} + +async function performStepOperation(state: HarnessState, step: AdapterStep): Promise { + if (!step.op || step.op === 'process_messages') { + return; + } + if (step.op === 'unmount_surface') { + state.rendered?.unmount(); + state.unmounted = true; + return; + } + const container = state.rendered!.container; + const targetId = step.targetNode!; + if (step.op === 'user_input') { + const compEl = findComponentElement(container, targetId); + const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; + expect(input, `Input element for "${targetId}"`).not.toBeNull(); + await act(async () => { + fireEvent.input(input!, {target: {value: String(step.inputValue ?? '')}}); + }); + return; + } + if (step.op === 'user_gesture') { + const compEl = findComponentElement(container, targetId); + const button = compEl?.querySelector('button'); + expect(button, `Button element for "${targetId}"`).not.toBeNull(); + await act(async () => { + fireEvent.click(button!); + }); + return; + } + if (step.op === 'programmatic_invoke') { + const compModel = state.surface!.componentsModel.get(targetId); + const compEl = findComponentElement(container, targetId) as A2uiWebComponentElement | undefined; + const dataContext = compEl?.context?.dataContext; + expect(dataContext, `DataContext for "${targetId}"`).toBeDefined(); + await act(async () => { + dataContext!.resolveAction(compModel!.properties.action as Record, { + userActivation: step.userActivation ?? false, + }); + }); + return; + } + throw new Error(`Unsupported adapter step op: ${String(step.op)}`); +} + +function assertExpectedActions( + actualActions: Array>, + expectedActions: ExpectedActionEvent[] | undefined, +): void { + if (expectedActions === undefined) { + return; + } + expect(actualActions).toHaveLength(expectedActions.length); + expectedActions.forEach((expected, i) => { + const actual = actualActions[i]; + expect(actual?.name).toBe(expected.name); + if (expected.surfaceId !== undefined) { + expect(actual?.surfaceId).toBe(expected.surfaceId); + } + if (expected.sourceComponentId !== undefined) { + expect(actual?.sourceComponentId).toBe(expected.sourceComponentId); + } + if (expected.context !== undefined) { + expect(actual?.context).toEqual(expected.context); + } + }); +} + +function assertExpectedErrors( + actualErrors: Array>, + expectedErrors: ExpectedErrorEntry[] | undefined, +): void { + if (expectedErrors === undefined) { + return; + } + expect(actualErrors).toHaveLength(expectedErrors.length); + expectedErrors.forEach((expected, i) => { + const actual = actualErrors[i]; + if (expected.code !== undefined) { + expect(actual?.code).toBe(expected.code); + } + if (expected.surfaceId !== undefined) { + expect(actual?.surfaceId).toBe(expected.surfaceId); + } + if (expected.message !== undefined) { + expect(String(actual?.message ?? '')).toContain(expected.message); + } + }); +} + +function assertStepIdentityAndModel( + state: HarnessState, + step: AdapterStep, + currentNodes: Map, + currentDom: Map, +): void { + if (step.expectSameViewIdentity) { + for (const nodePath of step.expectSameViewIdentity) { + const prevNode = state.previousNodes.get(nodePath); + const currNode = currentNodes.get(nodePath); + expect(currNode, `Node at ${nodePath}`).toBeDefined(); + expect(currNode).toBe(prevNode); + const prevEl = state.previousDomElements.get(nodePath); + const currEl = currentDom.get(nodePath); + expect(prevEl, `Previous DOM element for ${nodePath}`).toBeDefined(); + expect(currEl, `Current DOM element for ${nodePath}`).toBeDefined(); + expect(currEl).toBe(prevEl); + } + } + if (step.expectDataModel !== undefined) { + expect(state.surface!.dataModel.get('/')).toEqual(step.expectDataModel); + } + if (step.expectActiveSubscriptions !== undefined) { + expect(countActiveSubscriptions(state.surface!)).toBe(step.expectActiveSubscriptions); + } +} + +function mapNodesToDomElements( + container: HTMLElement, + nodes: Map, +): Map { + const result = new Map(); + const allElements = Array.from(container.querySelectorAll('*')); + for (const [nodePath, node] of nodes) { + const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); + if (match) { + result.set(nodePath, match as HTMLElement); + } + } + return result; +} + +export function mountSurface(caseSpec?: Partial): HarnessState { + const catalogs: Array> = [ + createDefaultAdapterCatalog(), + ...(caseSpec?.catalogs ?? []).map(buildExtraCatalog), + ]; + const processor = new MessageProcessor(catalogs); + const state: HarnessState = { + processor, + unmounted: false, + actions: [], + errors: [], + previousNodes: new Map(), + previousDomElements: new Map(), + }; + processor.model.onSurfaceCreated.subscribe(surface => { + state.surface = surface; + surface.onAction.subscribe(action => { + state.actions.push(action as unknown as Record); + }); + surface.onError.subscribe(err => { + state.errors.push(err as unknown as Record); + }); + }); + return state; +} + +export async function runStep(state: HarnessState, step: AdapterStep): Promise { + assertKnownKeys(step); + state.actions = []; + state.errors = []; + if (step.messages && step.messages.length > 0) { + await act(async () => { + state.processor.processMessages(step.messages!); + }); + } + if (!state.rendered && state.surface) { + state.rendered = render(); + } + if (state.rendered && !state.unmounted) { + await settleSurfaceDom(state.rendered.container); + } + await performStepOperation(state, step); + if (state.rendered && !state.unmounted) { + await settleSurfaceDom(state.rendered.container); + } + const container = state.rendered!.container; + const rootNode = extractViewTree(state); + const currentNodes = collectMountedNodes(rootNode); + const currentDom = state.unmounted + ? new Map() + : mapNodesToDomElements(container, currentNodes); + + if (step.expectViewTree) { + assertViewTree(rootNode, step.expectViewTree, state.surface!, container); + } + if (step.expectAccessibilityTree) { + for (const [compId, expectedA11y] of Object.entries(step.expectAccessibilityTree)) { + assertNodeAccessibility(container, compId, expectedA11y); + } + } + assertExpectedActions(state.actions, step.expectActions); + assertExpectedErrors(state.errors, step.expectErrors); + assertStepIdentityAndModel(state, step, currentNodes, currentDom); + state.previousNodes = currentNodes; + state.previousDomElements = currentDom; +} From 4d6322a7be61ae637d4151c15adb528a5f892c38 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 07:05:03 +0000 Subject: [PATCH 24/48] test(conformance): address review findings in renderer adapter harnesses --- .../test/adapter_conformance_test.dart | 14 ++-- dart/a2ui_flutter/test/adapter_harness.dart | 9 +++ .../AdapterConformanceTests.swift | 36 +++++++++- .../A2UISwiftUITests/AdapterHarness.swift | 66 ++++++++++++++++++ typescript/renderers/angular/karma.conf.js | 48 ++----------- .../angular/src/adapter_conformance.spec.ts | 20 +++--- .../renderers/angular/src/adapter_harness.ts | 68 +++++++++++-------- .../lit/src/tests/adapter_harness.ts | 10 +++ .../react/tests/adapter_conformance.test.tsx | 21 +++--- .../renderers/react/tests/adapter_harness.tsx | 10 +++ 10 files changed, 205 insertions(+), 97 deletions(-) diff --git a/dart/a2ui_flutter/test/adapter_conformance_test.dart b/dart/a2ui_flutter/test/adapter_conformance_test.dart index 80b04c56a5..8b9f60b686 100644 --- a/dart/a2ui_flutter/test/adapter_conformance_test.dart +++ b/dart/a2ui_flutter/test/adapter_conformance_test.dart @@ -43,13 +43,17 @@ void main() { }); }); try { - final String? skipReason = findUnsupportedReason( - name, - (testCase['steps'] as List?) ?? const [], - ); + assertKnownAccessibilityCase(testCase); + final List steps = + (testCase['steps'] as List?) ?? + [ + {'assertions': testCase['assertions']}, + ]; + final String? skipReason = findUnsupportedReason(name, steps); if (skipReason != null) { status = 'skipped'; reason = skipReason; + markTestSkipped(skipReason); return; } await runAccessibilityCase(tester, testCase); @@ -80,6 +84,7 @@ void main() { }); }); try { + assertKnownAdapterCase(testCase); final String? skipReason = findUnsupportedReason( name, (testCase['steps'] as List?) ?? const [], @@ -87,6 +92,7 @@ void main() { if (skipReason != null) { status = 'skipped'; reason = skipReason; + markTestSkipped(skipReason); return; } await runAdapterCase(tester, testCase); diff --git a/dart/a2ui_flutter/test/adapter_harness.dart b/dart/a2ui_flutter/test/adapter_harness.dart index e06d67449c..5cd35a3b64 100644 --- a/dart/a2ui_flutter/test/adapter_harness.dart +++ b/dart/a2ui_flutter/test/adapter_harness.dart @@ -305,6 +305,15 @@ String? findUnsupportedReason( return unsupported[stepKey]; } } + if (step['assertions'] case final Map assertions) { + for (final String key in assertions.keys) { + final assertionKey = '$caseName/$key'; + if (unsupported.containsKey(assertionKey)) { + _matchedUnsupportedKeys.add(assertionKey); + return unsupported[assertionKey]; + } + } + } } return null; } diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift index b67c27c3b3..672593a545 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift @@ -68,11 +68,26 @@ struct AdapterConformanceTests { let name = testCase["name"] as? String ?? "" do { try AdapterHarness.assertKnownAccessibilityCase(testCase) + let steps = + (testCase["steps"] as? [[String: Any]]) + ?? [["assertions": testCase["assertions"] as Any]] + if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "skipped", + reason: skipReason + ) + ) + continue + } checkedCount += try runAccessibilityCase(testCase, name: name) writeManifestRecord( AdapterManifestRecord(suite: suite, caseName: name, status: "passed") ) } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) writeManifestRecord( AdapterManifestRecord( suite: suite, @@ -81,10 +96,10 @@ struct AdapterConformanceTests { reason: String(describing: error) ) ) - throw error } } #expect(checkedCount > 0, "accessibility.yaml should verify at least one component") + AdapterHarness.finishSuite(suite) } @Test func adapterConformanceSuite() throws { @@ -98,6 +113,7 @@ struct AdapterConformanceTests { do { try AdapterHarness.assertKnownAdapterCase(testCase) } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) writeManifestRecord( AdapterManifestRecord( suite: suite, @@ -106,7 +122,20 @@ struct AdapterConformanceTests { reason: String(describing: error) ) ) - throw error + continue + } + + let steps = (testCase["steps"] as? [[String: Any]]) ?? [] + if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "skipped", + reason: skipReason + ) + ) + continue } if let reason = Self.expectedAdapterFailures[name] { @@ -129,6 +158,7 @@ struct AdapterConformanceTests { AdapterManifestRecord(suite: suite, caseName: name, status: "passed") ) } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) writeManifestRecord( AdapterManifestRecord( suite: suite, @@ -137,7 +167,6 @@ struct AdapterConformanceTests { reason: String(describing: error) ) ) - throw error } } } @@ -149,6 +178,7 @@ struct AdapterConformanceTests { unusedExpectedFailures.isEmpty, "Unused expectedAdapterFailures entries: \(unusedExpectedFailures)" ) + AdapterHarness.finishSuite(suite) } // MARK: - Accessibility Suite Runner diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift index b0d3abd018..e61467223b 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift @@ -183,6 +183,16 @@ final class AdapterSession { enum AdapterHarness { private static let manifestLock = NSLock() + /// Reasoned skip entries keyed by `""` or `"/"`. + static let unsupported: [String: String] = [:] + + private static var matchedUnsupportedKeys: Set = [] + private static var completedSuites: Set = [] + private static let allAdapterSuites: Set = [ + "core/accessibility.yaml", + "core/adapter_conformance.yaml", + ] + static let adapterCaseKnownKeys: Set = [ "name", "description", @@ -452,6 +462,62 @@ enum AdapterHarness { } } + /// Returns a skip reason if `caseName` or any step in `steps` matches `unsupported`. + static func findUnsupportedReason( + _ caseName: String, + steps: [[String: Any]] = [] + ) -> String? { + if let reason = unsupported[caseName] { + matchedUnsupportedKeys.insert(caseName) + return reason + } + for step in steps { + if let op = step["op"] as? String { + let opKey = "\(caseName)/\(op)" + if let reason = unsupported[opKey] { + matchedUnsupportedKeys.insert(opKey) + return reason + } + } + for key in step.keys.sorted() { + let stepKey = "\(caseName)/\(key)" + if let reason = unsupported[stepKey] { + matchedUnsupportedKeys.insert(stepKey) + return reason + } + } + if let assertions = step["assertions"] as? [String: Any] { + for key in assertions.keys.sorted() { + let assertionKey = "\(caseName)/\(key)" + if let reason = unsupported[assertionKey] { + matchedUnsupportedKeys.insert(assertionKey) + return reason + } + } + } + } + return nil + } + + /// Verifies that every entry in `unsupported` matched at least one executed case. + static func assertAllUnsupportedMatched() { + let unused = unsupported.keys + .filter { !matchedUnsupportedKeys.contains($0) } + .sorted() + #expect( + unused.isEmpty, + "Unused unsupported entries did not match any case: \(unused)" + ) + } + + /// Marks `suite` complete and checks `assertAllUnsupportedMatched()` once all suites finish. + static func finishSuite(_ suite: String) { + completedSuites.insert(suite) + if completedSuites.isSuperset(of: allAdapterSuites) { + assertAllUnsupportedMatched() + } + } + /// Appends a manifest record to `$A2UI_CONFORMANCE_MANIFEST_DIR/-.jsonl`. nonisolated static func writeManifestRecord(_ rec: AdapterManifestRecord) { guard diff --git a/typescript/renderers/angular/karma.conf.js b/typescript/renderers/angular/karma.conf.js index fb3fc8fea2..2504637dc8 100644 --- a/typescript/renderers/angular/karma.conf.js +++ b/typescript/renderers/angular/karma.conf.js @@ -37,25 +37,19 @@ const basicCatalogJson = JSON.parse( function ConformanceManifestReporter(baseReporterDecorator) { baseReporterDecorator(this); const manifestDir = process.env.A2UI_CONFORMANCE_MANIFEST_DIR; - const manifestPath = manifestDir ? path.join(manifestDir, 'angular-karma.jsonl') : null; - const recordedKeys = new Set(); + const manifestPath = manifestDir ? path.join(manifestDir, `angular-${process.pid}.jsonl`) : null; function appendRecord(rec) { if (!manifestPath || !rec || !rec.suite || !rec.case) { return; } - const key = `${rec.suite}::${rec.case}`; - if (recordedKeys.has(key)) { - return; - } - recordedKeys.add(key); - fs.mkdirSync(manifestDir, {recursive: true}); + fs.mkdirSync(manifestDir, { recursive: true }); const entry = { harness: rec.harness || 'angular', suite: String(rec.suite).replace(/\\/g, '/'), case: rec.case, status: rec.status, - ...(rec.reason ? {reason: rec.reason} : {}), + ...(rec.reason ? { reason: rec.reason } : {}), }; fs.appendFileSync(manifestPath, `${JSON.stringify(entry)}\n`, 'utf8'); } @@ -65,38 +59,6 @@ function ConformanceManifestReporter(baseReporterDecorator) { appendRecord(info.conformanceManifestRecord); } }; - - this.onSpecComplete = function (_browser, result) { - if (!manifestPath || !result) { - return; - } - const suiteHeader = Array.isArray(result.suite) ? result.suite.join(' ') : ''; - let suite = null; - if (suiteHeader.includes('conformance/core/accessibility.yaml')) { - suite = 'core/accessibility.yaml'; - } else if (suiteHeader.includes('conformance/core/adapter_conformance.yaml')) { - suite = 'core/adapter_conformance.yaml'; - } - if (!suite) { - return; - } - const desc = String(result.description || ''); - const colonIdx = desc.indexOf(':'); - const caseName = (colonIdx >= 0 ? desc.slice(0, colonIdx) : desc).trim(); - const status = result.skipped ? 'skipped' : result.success ? 'passed' : 'failed'; - const reason = result.skipped - ? result.pendingReason || 'skipped' - : !result.success && Array.isArray(result.log) && result.log.length > 0 - ? String(result.log[0]).split('\n')[0] - : undefined; - appendRecord({ - harness: 'angular', - suite, - case: caseName, - status, - reason, - }); - }; } ConformanceManifestReporter.$inject = ['baseReporterDecorator']; @@ -109,7 +71,7 @@ module.exports = function (config) { require('karma-chrome-launcher'), require('karma-jasmine-html-reporter'), require('karma-coverage'), - {'reporter:conformance-manifest': ['type', ConformanceManifestReporter]}, + { 'reporter:conformance-manifest': ['type', ConformanceManifestReporter] }, ], client: { jasmine: { @@ -130,7 +92,7 @@ module.exports = function (config) { coverageReporter: { dir: require('path').join(__dirname, './coverage/lib'), subdir: '.', - reporters: [{type: 'html'}, {type: 'text-summary'}], + reporters: [{ type: 'html' }, { type: 'text-summary' }], }, reporters: ['progress', 'kjhtml', 'conformance-manifest'], browsers: ['Chrome'], diff --git a/typescript/renderers/angular/src/adapter_conformance.spec.ts b/typescript/renderers/angular/src/adapter_conformance.spec.ts index f40f8206f4..e4c4590738 100644 --- a/typescript/renderers/angular/src/adapter_conformance.spec.ts +++ b/typescript/renderers/angular/src/adapter_conformance.spec.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {ComponentFixture, TestBed} from '@angular/core/testing'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; import { assertAllUnsupportedMatched, assertAxeRules, @@ -31,9 +31,9 @@ import { type AccessibilityStep, type ManifestRecord, } from './adapter_harness'; -import {A2uiRendererService, SurfaceComponent, provideA2UI} from './public-api'; +import { A2uiRendererService, SurfaceComponent, provideA2UI } from './public-api'; -const specMetadataByFullName = new Map(); +const specMetadataByFullName = new Map(); let reporterInstalled = false; function ensureJasmineManifestReporter(): void { @@ -70,11 +70,15 @@ function ensureJasmineManifestReporter(): void { }); } +afterAll(() => { + assertAllUnsupportedMatched(); +}); + describe('@a2ui/angular Accessibility Conformance (conformance/core/accessibility.yaml)', () => { ensureJasmineManifestReporter(); const suiteTitle = '@a2ui/angular Accessibility Conformance (conformance/core/accessibility.yaml)'; - const {accessibility} = getConformanceSuites(); + const { accessibility } = getConformanceSuites(); for (const testCase of accessibility) { const specTitle = `${testCase.name}: ${testCase.description ?? ''}`; @@ -102,7 +106,7 @@ describe('@a2ui/angular Accessibility Conformance (conformance/core/accessibilit TestBed.configureTestingModule({ imports: [SurfaceComponent], - providers: [provideA2UI({catalogs: [createDefaultAdapterCatalog()]})], + providers: [provideA2UI({ catalogs: [createDefaultAdapterCatalog()] })], }); const rendererService = TestBed.inject(A2uiRendererService); let fixture: ComponentFixture | undefined; @@ -138,16 +142,12 @@ describe('@a2ui/angular Adapter Conformance (conformance/core/adapter_conformanc ensureJasmineManifestReporter(); const suiteTitle = '@a2ui/angular Adapter Conformance (conformance/core/adapter_conformance.yaml)'; - const {adapterConformance} = getConformanceSuites(); + const { adapterConformance } = getConformanceSuites(); beforeEach(() => { spyOn(window, 'open').and.returnValue(null); }); - afterAll(() => { - assertAllUnsupportedMatched(); - }); - for (const testCase of adapterConformance) { const specTitle = `${testCase.name}: ${testCase.description ?? ''}`; specMetadataByFullName.set(`${suiteTitle} ${specTitle}`, { diff --git a/typescript/renderers/angular/src/adapter_harness.ts b/typescript/renderers/angular/src/adapter_harness.ts index 508477daba..446f817d3e 100644 --- a/typescript/renderers/angular/src/adapter_harness.ts +++ b/typescript/renderers/angular/src/adapter_harness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import {ChangeDetectionStrategy, Component, computed} from '@angular/core'; -import {ComponentFixture, TestBed} from '@angular/core/testing'; +import { ChangeDetectionStrategy, Component, computed } from '@angular/core'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; import { CommonSchemas, type ComponentNode, @@ -26,9 +26,9 @@ import { ResolvedBinding, type SurfaceModel, } from '@a2ui/web_core/v0_9'; -import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; -import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; -import {z} from 'zod'; +import { basicCatalog } from '@a2ui/web_core/catalogs/basic/v1'; +import type { A2uiWebComponentElement } from '@a2ui/web_core/v0_9/universal'; +import { z } from 'zod'; import { A2uiRendererService, AngularCatalog, @@ -106,7 +106,7 @@ export interface ExtraCatalogEntry { { type: string; required?: string[]; - properties?: Record; + properties?: Record; } >; } @@ -249,9 +249,9 @@ function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: str if (typeof obj !== 'object' || Array.isArray(obj)) { throw new Error(`${where}: expected object`); } - const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); + const unknown = Object.keys(obj as Record).filter((key) => !allowed.has(key)); if (unknown.length > 0) { - throw new Error(`${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`); + throw new Error(`${where}: unknown keys: ${unknown.map((k) => `${where}.${k}`).join(', ')}`); } } @@ -392,12 +392,22 @@ export function findUnsupportedReason( return UNSUPPORTED[stepKey]; } } + const assertions = step['assertions']; + if (assertions && typeof assertions === 'object' && !Array.isArray(assertions)) { + for (const key of Object.keys(assertions as Record)) { + const assertionKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, assertionKey)) { + matchedUnsupportedKeys.add(assertionKey); + return UNSUPPORTED[assertionKey]; + } + } + } } return undefined; } export function assertAllUnsupportedMatched(): void { - const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); + const unused = Object.keys(UNSUPPORTED).filter((key) => !matchedUnsupportedKeys.has(key)); if (unused.length > 0) { throw new Error( `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, @@ -408,10 +418,10 @@ export function assertAllUnsupportedMatched(): void { export function writeManifestRecord(rec: ManifestRecord): void { const karma = ( window as unknown as { - __karma__?: {info?: (payload: {conformanceManifestRecord: ManifestRecord}) => void}; + __karma__?: { info?: (payload: { conformanceManifestRecord: ManifestRecord }) => void }; } ).__karma__; - karma?.info?.({conformanceManifestRecord: rec}); + karma?.info?.({ conformanceManifestRecord: rec }); } const RESERVED_VIEW_KEYS = new Set([ @@ -451,7 +461,7 @@ class ConformanceCustomWidgetComponent extends CatalogComponent(resolve => setTimeout(resolve, 0)); + await new Promise((resolve) => setTimeout(resolve, 0)); const elements = Array.from(container.querySelectorAll('*')); for (const el of elements) { - const litEl = el as unknown as {updateComplete?: Promise}; + const litEl = el as unknown as { updateComplete?: Promise }; if (litEl.updateComplete) { await litEl.updateComplete; } @@ -632,7 +642,7 @@ function findDomElementForNode( node: ComponentNode, ): HTMLElement | undefined { const match = Array.from(container.querySelectorAll('*')).find( - el => (el as A2uiWebComponentElement).node === node, + (el) => (el as A2uiWebComponentElement).node === node, ); return match as HTMLElement | undefined; } @@ -789,19 +799,19 @@ export function assertAxeRules(container: HTMLElement, rules: string[]): void { function countActiveSubscriptions(surface: SurfaceModel): number { const createdListeners = - (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? - 0; + (surface.componentsModel.onCreated as unknown as { listeners?: Set }).listeners + ?.size ?? 0; const deletedListeners = - (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? - 0; + (surface.componentsModel.onDeleted as unknown as { listeners?: Set }).listeners + ?.size ?? 0; let componentUpdatedListeners = 0; for (const comp of surface.componentsModel.values) { componentUpdatedListeners += - (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; + (comp.onUpdated as unknown as { listeners?: Set }).listeners?.size ?? 0; } const dataModelInternal = surface.dataModel as unknown as { subscriptions?: Set; - signals?: Map; + signals?: Map; }; let activeSignalTargets = 0; for (const sig of dataModelInternal.signals?.values() ?? []) { @@ -835,8 +845,8 @@ async function performStepOperation(state: HarnessState, step: AdapterStep): Pro const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; expect(input).withContext(`Input element for "${targetId}"`).not.toBeNull(); input!.value = String(step.inputValue ?? ''); - input!.dispatchEvent(new Event('input', {bubbles: true, composed: true})); - input!.dispatchEvent(new Event('change', {bubbles: true, composed: true})); + input!.dispatchEvent(new Event('input', { bubbles: true, composed: true })); + input!.dispatchEvent(new Event('change', { bubbles: true, composed: true })); return; } if (step.op === 'user_gesture') { @@ -938,7 +948,7 @@ function mapNodesToDomElements( const result = new Map(); const allElements = Array.from(container.querySelectorAll('*')); for (const [nodePath, node] of nodes) { - const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); + const match = allElements.find((el) => (el as A2uiWebComponentElement).node === node); if (match) { result.set(nodePath, match as HTMLElement); } @@ -953,7 +963,7 @@ export function mountSurface(caseSpec?: Partial): HarnessState ]; TestBed.configureTestingModule({ imports: [SurfaceComponent], - providers: [provideA2UI({catalogs})], + providers: [provideA2UI({ catalogs })], }); const rendererService = TestBed.inject(A2uiRendererService); const state: HarnessState = { @@ -964,12 +974,12 @@ export function mountSurface(caseSpec?: Partial): HarnessState previousNodes: new Map(), previousDomElements: new Map(), }; - rendererService.surfaceGroup.onSurfaceCreated.subscribe(surface => { + rendererService.surfaceGroup.onSurfaceCreated.subscribe((surface) => { state.surface = surface; - surface.onAction.subscribe(action => { + surface.onAction.subscribe((action) => { state.actions.push(action as unknown as Record); }); - surface.onError.subscribe(err => { + surface.onError.subscribe((err) => { state.errors.push(err as unknown as Record); }); }); diff --git a/typescript/renderers/lit/src/tests/adapter_harness.ts b/typescript/renderers/lit/src/tests/adapter_harness.ts index 6d80a3065f..9d89d332ea 100644 --- a/typescript/renderers/lit/src/tests/adapter_harness.ts +++ b/typescript/renderers/lit/src/tests/adapter_harness.ts @@ -382,6 +382,16 @@ export function findUnsupportedReason( return UNSUPPORTED[stepKey]; } } + const assertions = step.assertions; + if (assertions && typeof assertions === 'object' && !Array.isArray(assertions)) { + for (const key of Object.keys(assertions as Record)) { + const assertionKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, assertionKey)) { + matchedUnsupportedKeys.add(assertionKey); + return UNSUPPORTED[assertionKey]; + } + } + } } return undefined; } diff --git a/typescript/renderers/react/tests/adapter_conformance.test.tsx b/typescript/renderers/react/tests/adapter_conformance.test.tsx index d9bc5ac794..4fa96b70e6 100644 --- a/typescript/renderers/react/tests/adapter_conformance.test.tsx +++ b/typescript/renderers/react/tests/adapter_conformance.test.tsx @@ -95,7 +95,7 @@ describe('@a2ui/react Accessibility Conformance (conformance/core/accessibility. if (skipReason) { record.status = 'skipped'; record.reason = skipReason; - ctx.skip(); + ctx.skip(skipReason); return; } @@ -124,8 +124,10 @@ describe('@a2ui/react Accessibility Conformance (conformance/core/accessibility. } record.status = 'passed'; } catch (err) { - record.status = 'failed'; - record.reason = err instanceof Error ? err.message : String(err); + if ((record.status as string) !== 'skipped') { + record.status = 'failed'; + record.reason = err instanceof Error ? err.message : String(err); + } throw err; } finally { rendered?.unmount(); @@ -155,7 +157,7 @@ describe('@a2ui/react Adapter Conformance (conformance/core/adapter_conformance. status: 'failed', }; pendingManifestByTestName.set(testTitle, record); - const state = mountSurface(testCase); + let state: ReturnType | undefined; try { assertKnownAdapterCase(testCase); const skipReason = findUnsupportedReason( @@ -165,20 +167,23 @@ describe('@a2ui/react Adapter Conformance (conformance/core/adapter_conformance. if (skipReason) { record.status = 'skipped'; record.reason = skipReason; - ctx.skip(); + ctx.skip(skipReason); return; } + state = mountSurface(testCase); for (const step of testCase.steps) { await runStep(state, step); } record.status = 'passed'; } catch (err) { - record.status = 'failed'; - record.reason = err instanceof Error ? err.message : String(err); + if ((record.status as string) !== 'skipped') { + record.status = 'failed'; + record.reason = err instanceof Error ? err.message : String(err); + } throw err; } finally { - if (!state.unmounted) { + if (state && !state.unmounted) { state.rendered?.unmount(); } } diff --git a/typescript/renderers/react/tests/adapter_harness.tsx b/typescript/renderers/react/tests/adapter_harness.tsx index 208cd26b32..420a70d2eb 100644 --- a/typescript/renderers/react/tests/adapter_harness.tsx +++ b/typescript/renderers/react/tests/adapter_harness.tsx @@ -379,6 +379,16 @@ export function findUnsupportedReason( return UNSUPPORTED[stepKey]; } } + const assertions = step.assertions; + if (assertions && typeof assertions === 'object' && !Array.isArray(assertions)) { + for (const key of Object.keys(assertions as Record)) { + const assertionKey = `${caseName}/${key}`; + if (Object.hasOwn(UNSUPPORTED, assertionKey)) { + matchedUnsupportedKeys.add(assertionKey); + return UNSUPPORTED[assertionKey]; + } + } + } } return undefined; } From e89d67629576802d89150caa6b5c69f0ecb5b4f2 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 07:57:59 +0000 Subject: [PATCH 25/48] test(conformance): fix Swift MainActor isolation and Angular formatting --- .../AdapterConformanceTests.swift | 11 +++- .../A2UISwiftUITests/AdapterHarness.swift | 3 +- typescript/renderers/angular/karma.conf.js | 8 +-- .../angular/src/adapter_conformance.spec.ts | 12 ++-- .../renderers/angular/src/adapter_harness.ts | 58 +++++++++---------- 5 files changed, 49 insertions(+), 43 deletions(-) diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift index 672593a545..a41872ef92 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift @@ -68,9 +68,14 @@ struct AdapterConformanceTests { let name = testCase["name"] as? String ?? "" do { try AdapterHarness.assertKnownAccessibilityCase(testCase) - let steps = - (testCase["steps"] as? [[String: Any]]) - ?? [["assertions": testCase["assertions"] as Any]] + let steps: [[String: Any]] + if let rawSteps = testCase["steps"] as? [[String: Any]] { + steps = rawSteps + } else if let assertions = testCase["assertions"] as? [String: Any] { + steps = [["assertions": assertions]] + } else { + steps = [] + } if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { writeManifestRecord( AdapterManifestRecord( diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift index e61467223b..e62d0b8cdc 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift @@ -519,7 +519,7 @@ enum AdapterHarness { } /// Appends a manifest record to `$A2UI_CONFORMANCE_MANIFEST_DIR/-.jsonl`. - nonisolated static func writeManifestRecord(_ rec: AdapterManifestRecord) { + static func writeManifestRecord(_ rec: AdapterManifestRecord) { guard let manifestDir = ProcessInfo.processInfo.environment["A2UI_CONFORMANCE_MANIFEST_DIR"], !manifestDir.isEmpty @@ -1283,6 +1283,7 @@ func assertKnownKeys(_ step: [String: Any]) throws { /// Top-level helper appending a conformance manifest record when `A2UI_CONFORMANCE_MANIFEST_DIR` /// is set. +@MainActor func writeManifestRecord(_ rec: AdapterManifestRecord) { AdapterHarness.writeManifestRecord(rec) } diff --git a/typescript/renderers/angular/karma.conf.js b/typescript/renderers/angular/karma.conf.js index 2504637dc8..14c064f2a8 100644 --- a/typescript/renderers/angular/karma.conf.js +++ b/typescript/renderers/angular/karma.conf.js @@ -43,13 +43,13 @@ function ConformanceManifestReporter(baseReporterDecorator) { if (!manifestPath || !rec || !rec.suite || !rec.case) { return; } - fs.mkdirSync(manifestDir, { recursive: true }); + fs.mkdirSync(manifestDir, {recursive: true}); const entry = { harness: rec.harness || 'angular', suite: String(rec.suite).replace(/\\/g, '/'), case: rec.case, status: rec.status, - ...(rec.reason ? { reason: rec.reason } : {}), + ...(rec.reason ? {reason: rec.reason} : {}), }; fs.appendFileSync(manifestPath, `${JSON.stringify(entry)}\n`, 'utf8'); } @@ -71,7 +71,7 @@ module.exports = function (config) { require('karma-chrome-launcher'), require('karma-jasmine-html-reporter'), require('karma-coverage'), - { 'reporter:conformance-manifest': ['type', ConformanceManifestReporter] }, + {'reporter:conformance-manifest': ['type', ConformanceManifestReporter]}, ], client: { jasmine: { @@ -92,7 +92,7 @@ module.exports = function (config) { coverageReporter: { dir: require('path').join(__dirname, './coverage/lib'), subdir: '.', - reporters: [{ type: 'html' }, { type: 'text-summary' }], + reporters: [{type: 'html'}, {type: 'text-summary'}], }, reporters: ['progress', 'kjhtml', 'conformance-manifest'], browsers: ['Chrome'], diff --git a/typescript/renderers/angular/src/adapter_conformance.spec.ts b/typescript/renderers/angular/src/adapter_conformance.spec.ts index e4c4590738..f6debefcdd 100644 --- a/typescript/renderers/angular/src/adapter_conformance.spec.ts +++ b/typescript/renderers/angular/src/adapter_conformance.spec.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import {ComponentFixture, TestBed} from '@angular/core/testing'; import { assertAllUnsupportedMatched, assertAxeRules, @@ -31,9 +31,9 @@ import { type AccessibilityStep, type ManifestRecord, } from './adapter_harness'; -import { A2uiRendererService, SurfaceComponent, provideA2UI } from './public-api'; +import {A2uiRendererService, SurfaceComponent, provideA2UI} from './public-api'; -const specMetadataByFullName = new Map(); +const specMetadataByFullName = new Map(); let reporterInstalled = false; function ensureJasmineManifestReporter(): void { @@ -78,7 +78,7 @@ describe('@a2ui/angular Accessibility Conformance (conformance/core/accessibilit ensureJasmineManifestReporter(); const suiteTitle = '@a2ui/angular Accessibility Conformance (conformance/core/accessibility.yaml)'; - const { accessibility } = getConformanceSuites(); + const {accessibility} = getConformanceSuites(); for (const testCase of accessibility) { const specTitle = `${testCase.name}: ${testCase.description ?? ''}`; @@ -106,7 +106,7 @@ describe('@a2ui/angular Accessibility Conformance (conformance/core/accessibilit TestBed.configureTestingModule({ imports: [SurfaceComponent], - providers: [provideA2UI({ catalogs: [createDefaultAdapterCatalog()] })], + providers: [provideA2UI({catalogs: [createDefaultAdapterCatalog()]})], }); const rendererService = TestBed.inject(A2uiRendererService); let fixture: ComponentFixture | undefined; @@ -142,7 +142,7 @@ describe('@a2ui/angular Adapter Conformance (conformance/core/adapter_conformanc ensureJasmineManifestReporter(); const suiteTitle = '@a2ui/angular Adapter Conformance (conformance/core/adapter_conformance.yaml)'; - const { adapterConformance } = getConformanceSuites(); + const {adapterConformance} = getConformanceSuites(); beforeEach(() => { spyOn(window, 'open').and.returnValue(null); diff --git a/typescript/renderers/angular/src/adapter_harness.ts b/typescript/renderers/angular/src/adapter_harness.ts index 446f817d3e..ebf0d6d0dc 100644 --- a/typescript/renderers/angular/src/adapter_harness.ts +++ b/typescript/renderers/angular/src/adapter_harness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { ChangeDetectionStrategy, Component, computed } from '@angular/core'; -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import {ChangeDetectionStrategy, Component, computed} from '@angular/core'; +import {ComponentFixture, TestBed} from '@angular/core/testing'; import { CommonSchemas, type ComponentNode, @@ -26,9 +26,9 @@ import { ResolvedBinding, type SurfaceModel, } from '@a2ui/web_core/v0_9'; -import { basicCatalog } from '@a2ui/web_core/catalogs/basic/v1'; -import type { A2uiWebComponentElement } from '@a2ui/web_core/v0_9/universal'; -import { z } from 'zod'; +import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; +import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; +import {z} from 'zod'; import { A2uiRendererService, AngularCatalog, @@ -106,7 +106,7 @@ export interface ExtraCatalogEntry { { type: string; required?: string[]; - properties?: Record; + properties?: Record; } >; } @@ -249,9 +249,9 @@ function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: str if (typeof obj !== 'object' || Array.isArray(obj)) { throw new Error(`${where}: expected object`); } - const unknown = Object.keys(obj as Record).filter((key) => !allowed.has(key)); + const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); if (unknown.length > 0) { - throw new Error(`${where}: unknown keys: ${unknown.map((k) => `${where}.${k}`).join(', ')}`); + throw new Error(`${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`); } } @@ -407,7 +407,7 @@ export function findUnsupportedReason( } export function assertAllUnsupportedMatched(): void { - const unused = Object.keys(UNSUPPORTED).filter((key) => !matchedUnsupportedKeys.has(key)); + const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); if (unused.length > 0) { throw new Error( `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, @@ -418,10 +418,10 @@ export function assertAllUnsupportedMatched(): void { export function writeManifestRecord(rec: ManifestRecord): void { const karma = ( window as unknown as { - __karma__?: { info?: (payload: { conformanceManifestRecord: ManifestRecord }) => void }; + __karma__?: {info?: (payload: {conformanceManifestRecord: ManifestRecord}) => void}; } ).__karma__; - karma?.info?.({ conformanceManifestRecord: rec }); + karma?.info?.({conformanceManifestRecord: rec}); } const RESERVED_VIEW_KEYS = new Set([ @@ -461,7 +461,7 @@ class ConformanceCustomWidgetComponent extends CatalogComponent((resolve) => setTimeout(resolve, 0)); + await new Promise(resolve => setTimeout(resolve, 0)); const elements = Array.from(container.querySelectorAll('*')); for (const el of elements) { - const litEl = el as unknown as { updateComplete?: Promise }; + const litEl = el as unknown as {updateComplete?: Promise}; if (litEl.updateComplete) { await litEl.updateComplete; } @@ -642,7 +642,7 @@ function findDomElementForNode( node: ComponentNode, ): HTMLElement | undefined { const match = Array.from(container.querySelectorAll('*')).find( - (el) => (el as A2uiWebComponentElement).node === node, + el => (el as A2uiWebComponentElement).node === node, ); return match as HTMLElement | undefined; } @@ -799,19 +799,19 @@ export function assertAxeRules(container: HTMLElement, rules: string[]): void { function countActiveSubscriptions(surface: SurfaceModel): number { const createdListeners = - (surface.componentsModel.onCreated as unknown as { listeners?: Set }).listeners - ?.size ?? 0; + (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? + 0; const deletedListeners = - (surface.componentsModel.onDeleted as unknown as { listeners?: Set }).listeners - ?.size ?? 0; + (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? + 0; let componentUpdatedListeners = 0; for (const comp of surface.componentsModel.values) { componentUpdatedListeners += - (comp.onUpdated as unknown as { listeners?: Set }).listeners?.size ?? 0; + (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; } const dataModelInternal = surface.dataModel as unknown as { subscriptions?: Set; - signals?: Map; + signals?: Map; }; let activeSignalTargets = 0; for (const sig of dataModelInternal.signals?.values() ?? []) { @@ -845,8 +845,8 @@ async function performStepOperation(state: HarnessState, step: AdapterStep): Pro const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; expect(input).withContext(`Input element for "${targetId}"`).not.toBeNull(); input!.value = String(step.inputValue ?? ''); - input!.dispatchEvent(new Event('input', { bubbles: true, composed: true })); - input!.dispatchEvent(new Event('change', { bubbles: true, composed: true })); + input!.dispatchEvent(new Event('input', {bubbles: true, composed: true})); + input!.dispatchEvent(new Event('change', {bubbles: true, composed: true})); return; } if (step.op === 'user_gesture') { @@ -948,7 +948,7 @@ function mapNodesToDomElements( const result = new Map(); const allElements = Array.from(container.querySelectorAll('*')); for (const [nodePath, node] of nodes) { - const match = allElements.find((el) => (el as A2uiWebComponentElement).node === node); + const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); if (match) { result.set(nodePath, match as HTMLElement); } @@ -963,7 +963,7 @@ export function mountSurface(caseSpec?: Partial): HarnessState ]; TestBed.configureTestingModule({ imports: [SurfaceComponent], - providers: [provideA2UI({ catalogs })], + providers: [provideA2UI({catalogs})], }); const rendererService = TestBed.inject(A2uiRendererService); const state: HarnessState = { @@ -974,12 +974,12 @@ export function mountSurface(caseSpec?: Partial): HarnessState previousNodes: new Map(), previousDomElements: new Map(), }; - rendererService.surfaceGroup.onSurfaceCreated.subscribe((surface) => { + rendererService.surfaceGroup.onSurfaceCreated.subscribe(surface => { state.surface = surface; - surface.onAction.subscribe((action) => { + surface.onAction.subscribe(action => { state.actions.push(action as unknown as Record); }); - surface.onError.subscribe((err) => { + surface.onError.subscribe(err => { state.errors.push(err as unknown as Record); }); }); From edc8042fbd72d0a6027845eba326b9ae8db3f51d Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 16:52:20 +0000 Subject: [PATCH 26/48] test(a2ui_agent-ts): include VITEST_WORKER_ID in manifest filename --- typescript/agents/a2ui_agent/tests/conformance/manifest.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/typescript/agents/a2ui_agent/tests/conformance/manifest.ts b/typescript/agents/a2ui_agent/tests/conformance/manifest.ts index 7ec04315a9..5478ded7e2 100644 --- a/typescript/agents/a2ui_agent/tests/conformance/manifest.ts +++ b/typescript/agents/a2ui_agent/tests/conformance/manifest.ts @@ -57,7 +57,8 @@ export function recordCase( if (reason) { record['reason'] = reason; } - const filePath = join(manifestDir, `ts-agent-${process.pid}.jsonl`); + const workerId = process.env['VITEST_WORKER_ID'] ?? '0'; + const filePath = join(manifestDir, `ts-agent-${process.pid}-${workerId}.jsonl`); appendFileSync(filePath, `${JSON.stringify(record)}\n`, 'utf8'); } From 28d8a3b9096e2156e646baf064e80c70278aee8d Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 19:22:19 +0000 Subject: [PATCH 27/48] test(a2ui_agent): record executed cases and enforce strict dispatch in Python agent harnesses Add a pytest hookwrapper in python/a2ui_agent/tests/conformance/conftest.py to write executed-case manifest records for the python-agent harness, emit explicit skip marks with reasons for skipped cases, performance-only scaling tests, and unsupported YAML suites, enforce strict expectation key allowlists across the Python agent conformance runners, and verify the python-agent manifest in Python CI. --- .github/workflows/python_ci.yml | 4 + .../tests/conformance/conformance_helpers.py | 39 +++ .../a2ui_agent/tests/conformance/conftest.py | 140 ++++++++ .../conformance/test_builder_conformance.py | 7 +- .../conformance/test_catalog_conformance.py | 53 ++- .../tests/conformance/test_conformance.py | 274 ++++++++++++--- .../conformance/test_macro_conformance.py | 40 ++- .../conformance/test_scaling_conformance.py | 24 +- .../conformance/test_skill_conformance.py | 323 +++++++++--------- .../conformance/test_unsupported_suites.py | 106 ++++++ 10 files changed, 782 insertions(+), 228 deletions(-) create mode 100644 python/a2ui_agent/tests/conformance/conftest.py create mode 100644 python/a2ui_agent/tests/conformance/test_unsupported_suites.py diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml index 413b3b9b62..22007ec555 100644 --- a/.github/workflows/python_ci.yml +++ b/.github/workflows/python_ci.yml @@ -129,6 +129,10 @@ jobs: --corpus extensions \ --only 'extensions/adk/**' \ --harness python-adk + uv run --project conformance python conformance/scripts/check_executed_cases.py \ + --manifest-dir "${{ runner.temp }}/a2ui-manifest" \ + --corpus agent \ + --harness python-agent - name: Set Up Dart uses: ./.github/actions/setup-dart - name: Run CLI Python Runtime Tests diff --git a/python/a2ui_agent/tests/conformance/conformance_helpers.py b/python/a2ui_agent/tests/conformance/conformance_helpers.py index a293534108..2f701773fc 100644 --- a/python/a2ui_agent/tests/conformance/conformance_helpers.py +++ b/python/a2ui_agent/tests/conformance/conformance_helpers.py @@ -14,10 +14,16 @@ """Shared utility functions for A2UI conformance test harnesses.""" +from __future__ import annotations + +from collections.abc import Iterable, Mapping import json import os from typing import Any + +import pytest import yaml + from ..repo_paths import find_repo_root @@ -53,3 +59,36 @@ def load_conformance_yaml(filename: str) -> Any: path = get_conformance_path(filename) with open(path, "r", encoding="utf-8") as f: return yaml.safe_load(f) + + +def case_params( + suite: str, + cases: Iterable[Any], + skip: Mapping[str, str] | None = None, + xfail: Mapping[str, str] | None = None, +) -> list[Any]: + """Builds parameterized `(suite, case)` entries with skip and xfail markers.""" + params: list[Any] = [] + for case in cases: + if isinstance(case, Mapping): + case_id = str(case.get("id") or case.get("name")) + else: + case_id = str(getattr(case, "id", None) or getattr(case, "name", None)) + marks: list[pytest.MarkDecorator] = [] + if skip and case_id in skip: + marks.append(pytest.mark.skip(reason=skip[case_id])) + elif xfail and case_id in xfail: + marks.append(pytest.mark.xfail(reason=xfail[case_id], strict=True)) + params.append(pytest.param(suite, case, marks=marks, id=case_id)) + return params + + +def assert_known_keys( + obj: Mapping[str, Any], + allowed: frozenset[str], + where: str, +) -> None: + """Fails the test if `obj` contains any keys not in `allowed`.""" + unknown = sorted(set(obj) - allowed) + if unknown: + pytest.fail(f"Unknown {where} keys: {unknown}") diff --git a/python/a2ui_agent/tests/conformance/conftest.py b/python/a2ui_agent/tests/conformance/conftest.py new file mode 100644 index 0000000000..039b8b5d1d --- /dev/null +++ b/python/a2ui_agent/tests/conformance/conftest.py @@ -0,0 +1,140 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pytest configuration for the a2ui_agent conformance tests.""" + +from __future__ import annotations + +from collections.abc import Generator, Sequence +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +from .test_scaling_conformance import SCALING_SKIP_REASON + + +def _extract_case_identity(item: pytest.Item) -> tuple[str, str] | None: + """Returns `(suite, case_name)` for a parameterized conformance item.""" + callspec = getattr(item, "callspec", None) + params = getattr(callspec, "params", None) + if not isinstance(params, dict): + return None + suite = params.get("suite") + if not isinstance(suite, str) or not suite: + return None + normalized_suite = suite.replace("\\", "/") + name = params.get("name") + if isinstance(name, str) and name: + return normalized_suite, name + case = params.get("case") or params.get("test_case") + if isinstance(case, dict): + case_name = case.get("id") or case.get("name") + if isinstance(case_name, str) and case_name: + return normalized_suite, case_name + else: + case_id = getattr(case, "id", None) or getattr(case, "name", None) + if isinstance(case_id, str) and case_id: + return normalized_suite, case_id + return None + + +def _append_manifest_record( + suite: str, case_name: str, status: str, reason: str | None = None +) -> None: + """Appends one executed-case record when `A2UI_CONFORMANCE_MANIFEST_DIR` is set.""" + manifest_dir = os.environ.get("A2UI_CONFORMANCE_MANIFEST_DIR") + if not manifest_dir: + return + os.makedirs(manifest_dir, exist_ok=True) + manifest_path = Path(manifest_dir) / f"python-agent-{os.getpid()}.jsonl" + record: dict[str, str] = { + "harness": "python-agent", + "suite": suite, + "case": case_name, + "status": status, + } + if reason: + record["reason"] = reason + with manifest_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record) + "\n") + + +def _extract_skip_reason(report: pytest.TestReport) -> str: + """Extracts a human-readable skip or xfail reason from a pytest TestReport.""" + wasxfail = getattr(report, "wasxfail", None) + if isinstance(wasxfail, str) and wasxfail.strip(): + return wasxfail.strip() + if wasxfail is not None: + return "expected failure" + longrepr = getattr(report, "longrepr", None) + if isinstance(longrepr, tuple) and len(longrepr) >= 3: + text = str(longrepr[2]).removeprefix("Skipped: ").strip() + return text or "skipped" + if longrepr: + return str(longrepr).removeprefix("Skipped: ").strip() or "skipped" + return "skipped" + + +def _report_status_and_reason( + report: pytest.TestReport, +) -> tuple[str, str | None]: + """Maps a pytest TestReport to `(status, reason)` for the manifest.""" + wasxfail = getattr(report, "wasxfail", None) + if wasxfail is not None: + if report.outcome == "skipped": + return "xfailed", _extract_skip_reason(report) + return "failed", None + if report.outcome == "skipped": + return "skipped", _extract_skip_reason(report) + if report.outcome == "passed": + return "passed", None + return "failed", None + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[Any] +) -> Generator[None, Any, None]: + """Records executed conformance test outcomes to the manifest JSONL file.""" + del call + outcome = yield + report: pytest.TestReport = outcome.get_result() + if report.when != "call" and not ( + report.when == "setup" and report.outcome != "passed" + ): + return + identity = _extract_case_identity(item) + if identity is None: + return + status, reason = _report_status_and_reason(report) + _append_manifest_record(identity[0], identity[1], status, reason) + + +def pytest_deselected(items: Sequence[pytest.Item]) -> None: + """Records deselected performance timing cases as skipped in the manifest.""" + for item in items: + if getattr(item, "originalname", None) != "test_scaling_conformance": + continue + identity = _extract_case_identity(item) + if identity is None: + continue + _append_manifest_record( + identity[0], + identity[1], + "skipped", + SCALING_SKIP_REASON, + ) diff --git a/python/a2ui_agent/tests/conformance/test_builder_conformance.py b/python/a2ui_agent/tests/conformance/test_builder_conformance.py index a7eeff54bf..e3bca4ee90 100644 --- a/python/a2ui_agent/tests/conformance/test_builder_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_builder_conformance.py @@ -19,13 +19,16 @@ import pytest from .builder_suite import Case, load_cases, run_case, validate_payload +from .conformance_helpers import case_params +SUITE = "agent/builder/builder.yaml" CASES = load_cases() -@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id) -def test_matches_golden(case: Case) -> None: +@pytest.mark.parametrize("suite, case", case_params(SUITE, CASES)) +def test_matches_golden(suite: str, case: Case) -> None: """The builder emits exactly the recorded wire payload.""" + del suite assert run_case(case) == case.load_golden(), case.description diff --git a/python/a2ui_agent/tests/conformance/test_catalog_conformance.py b/python/a2ui_agent/tests/conformance/test_catalog_conformance.py index d38ae99dfe..f9ff1cd7a8 100644 --- a/python/a2ui_agent/tests/conformance/test_catalog_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_catalog_conformance.py @@ -20,6 +20,8 @@ directly. """ +from __future__ import annotations + from collections.abc import Mapping from typing import Any @@ -40,11 +42,31 @@ from a2ui.utils import resolve_catalogs from .conformance_helpers import ( + assert_known_keys, + case_params, get_conformance_path, load_conformance_json, load_conformance_yaml, ) +_CASE_KEYS = frozenset({ + "name", + "description", + "features", + "action", + "args", + "expect", + "expect_error", +}) +_EXPECT_ERROR_KEYS = frozenset({"category", "message"}) +_CATALOG_EXPECT_KEYS = frozenset({ + "catalog_id", + "protocol_version", + "components", + "functions", +}) +_RESOLVE_EXPECT_KEYS = frozenset({"active_catalog_ids", "catalogs"}) + def _transformer(spec: Mapping[str, Any]) -> CatalogTransformer: if "component_pruning" in spec: @@ -78,6 +100,7 @@ def _catalog_config(entry: str | Mapping[str, Any]) -> CatalogConfig: def _expect_catalog(catalog: CatalogApi, expected: Mapping[str, Any]) -> None: """Checks a catalog against a case's expectations. Name lists are exhaustive.""" + assert_known_keys(expected, _CATALOG_EXPECT_KEYS, "expect.catalog") if "catalog_id" in expected: assert catalog.catalog_id == expected["catalog_id"] if "protocol_version" in expected: @@ -99,15 +122,18 @@ def _expect_catalog(catalog: CatalogApi, expected: Mapping[str, Any]) -> None: provider_cases = load_conformance_yaml("agent/catalog_provider.yaml") -def test_catalog_provider_suite_is_not_empty(): +def test_catalog_provider_suite_is_not_empty() -> None: assert provider_cases @pytest.mark.parametrize( - "case", provider_cases, ids=[case["name"] for case in provider_cases] + "suite, case", + case_params("agent/catalog_provider.yaml", provider_cases), ) -def test_catalog_provider_conformance(case): +def test_catalog_provider_conformance(suite: str, case: Mapping[str, Any]) -> None: + del suite assert case["action"] == "provide_catalog" + assert_known_keys(case, _CASE_KEYS, "provide_catalog case") args = case["args"] def provide() -> CatalogApi: @@ -127,6 +153,7 @@ def provide() -> CatalogApi: raise ValueError(f"Unknown provider: {provider_type}") if "expect_error" in case: + assert_known_keys(case["expect_error"], _EXPECT_ERROR_KEYS, "expect_error") with pytest.raises(_ERROR_CATEGORIES[case["expect_error"]["category"]]): provide() return @@ -138,15 +165,18 @@ def provide() -> CatalogApi: transformer_cases = load_conformance_yaml("agent/catalog_transformer.yaml") -def test_catalog_transformer_suite_is_not_empty(): +def test_catalog_transformer_suite_is_not_empty() -> None: assert transformer_cases @pytest.mark.parametrize( - "case", transformer_cases, ids=[case["name"] for case in transformer_cases] + "suite, case", + case_params("agent/catalog_transformer.yaml", transformer_cases), ) -def test_catalog_transformer_conformance(case): +def test_catalog_transformer_conformance(suite: str, case: Mapping[str, Any]) -> None: + del suite assert case["action"] == "transform_catalog" + assert_known_keys(case, _CASE_KEYS, "transform_catalog case") catalog = _catalog_config(case["args"]).transformed_catalog _expect_catalog(catalog, case["expect"]) @@ -154,15 +184,18 @@ def test_catalog_transformer_conformance(case): resolution_cases = load_conformance_yaml("agent/catalog_resolution.yaml") -def test_catalog_resolution_suite_is_not_empty(): +def test_catalog_resolution_suite_is_not_empty() -> None: assert resolution_cases @pytest.mark.parametrize( - "case", resolution_cases, ids=[case["name"] for case in resolution_cases] + "suite, case", + case_params("agent/catalog_resolution.yaml", resolution_cases), ) -def test_catalog_resolution_conformance(case): +def test_catalog_resolution_conformance(suite: str, case: Mapping[str, Any]) -> None: + del suite assert case["action"] == "resolve_catalogs" + assert_known_keys(case, _CASE_KEYS, "resolve_catalogs case") args = case["args"] def resolve() -> list[CatalogApi]: @@ -173,12 +206,14 @@ def resolve() -> list[CatalogApi]: ) if "expect_error" in case: + assert_known_keys(case["expect_error"], _EXPECT_ERROR_KEYS, "expect_error") with pytest.raises(_ERROR_CATEGORIES[case["expect_error"]["category"]]): resolve() return catalogs = resolve() expect = case["expect"] + assert_known_keys(expect, _RESOLVE_EXPECT_KEYS, "resolve_catalogs expect") catalog_ids = [catalog.catalog_id for catalog in catalogs] assert sorted(catalog_ids) == sorted(expect["active_catalog_ids"]) by_id = {catalog.catalog_id: catalog for catalog in catalogs} diff --git a/python/a2ui_agent/tests/conformance/test_conformance.py b/python/a2ui_agent/tests/conformance/test_conformance.py index b6cf16e852..f1f4375fee 100644 --- a/python/a2ui_agent/tests/conformance/test_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_conformance.py @@ -14,6 +14,7 @@ from __future__ import annotations +from collections.abc import Mapping import contextlib import os import re @@ -47,6 +48,8 @@ from a2ui.processor import InMemoryCatalogProvider from a2ui.utils import load_examples, resolve_catalogs, validate_payload +from .conformance_helpers import assert_known_keys +from .conformance_helpers import case_params from .conformance_helpers import get_conformance_path from .conformance_helpers import load_conformance_json as load_json_file from .conformance_helpers import load_conformance_yaml as load_tests @@ -64,17 +67,131 @@ SUPPORTED_PROTOCOL_VERSIONS = {"v0.8", "v0.9", "1.0"} # Transition skip list containing specific test case names to skip during active feature transitions. -SKIP_TEST_NAMES: set[str] = set() +SKIP_TEST_NAMES: dict[str, str] | set[str] = {} # Transition skip list containing specific test suite files to skip during active feature transitions. -SKIP_TEST_SUITES = { - "core/catalog.yaml", +SKIP_TEST_SUITES: dict[str, str] = { + "core/catalog.yaml": "Core catalog suite runs in python-core.", } +_CASE_METADATA_KEYS = frozenset({ + "name", + "description", + "features", + "protocolVersion", + "catalog", + "catalogs", +}) +_EXPECT_ERROR_KEYS = frozenset({"category", "message", "details"}) +_EXPECT_ERROR_DETAIL_KEYS = frozenset({"path", "code", "message"}) +_COMPILE_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "input", + "expect", + "expect_error", + "expect_present", +}) +_DECOMPILE_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "messages", + "expect_contains", + "expect_absent", + "expect_error", + "expect_recompiled", + "expect_round_trip", +}) +_UNWRAP_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "input", + "expect", +}) +_HAS_FORMAT_CONTENT_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "input", + "expect", +}) +_WRAP_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "parts", + "expect_output", + "expect_contains", + "expect_round_trip", +}) +_PARSE_RESPONSE_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "input", + "expect", + "expect_error", +}) +_PARSE_CHUNK_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "steps", + "expect_close", + "expect_error", + "expect_matches_single_shot", +}) +_PARSE_CHUNK_STEP_KEYS = frozenset({ + "input", + "expect", + "expect_error", + "expect_in_block", + "expect_partial_results", +}) +_GENERATE_PROMPT_SNIPPET_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "expect_file", + "expect_contains", + "expect_absent", + "expect_deterministic", + "expect_error", +}) +_CREATE_FORMAT_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "expect", + "expect_error", +}) +_CREATE_FORMAT_EXPECT_KEYS = frozenset({ + "parsers_are_distinct", + "parser_state_isolated", + "prompt_snippet_contains", + "prompt_snippet_absent", +}) +_CREATE_PROCESSOR_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "action", + "args", + "expect", + "expect_error", + "then_parse", +}) +_CREATE_PROCESSOR_EXPECT_KEYS = frozenset({ + "active_catalog_ids", + "catalogs", + "generator_catalogs_unchanged", + "prompt_snippet_contains", + "prompt_snippet_absent", +}) +_THEN_PARSE_KEYS = frozenset({"input", "expect", "expect_error"}) +_CATALOG_EXPECT_KEYS = frozenset({ + "catalog_id", + "components", + "functions", + "protocol_version", +}) + @contextlib.contextmanager def assert_raises(expect_error): if isinstance(expect_error, dict): + assert_known_keys(expect_error, _EXPECT_ERROR_KEYS, "expect_error") category = expect_error.get("category") message = expect_error.get("message", "") expected_class = CATEGORY_TO_EXCEPTION.get(category, A2uiError) @@ -93,6 +210,12 @@ def assert_raises(expect_error): if expected_details is not None: actual_details = getattr(excinfo.value, "details", []) for expected in expected_details: + if isinstance(expected, Mapping): + assert_known_keys( + expected, + _EXPECT_ERROR_DETAIL_KEYS, + "expect_error.details[]", + ) exp_path = expected["path"] exp_code = expected["code"] found = False @@ -146,6 +269,25 @@ def assert_parts_match(actual_parts, expected_parts): assert actual_dicts == expected_parts +def _case_protocol_version(case: Mapping[str, Any]) -> str: + catalog = case.get("catalog", {}) if isinstance(case.get("catalog"), dict) else {} + catalogs = ( + case.get("catalogs", []) if isinstance(case.get("catalogs"), list) else [] + ) + first_catalog = catalogs[0] if catalogs and isinstance(catalogs[0], dict) else {} + version = str( + case.get("protocolVersion") + or catalog.get("protocolVersion") + or first_catalog.get("protocolVersion") + or "v0.9" + ) + if version.startswith("0."): + return f"v{version}" + if version == "v1.0": + return "1.0" + return version + + def get_conformance_cases(filename): if filename in SKIP_TEST_SUITES or os.path.basename(filename) in SKIP_TEST_SUITES: return [] @@ -154,26 +296,7 @@ def get_conformance_cases(filename): filtered = [] for case in cases: name = case.get("name") - catalog = ( - case.get("catalog", {}) if isinstance(case.get("catalog"), dict) else {} - ) - catalogs = ( - case.get("catalogs", []) if isinstance(case.get("catalogs"), list) else [] - ) - first_catalog = ( - catalogs[0] if catalogs and isinstance(catalogs[0], dict) else {} - ) - version = str( - case.get("protocolVersion") - or catalog.get("protocolVersion") - or first_catalog.get("protocolVersion") - or "v0.9" - ) - if version.startswith("0."): - version = f"v{version}" - elif version == "v1.0": - version = "1.0" - + version = _case_protocol_version(case) if version not in SUPPORTED_PROTOCOL_VERSIONS or name in SKIP_TEST_NAMES: continue filtered.append((name, case)) @@ -398,13 +521,38 @@ def get_marked_conformance_cases(*filenames): """Loads cases from several suites, marking the ones this SDK cannot pass.""" params = [] for filename in filenames: - for name, case in get_conformance_cases(filename): - marks = [] - if name in UNSUPPORTED: - marks.append(pytest.mark.skip(reason=UNSUPPORTED[name])) - elif name in KNOWN_GAPS: - marks.append(pytest.mark.xfail(reason=KNOWN_GAPS[name], strict=True)) - params.append(pytest.param(name, case, marks=marks, id=name)) + cases = load_tests(filename) + suite_skip_reason = None + if filename in SKIP_TEST_SUITES: + suite_skip_reason = ( + SKIP_TEST_SUITES[filename] + if isinstance(SKIP_TEST_SUITES, Mapping) + else f"Suite skipped by SKIP_TEST_SUITES: {filename}" + ) + elif os.path.basename(filename) in SKIP_TEST_SUITES: + base = os.path.basename(filename) + suite_skip_reason = ( + SKIP_TEST_SUITES[base] + if isinstance(SKIP_TEST_SUITES, Mapping) + else f"Suite skipped by SKIP_TEST_SUITES: {base}" + ) + + skip: dict[str, str] = dict(UNSUPPORTED) + for case in cases: + name = str(case.get("name")) + if suite_skip_reason is not None: + skip[name] = suite_skip_reason + continue + version = _case_protocol_version(case) + if version not in SUPPORTED_PROTOCOL_VERSIONS: + skip[name] = f"Unsupported protocol version: {version}" + elif name in SKIP_TEST_NAMES: + skip[name] = ( + SKIP_TEST_NAMES[name] + if isinstance(SKIP_TEST_NAMES, Mapping) and SKIP_TEST_NAMES[name] + else f"Skipped in SKIP_TEST_NAMES: {name}" + ) + params.extend(case_params(filename, cases, skip=skip, xfail=KNOWN_GAPS)) return params @@ -414,8 +562,11 @@ def get_marked_conformance_cases(*filenames): ) -@pytest.mark.parametrize("name, test_case", cases_compiler) -def test_compiler_conformance(name, test_case): +@pytest.mark.parametrize("suite, test_case", cases_compiler) +def test_compiler_conformance(suite, test_case): + del suite + assert test_case.get("action") == "compile" + assert_known_keys(test_case, _COMPILE_CASE_KEYS, "compile case") parser = make_parser(test_case["args"]) payload = test_case["input"] @@ -443,8 +594,11 @@ def test_compiler_conformance(name, test_case): ) -@pytest.mark.parametrize("name, test_case", cases_decompiler) -def test_decompiler_conformance(name, test_case): +@pytest.mark.parametrize("suite, test_case", cases_decompiler) +def test_decompiler_conformance(suite, test_case): + del suite + assert test_case.get("action") == "decompile" + assert_known_keys(test_case, _DECOMPILE_CASE_KEYS, "decompile case") parser = make_parser(test_case["args"]) messages = test_case["messages"] @@ -534,13 +688,15 @@ def wrap_parts(parser, parts): ) -@pytest.mark.parametrize("name, test_case", cases_response_parser) -def test_response_parser_conformance(name, test_case): +@pytest.mark.parametrize("suite, test_case", cases_response_parser) +def test_response_parser_conformance(suite, test_case): + del suite args = test_case["args"] parser = make_parser(args) action = test_case["action"] if action == "unwrap": + assert_known_keys(test_case, _UNWRAP_CASE_KEYS, "unwrap case") expected = test_case["expect"] assert_raw_parts_match(parser.unwrap(test_case["input"]), expected) has_any = any("a2ui_raw" in p for p in expected) @@ -553,6 +709,9 @@ def test_response_parser_conformance(name, test_case): ) elif action == "has_format_content": + assert_known_keys( + test_case, _HAS_FORMAT_CONTENT_CASE_KEYS, "has_format_content case" + ) complete = args.get("complete", False) assert ( parser.has_format_content(test_case["input"], complete=complete) @@ -560,6 +719,7 @@ def test_response_parser_conformance(name, test_case): ) elif action == "wrap": + assert_known_keys(test_case, _WRAP_CASE_KEYS, "wrap case") parts = test_case["parts"] output = wrap_parts(parser, parts) @@ -571,6 +731,7 @@ def test_response_parser_conformance(name, test_case): assert_raw_parts_match(parser.unwrap(output), parts) elif action == "parse_response": + assert_known_keys(test_case, _PARSE_RESPONSE_CASE_KEYS, "parse_response case") kwargs: dict[str, Any] = {} if not args.get("wrapped", True): kwargs["wrapped"] = False @@ -584,7 +745,7 @@ def test_response_parser_conformance(name, test_case): assert_parts_match(parts, test_case["expect"]) else: - raise ValueError(f"Unknown response parser action: {action}") + pytest.fail(f"Unknown response parser action: {action!r}") # --- Streaming Response Conformance (Direct JSON) --- @@ -606,8 +767,11 @@ def _merge_text_parts(parts: list[dict[str, Any]]) -> list[dict[str, Any]]: return merged -@pytest.mark.parametrize("name, test_case", cases_response_streaming) -def test_response_streaming_conformance(name, test_case): +@pytest.mark.parametrize("suite, test_case", cases_response_streaming) +def test_response_streaming_conformance(suite, test_case): + del suite + assert test_case.get("action") == "parse_chunk" + assert_known_keys(test_case, _PARSE_CHUNK_CASE_KEYS, "parse_chunk case") args = test_case["args"] wrapped = args.get("wrapped", True) stream = make_parser(args).open_stream( @@ -618,6 +782,7 @@ def test_response_streaming_conformance(name, test_case): yielded: list[dict] = [] for step in test_case["steps"]: + assert_known_keys(step, _PARSE_CHUNK_STEP_KEYS, "parse_chunk step") chunk = step["input"] chunks.append(chunk) if "expect_error" in step: @@ -694,8 +859,9 @@ def _expect_snippet(snippet, contains, absent): assert fragment not in snippet, f"{fragment!r} unexpectedly in {snippet!r}" -@pytest.mark.parametrize("name, test_case", cases_multi_catalog) -def test_multi_catalog_formats_conformance(name, test_case, tmp_path): +@pytest.mark.parametrize("suite, test_case", cases_multi_catalog) +def test_multi_catalog_formats_conformance(suite, test_case, tmp_path): + del suite run_format_case(test_case, tmp_path) @@ -705,6 +871,7 @@ def run_format_case(test_case, tmp_path): args = test_case["args"] if action == "compile": + assert_known_keys(test_case, _COMPILE_CASE_KEYS, "compile case") parser = make_parser(args) if "expect_error" in test_case: with assert_raises(test_case["expect_error"]): @@ -719,6 +886,7 @@ def run_format_case(test_case, tmp_path): ) elif action == "decompile": + assert_known_keys(test_case, _DECOMPILE_CASE_KEYS, "decompile case") parser = make_parser(args) messages = test_case["messages"] if "expect_error" in test_case: @@ -741,6 +909,7 @@ def run_format_case(test_case, tmp_path): ) elif action == "parse_response": + assert_known_keys(test_case, _PARSE_RESPONSE_CASE_KEYS, "parse_response case") parser = make_parser(args) if "expect_error" in test_case: with assert_raises(test_case["expect_error"]): @@ -750,6 +919,11 @@ def run_format_case(test_case, tmp_path): assert_parts_match(parts, test_case["expect"]) elif action == "generate_prompt_snippet": + assert_known_keys( + test_case, + _GENERATE_PROMPT_SNIPPET_CASE_KEYS, + "generate_prompt_snippet case", + ) def generate(fmt): return fmt.prompt_generator.generate() @@ -774,12 +948,14 @@ def generate(fmt): assert snippet == generate(fmt) elif action == "create_format": + assert_known_keys(test_case, _CREATE_FORMAT_CASE_KEYS, "create_format case") if "expect_error" in test_case: with assert_raises(test_case["expect_error"]): make_format(args, tmp_path=tmp_path) return fmt = make_format(args, tmp_path=tmp_path) expect = test_case.get("expect", {}) + assert_known_keys(expect, _CREATE_FORMAT_EXPECT_KEYS, "create_format expect") if expect.get("parsers_are_distinct"): assert fmt.create_parser() is not fmt.create_parser() @@ -806,7 +982,7 @@ def read(parser): run_create_processor_case(test_case, tmp_path) else: - raise ValueError(f"Unknown format case action: {action}") + pytest.fail(f"Unknown format case action: {action!r}") # --- Prompt Generator Conformance --- @@ -817,8 +993,9 @@ def read(parser): ) -@pytest.mark.parametrize("name, test_case", cases_prompt_generator) -def test_prompt_generator_conformance(name, test_case, tmp_path): +@pytest.mark.parametrize("suite, test_case", cases_prompt_generator) +def test_prompt_generator_conformance(suite, test_case, tmp_path): + del suite run_format_case(test_case, tmp_path) @@ -844,6 +1021,7 @@ def _describe_configs(configs): def _expect_catalog(catalog, expected): """Checks a catalog against a case's expectations. Name lists are exhaustive.""" + assert_known_keys(expected, _CATALOG_EXPECT_KEYS, "expect.catalog") if "catalog_id" in expected: assert catalog.catalog_id == expected["catalog_id"] if "components" in expected: @@ -891,6 +1069,7 @@ def _factory_for(format_name, allowed_messages=None, progressive_keys=None): def run_create_processor_case(test_case, tmp_path): from a2ui.processor import A2uiGenerator + assert_known_keys(test_case, _CREATE_PROCESSOR_CASE_KEYS, "create_processor case") args = test_case["args"] configs = [catalog_config_from_document(entry) for entry in args["catalogs"]] before = _describe_configs(configs) @@ -936,6 +1115,7 @@ def create(): processor, snippet = create() active = processor.active_catalogs expect = test_case.get("expect", {}) + assert_known_keys(expect, _CREATE_PROCESSOR_EXPECT_KEYS, "create_processor expect") if "active_catalog_ids" in expect: assert [c.catalog_id for c in active] == expect["active_catalog_ids"] for catalog, expected in zip(active, expect.get("catalogs", [])): @@ -950,6 +1130,7 @@ def create(): parse = test_case.get("then_parse") if parse is not None: + assert_known_keys(parse, _THEN_PARSE_KEYS, "then_parse") if "expect_error" in parse: with assert_raises(parse["expect_error"]): processor.parse_response(parse["input"]) @@ -959,6 +1140,7 @@ def create(): ) -@pytest.mark.parametrize("name, test_case", cases_request_processor) -def test_request_processor_conformance(name, test_case, tmp_path): +@pytest.mark.parametrize("suite, test_case", cases_request_processor) +def test_request_processor_conformance(suite, test_case, tmp_path): + del suite run_format_case(test_case, tmp_path) diff --git a/python/a2ui_agent/tests/conformance/test_macro_conformance.py b/python/a2ui_agent/tests/conformance/test_macro_conformance.py index ef4ebfd206..2ed8db726a 100644 --- a/python/a2ui_agent/tests/conformance/test_macro_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_macro_conformance.py @@ -24,8 +24,11 @@ A2uiRecursionError, A2uiValidationError, ) + +from .conformance_helpers import assert_known_keys, case_params from .macro_suite import Case, load_cases, run_case, validate_payload +SUITE = "agent/macros/macros.yaml" CASES = load_cases() GOLDEN_CASES = [c for c in CASES if c.golden is not None] ERROR_CASES = [c for c in CASES if c.expect_error is not None] @@ -38,10 +41,21 @@ "ValidationError": A2uiValidationError, } +_EXPECT_ERROR_KEYS = frozenset({"category", "message"}) +_MACRO_EXPECT_KEYS = frozenset({ + "augmented_components", + "absent_components", + "any_component_references", + "absent_any_component_references", + "component_properties", + "protocol_version", +}) -@pytest.mark.parametrize("case", GOLDEN_CASES, ids=lambda c: c.id) -def test_matches_golden(case: Case) -> None: + +@pytest.mark.parametrize("suite, case", case_params(SUITE, GOLDEN_CASES)) +def test_matches_golden(suite: str, case: Case) -> None: """Macro expansion emits exactly the recorded wire payload.""" + del suite assert run_case(case) == case.load_golden(), case.description @@ -51,22 +65,27 @@ def test_golden_is_spec_valid(case: Case) -> None: validate_payload(case.load_golden(), case) -@pytest.mark.parametrize("case", ERROR_CASES, ids=lambda c: c.id) -def test_expect_error(case: Case) -> None: +@pytest.mark.parametrize("suite, case", case_params(SUITE, ERROR_CASES)) +def test_expect_error(suite: str, case: Case) -> None: """Error cases trigger the expected exception category.""" + del suite + assert case.expect_error is not None + assert_known_keys(case.expect_error, _EXPECT_ERROR_KEYS, "expect_error") category = case.expect_error.get("category") - expected_exc = CATEGORY_TO_EXCEPTION.get(category, A2uiError) + expected_exc = CATEGORY_TO_EXCEPTION.get(str(category), A2uiError) with pytest.raises(expected_exc): payload = run_case(case) if payload: validate_payload(payload, case) -@pytest.mark.parametrize("case", EXPECT_CASES, ids=lambda c: c.id) -def test_expect_assertion(case: Case) -> None: +@pytest.mark.parametrize("suite, case", case_params(SUITE, EXPECT_CASES)) +def test_expect_assertion(suite: str, case: Case) -> None: """Evaluates expect assertions for catalog synthesis and metadata preservation.""" + del suite result = run_case(case) expect = case.expect or {} + assert_known_keys(expect, _MACRO_EXPECT_KEYS, "macros expect") if "augmented_components" in expect: catalog_schema = result.validation_schema @@ -100,7 +119,7 @@ def test_expect_assertion(case: Case) -> None: .get("anyComponent", {}) .get("oneOf", []) ) - ref_targets = [r.get("$ref", "") for r in refs if isinstance(r, dict)] + ref_targets = [str(r.get("$ref") or "") for r in refs if isinstance(r, dict)] for name in expect["absent_any_component_references"]: assert not any( r.endswith(f"/{name}") for r in ref_targets @@ -123,9 +142,10 @@ def test_expect_assertion(case: Case) -> None: assert result.protocol_version == expect["protocol_version"] -@pytest.mark.parametrize("case", PASSTHROUGH_CASES, ids=lambda c: c.id) -def test_reverse_passthrough(case: Case) -> None: +@pytest.mark.parametrize("suite, case", case_params(SUITE, PASSTHROUGH_CASES)) +def test_reverse_passthrough(suite: str, case: Case) -> None: """Verifies transform_to_inference returns messages untouched.""" + del suite result = run_case(case) assert len(result) == 1 assert result[0]["updateComponents"]["components"] == [case.input] diff --git a/python/a2ui_agent/tests/conformance/test_scaling_conformance.py b/python/a2ui_agent/tests/conformance/test_scaling_conformance.py index 6b754a5fe7..4cc130746f 100644 --- a/python/a2ui_agent/tests/conformance/test_scaling_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_scaling_conformance.py @@ -42,14 +42,16 @@ from a2ui.inference_format import InferenceFormat from a2ui.utils import RendererState -from .conformance_helpers import load_conformance_yaml +from .conformance_helpers import case_params, load_conformance_yaml from .test_conformance import _catalogs_for, _format_for +SCALING_SKIP_REASON = "timing suite, run by pytest -m performance" + _SUITES = ("agent/direct_json/scaling.yaml", "agent/express/scaling.yaml") _CASES = [ - pytest.param(case, id=case["name"]) + param for suite in _SUITES - for case in load_conformance_yaml(suite) + for param in case_params(suite, load_conformance_yaml(suite)) ] @@ -145,7 +147,8 @@ def test_scaling_cases_have_handlers() -> None: """Every timing case is one `test_scaling_conformance` can run.""" assert _CASES, "No timing cases found under conformance/agent." for param in _CASES: - case = param.values[0] + _, case = param.values + assert isinstance(case, dict) assert case["action"] in ("measure_growth", "measure_overhead"), case["name"] operations = [case["operation"], case.get("baseline", "compile")] assert set(operations) <= {"parse_chunk", "compile"}, case["name"] @@ -153,8 +156,17 @@ def test_scaling_cases_have_handlers() -> None: @pytest.mark.performance -@pytest.mark.parametrize("case", _CASES) -def test_scaling_conformance(case: dict[str, Any], timing: Any) -> None: +@pytest.mark.parametrize("suite, case", _CASES) +def test_scaling_conformance( + suite: str, + case: dict[str, Any], + timing: Any, + request: pytest.FixtureRequest, +) -> None: + del suite + markexpr = (request.config.option.markexpr or "").strip() + if "not performance" in markexpr or "performance" not in markexpr: + pytest.skip(SCALING_SKIP_REASON) if case["action"] == "measure_growth": timing.measure_growth(case, _prepare(case)) return diff --git a/python/a2ui_agent/tests/conformance/test_skill_conformance.py b/python/a2ui_agent/tests/conformance/test_skill_conformance.py index cbe522b13c..a464da34d9 100644 --- a/python/a2ui_agent/tests/conformance/test_skill_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_skill_conformance.py @@ -14,11 +14,13 @@ """Conformance tests for A2UI Skill and SkillSet APIs driven by conformance/agent/skill.yaml.""" +from __future__ import annotations + import os import tempfile -import unittest +from typing import Any -import yaml +import pytest from a2ui.core import A2uiValidationError from a2ui.inference_formats.experimental.atom import AtomFormat @@ -28,168 +30,179 @@ from a2ui.skill import Skill, SkillGenerator, SkillSet from ..repo_paths import find_repo_root +from .conformance_helpers import ( + assert_known_keys, + case_params, + load_conformance_yaml, +) REPO_ROOT = find_repo_root(os.path.dirname(__file__)) or "" -SPEC_YAML_PATH = os.path.join(REPO_ROOT, "conformance", "agent", "skill.yaml") +SUITE = "agent/skill.yaml" GOLDENS_DIR = os.path.join(REPO_ROOT, "conformance", "test_data", "skills") _ERROR_CATEGORIES: dict[str, type[Exception]] = { "ValidationError": A2uiValidationError, } - -class TestSkillConformance(unittest.TestCase): - """Verifies skill generation and decomposed prompt APIs against specification YAML and golden files.""" - - def test_prompt_generator_base_rules_conformance(self): - """Asserts prompt_generator.generate_base_rules() matches golden file exactly.""" - cat_path = os.path.join(REPO_ROOT, "catalogs", "basic", "v1", "catalog.json") - cat_config = CatalogConfig.from_path(cat_path) - catalog = cat_config.transformed_catalog - express_fmt = ExpressFormat([catalog]) - prompt_gen = express_fmt.prompt_generator - - golden_path = os.path.join(GOLDENS_DIR, "express_base_rules.txt") - with open(golden_path, "r", encoding="utf-8") as f: - expected = f.read() - - actual = prompt_gen.generate_base_rules() - self.assertEqual(actual, expected) - - def test_prompt_generator_catalog_instructions_conformance(self): - """Asserts prompt_generator.generate_catalog_instructions() matches golden file exactly.""" - cat_path = os.path.join(REPO_ROOT, "catalogs", "basic", "v1", "catalog.json") - cat_config = CatalogConfig.from_path(cat_path) - catalog = cat_config.transformed_catalog - express_fmt = ExpressFormat([catalog]) - prompt_gen = express_fmt.prompt_generator - - golden_path = os.path.join(GOLDENS_DIR, "express_catalog_instructions.txt") - with open(golden_path, "r", encoding="utf-8") as f: - expected = f.read() - - actual = prompt_gen.generate_catalog_instructions(catalog=catalog) - self.assertEqual(actual, expected) - - def test_skill_yaml_conformance_cases(self): - """Parses conformance/agent/skill.yaml and executes declared test cases.""" - self.assertTrue( - os.path.exists(SPEC_YAML_PATH), - f"Missing conformance spec: {SPEC_YAML_PATH}", - ) - - with open(SPEC_YAML_PATH, "r", encoding="utf-8") as f: - test_cases = yaml.safe_load(f) - - for case in test_cases: - name = case["name"] - action = case["action"] - args = case.get("args", {}) - - with self.subTest(name=name): - if action == "export_to_directory": - self._run_export_to_directory_case(case, args) - continue - - cat_rel_path = args.get("catalog") - catalog = None - if cat_rel_path: - abs_cat_path = os.path.join(REPO_ROOT, cat_rel_path) - cat_config = CatalogConfig.from_path(abs_cat_path) - catalog = cat_config.transformed_catalog - - fmt_name = args.get("format", "express") - if fmt_name == "express": - fmt = ExpressFormat([catalog]) if catalog else ExpressFormat() - elif fmt_name == "atom": - fmt = AtomFormat([catalog]) if catalog else AtomFormat() - elif fmt_name == "elemental": - fmt = ElementalFormat([catalog]) if catalog else ElementalFormat() - else: - raise ValueError(f"Unsupported format: {fmt_name}") - - generator = SkillGenerator(fmt) - - if action == "from_format": - mono_name = args.get("name", "a2ui") - skill_obj = generator.generate_skill(name=mono_name) - expected_rel = case["expected_file"] - expected_abs = os.path.join(REPO_ROOT, expected_rel) - with open(expected_abs, "r", encoding="utf-8") as gf: - expected_content = gf.read() - self.assertEqual(skill_obj.to_markdown(), expected_content) - - elif action == "core_syntax": - core_name = args.get("name", "a2ui-core") - skill_obj = generator.generate_core_skill(name=core_name) - expected_rel = case["expected_file"] - expected_abs = os.path.join(REPO_ROOT, expected_rel) - with open(expected_abs, "r", encoding="utf-8") as gf: - expected_content = gf.read() - self.assertEqual(skill_obj.to_markdown(), expected_content) - - elif action == "from_catalog": - skill_obj = generator.generate_catalog_skill(catalog) - expected_rel = case["expected_file"] - expected_abs = os.path.join(REPO_ROOT, expected_rel) - with open(expected_abs, "r", encoding="utf-8") as gf: - expected_content = gf.read() - self.assertEqual(skill_obj.to_markdown(), expected_content) - - elif action == "skill_set": - skill_set = generator.generate_skillset() - skills_by_file = skill_set.to_dict() - for k in case.get("expected_skills", []): - self.assertIn(k, skills_by_file) - for k, golden_rel in case.get("expected_skill_files", {}).items(): - with open( - os.path.join(REPO_ROOT, golden_rel), "r", encoding="utf-8" - ) as gf: - self.assertEqual(skills_by_file[k], gf.read(), k) - - else: - self.fail(f"Unhandled conformance action: {action}") - - def _run_export_to_directory_case(self, case: dict, args: dict) -> None: - """Exports literal skills into a temp dir and checks what survives.""" - with tempfile.TemporaryDirectory(prefix="a2ui-skill-export-") as temp_dir: - for rel in args.get("pre_existing_files", []): - abs_path = os.path.join(temp_dir, rel) - os.makedirs(os.path.dirname(abs_path), exist_ok=True) - with open(abs_path, "w", encoding="utf-8") as f: - f.write("pre-existing") - - def run() -> None: - skill_set = SkillSet() - for spec in args.get("skills", []): - skill_set.add( - Skill( - name=spec["name"], - description=spec["description"], - content=spec["content"], - filename=spec.get("filename"), - ) +_CASE_METADATA_KEYS = frozenset({"name", "description", "features", "action", "args"}) +_SINGLE_FILE_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({"expected_file"}) +_SKILL_SET_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "expected_skills", + "expected_skill_files", +}) +_EXPORT_CASE_KEYS = _CASE_METADATA_KEYS | frozenset({ + "expect_error", + "expected_files", + "expected_absent_files", +}) +_EXPECT_ERROR_KEYS = frozenset({"category", "message"}) + +SKILL_CASES = load_conformance_yaml(SUITE) + + +def test_prompt_generator_base_rules_conformance() -> None: + """Asserts prompt_generator.generate_base_rules() matches golden file exactly.""" + cat_path = os.path.join(REPO_ROOT, "catalogs", "basic", "v1", "catalog.json") + cat_config = CatalogConfig.from_path(cat_path) + catalog = cat_config.transformed_catalog + express_fmt = ExpressFormat([catalog]) + prompt_gen = express_fmt.prompt_generator + + golden_path = os.path.join(GOLDENS_DIR, "express_base_rules.txt") + with open(golden_path, "r", encoding="utf-8") as f: + expected = f.read() + + actual = prompt_gen.generate_base_rules() + assert actual == expected + + +def test_prompt_generator_catalog_instructions_conformance() -> None: + """Asserts prompt_generator.generate_catalog_instructions() matches golden file exactly.""" + cat_path = os.path.join(REPO_ROOT, "catalogs", "basic", "v1", "catalog.json") + cat_config = CatalogConfig.from_path(cat_path) + catalog = cat_config.transformed_catalog + express_fmt = ExpressFormat([catalog]) + prompt_gen = express_fmt.prompt_generator + + golden_path = os.path.join(GOLDENS_DIR, "express_catalog_instructions.txt") + with open(golden_path, "r", encoding="utf-8") as f: + expected = f.read() + + actual = prompt_gen.generate_catalog_instructions(catalog=catalog) + assert actual == expected + + +def _run_export_to_directory_case(case: dict[str, Any], args: dict[str, Any]) -> None: + """Exports literal skills into a temp dir and checks what survives.""" + assert_known_keys(case, _EXPORT_CASE_KEYS, "export_to_directory case") + with tempfile.TemporaryDirectory(prefix="a2ui-skill-export-") as temp_dir: + for rel in args.get("pre_existing_files", []): + abs_path = os.path.join(temp_dir, rel) + os.makedirs(os.path.dirname(abs_path), exist_ok=True) + with open(abs_path, "w", encoding="utf-8") as f: + f.write("pre-existing") + + def run() -> None: + skill_set = SkillSet() + for spec in args.get("skills", []): + skill_set.add( + Skill( + name=spec["name"], + description=spec["description"], + content=spec["content"], + filename=spec.get("filename"), ) - skill_set.export_to_directory(temp_dir) - - expect_error = case.get("expect_error") - if expect_error: - with self.assertRaises(_ERROR_CATEGORIES[expect_error["category"]]): - run() - else: - run() - - for rel in case.get("expected_files", []): - self.assertTrue( - os.path.exists(os.path.join(temp_dir, rel)), - f"expected {rel} to exist", - ) - for rel in case.get("expected_absent_files", []): - self.assertFalse( - os.path.exists(os.path.join(temp_dir, rel)), - f"expected {rel} to be removed", ) + skill_set.export_to_directory(temp_dir) + expect_error = case.get("expect_error") + if expect_error is not None: + assert_known_keys(expect_error, _EXPECT_ERROR_KEYS, "expect_error") + with pytest.raises(_ERROR_CATEGORIES[expect_error["category"]]): + run() + else: + run() + + for rel in case.get("expected_files", []): + assert os.path.exists( + os.path.join(temp_dir, rel) + ), f"expected {rel} to exist" + for rel in case.get("expected_absent_files", []): + assert not os.path.exists( + os.path.join(temp_dir, rel) + ), f"expected {rel} to be removed" + + +@pytest.mark.parametrize("suite, case", case_params(SUITE, SKILL_CASES)) +def test_skill_yaml_conformance_cases(suite: str, case: dict[str, Any]) -> None: + """Executes one declared test case from conformance/agent/skill.yaml.""" + del suite + action = case["action"] + args = case.get("args", {}) + + if action == "export_to_directory": + _run_export_to_directory_case(case, args) + return + + cat_rel_path = args.get("catalog") + catalog = None + if cat_rel_path: + abs_cat_path = os.path.join(REPO_ROOT, cat_rel_path) + cat_config = CatalogConfig.from_path(abs_cat_path) + catalog = cat_config.transformed_catalog -if __name__ == "__main__": - unittest.main() + fmt_name = args.get("format", "express") + catalogs = [catalog] if catalog else [] + if fmt_name == "express": + fmt: Any = ExpressFormat(catalogs) + elif fmt_name == "atom": + fmt = AtomFormat(catalogs) + elif fmt_name == "elemental": + fmt = ElementalFormat(catalogs) + else: + raise ValueError(f"Unsupported format: {fmt_name}") + + generator = SkillGenerator(fmt) + + if action == "from_format": + assert_known_keys(case, _SINGLE_FILE_CASE_KEYS, "from_format case") + mono_name = args.get("name", "a2ui") + skill_obj = generator.generate_skill(name=mono_name) + expected_rel = case["expected_file"] + expected_abs = os.path.join(REPO_ROOT, expected_rel) + with open(expected_abs, "r", encoding="utf-8") as gf: + expected_content = gf.read() + assert skill_obj.to_markdown() == expected_content + + elif action == "core_syntax": + assert_known_keys(case, _SINGLE_FILE_CASE_KEYS, "core_syntax case") + core_name = args.get("name", "a2ui-core") + skill_obj = generator.generate_core_skill(name=core_name) + expected_rel = case["expected_file"] + expected_abs = os.path.join(REPO_ROOT, expected_rel) + with open(expected_abs, "r", encoding="utf-8") as gf: + expected_content = gf.read() + assert skill_obj.to_markdown() == expected_content + + elif action == "from_catalog": + assert_known_keys(case, _SINGLE_FILE_CASE_KEYS, "from_catalog case") + assert catalog is not None + skill_obj = generator.generate_catalog_skill(catalog) + expected_rel = case["expected_file"] + expected_abs = os.path.join(REPO_ROOT, expected_rel) + with open(expected_abs, "r", encoding="utf-8") as gf: + expected_content = gf.read() + assert skill_obj.to_markdown() == expected_content + + elif action == "skill_set": + assert_known_keys(case, _SKILL_SET_CASE_KEYS, "skill_set case") + skill_set = generator.generate_skillset() + skills_by_file = skill_set.to_dict() + for k in case.get("expected_skills", []): + assert k in skills_by_file + for k, golden_rel in case.get("expected_skill_files", {}).items(): + with open(os.path.join(REPO_ROOT, golden_rel), "r", encoding="utf-8") as gf: + assert skills_by_file[k] == gf.read(), k + + else: + pytest.fail(f"Unhandled conformance action: {action!r}") diff --git a/python/a2ui_agent/tests/conformance/test_unsupported_suites.py b/python/a2ui_agent/tests/conformance/test_unsupported_suites.py new file mode 100644 index 0000000000..fa7ac4f72f --- /dev/null +++ b/python/a2ui_agent/tests/conformance/test_unsupported_suites.py @@ -0,0 +1,106 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Records reasoned skips for `conformance/agent/**/*.yaml` suites not run by Python.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from .conformance_helpers import ( + case_params, + get_conformance_path, + load_conformance_yaml, +) + +ACTIVE_SUITES: frozenset[str] = frozenset({ + "agent/builder/builder.yaml", + "agent/catalog_provider.yaml", + "agent/catalog_resolution.yaml", + "agent/catalog_transformer.yaml", + "agent/direct_json/compiler.yaml", + "agent/direct_json/decompiler.yaml", + "agent/direct_json/prompt_generator.yaml", + "agent/direct_json/response_parser.yaml", + "agent/direct_json/response_streaming.yaml", + "agent/direct_json/scaling.yaml", + "agent/express/compiler.yaml", + "agent/express/decompiler.yaml", + "agent/express/prompt_generator.yaml", + "agent/express/response_parser.yaml", + "agent/express/response_streaming.yaml", + "agent/express/scaling.yaml", + "agent/macros/macros.yaml", + "agent/multi_catalog_formats.yaml", + "agent/request_processor.yaml", + "agent/skill.yaml", +}) + +UNSUPPORTED_SUITES: dict[str, str] = { + "agent/streamed_answer.yaml": ( + "streamed_answer covers the optional agent_renderer_app feature; the" + " Python Agent SDK has no renderer and does not implement" + " StreamedAnswer." + ), + "agent/legacy/inference_format.yaml": ( + "Pre-blueprint legacy suite superseded by the current agent suites." + ), + "agent/legacy/parser.yaml": ( + "Pre-blueprint legacy suite superseded by the current agent suites." + ), + "agent/legacy/streaming_parser.yaml": ( + "Pre-blueprint legacy suite superseded by" + " agent/direct_json/response_streaming.yaml and" + " agent/express/response_streaming.yaml." + ), +} + + +def _build_unsupported_params() -> list[Any]: + params: list[Any] = [] + for suite, reason in UNSUPPORTED_SUITES.items(): + raw = load_conformance_yaml(suite) + cases = raw["tests"] if isinstance(raw, dict) and "tests" in raw else raw + skip_map = { + str(item.get("id") or item.get("name")): reason + for item in cases + if isinstance(item, dict) and (item.get("id") or item.get("name")) + } + params.extend(case_params(suite, cases, skip=skip_map)) + return params + + +_UNSUPPORTED_CASES = _build_unsupported_params() + + +def test_all_agent_suites_are_accounted_for() -> None: + """Every `conformance/agent/**/*.yaml` file is in ACTIVE_SUITES or UNSUPPORTED_SUITES.""" + conformance_root = Path(get_conformance_path("")) + on_disk = { + path.relative_to(conformance_root).as_posix() + for path in (conformance_root / "agent").rglob("*.yaml") + if path.is_file() + } + unsupported = set(UNSUPPORTED_SUITES) + assert not (ACTIVE_SUITES & unsupported) + assert ACTIVE_SUITES | unsupported == on_disk + + +@pytest.mark.parametrize("suite, case", _UNSUPPORTED_CASES) +def test_unsupported_agent_suite_case(suite: str, case: dict[str, Any]) -> None: + """Placeholder reached only if a skip marker was omitted.""" + pytest.fail(f"Expected {suite}#{case.get('name')} to be skipped") From b12b6f44c708de98e7024b36cf9f99d5d3ada608 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 19:23:40 +0000 Subject: [PATCH 28/48] test(a2ui_a2a): validate v1.0 AgentCard params against agent_capabilities.json Validate that get_a2ui_agent_extension emits v1.0 AgentCard extension parameters matching the inner v1.0 object schema in specification/v1_0/json/agent_capabilities.json and remains flat without a redundant version key. --- python/a2ui_a2a/tests/test_extension.py | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/python/a2ui_a2a/tests/test_extension.py b/python/a2ui_a2a/tests/test_extension.py index 76aaa43cc9..d17886535c 100644 --- a/python/a2ui_a2a/tests/test_extension.py +++ b/python/a2ui_a2a/tests/test_extension.py @@ -14,9 +14,12 @@ """Tests that the A2A helpers accept every protocol version in every form.""" +import json +from pathlib import Path from unittest.mock import MagicMock from a2a.server.agent_execution import RequestContext +import jsonschema import pytest from a2ui.a2a import ( @@ -99,3 +102,59 @@ def test_activation_skips_versions_this_sdk_does_not_define(): version, context = _activate([known, unknown], [known, unknown]) assert version is ProtocolVersion.V1_0 context.add_activated_extension.assert_called_once_with(known) + + +_AGENT_CAPABILITIES_SCHEMA_PATH = ( + Path(__file__).resolve().parents[3] + / "specification" + / "v1_0" + / "json" + / "agent_capabilities.json" +) + + +@pytest.mark.parametrize( + "accepts_inline_catalogs, supported_catalog_ids", + [ + (False, []), + (True, []), + (False, ["a"]), + (True, ["a", "b"]), + ], +) +def test_v1_0_params_match_agent_capabilities_schema( + accepts_inline_catalogs: bool, + supported_catalog_ids: list[str], +) -> None: + schema = json.loads(_AGENT_CAPABILITIES_SCHEMA_PATH.read_text(encoding="utf-8")) + v1_0_schema = dict(schema["properties"]["1.0"]) + if not supported_catalog_ids: + v1_0_schema.pop("required", None) + + ext = get_a2ui_agent_extension( + ProtocolVersion.V1_0, + accepts_inline_catalogs=accepts_inline_catalogs, + supported_catalog_ids=supported_catalog_ids, + ) + jsonschema.Draft202012Validator(v1_0_schema).validate(ext.params or {}) + + +@pytest.mark.parametrize( + "accepts_inline_catalogs, supported_catalog_ids", + [ + (False, []), + (True, []), + (False, ["a"]), + (True, ["a", "b"]), + ], +) +def test_v1_0_params_are_not_version_keyed( + accepts_inline_catalogs: bool, + supported_catalog_ids: list[str], +) -> None: + ext = get_a2ui_agent_extension( + ProtocolVersion.V1_0, + accepts_inline_catalogs=accepts_inline_catalogs, + supported_catalog_ids=supported_catalog_ids, + ) + assert "1.0" not in (ext.params or {}) From 32f7167aa1068b421a68f1e65b58b654fa82f35b Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 19:49:31 +0000 Subject: [PATCH 29/48] test(a2ui_agent): enforce protocol_version, error message, and schema property checks --- python/a2ui_a2a/tests/test_extension.py | 7 ++++--- .../conformance/test_catalog_conformance.py | 10 ++++++++-- .../tests/conformance/test_conformance.py | 20 +++++-------------- .../conformance/test_macro_conformance.py | 6 ++---- .../conformance/test_skill_conformance.py | 5 ++++- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/python/a2ui_a2a/tests/test_extension.py b/python/a2ui_a2a/tests/test_extension.py index d17886535c..461b2635da 100644 --- a/python/a2ui_a2a/tests/test_extension.py +++ b/python/a2ui_a2a/tests/test_extension.py @@ -127,9 +127,10 @@ def test_v1_0_params_match_agent_capabilities_schema( supported_catalog_ids: list[str], ) -> None: schema = json.loads(_AGENT_CAPABILITIES_SCHEMA_PATH.read_text(encoding="utf-8")) - v1_0_schema = dict(schema["properties"]["1.0"]) - if not supported_catalog_ids: - v1_0_schema.pop("required", None) + v1_0_schema = { + **schema["properties"]["1.0"], + "additionalProperties": False, + } ext = get_a2ui_agent_extension( ProtocolVersion.V1_0, diff --git a/python/a2ui_agent/tests/conformance/test_catalog_conformance.py b/python/a2ui_agent/tests/conformance/test_catalog_conformance.py index f9ff1cd7a8..e64a60b014 100644 --- a/python/a2ui_agent/tests/conformance/test_catalog_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_catalog_conformance.py @@ -154,7 +154,10 @@ def provide() -> CatalogApi: if "expect_error" in case: assert_known_keys(case["expect_error"], _EXPECT_ERROR_KEYS, "expect_error") - with pytest.raises(_ERROR_CATEGORIES[case["expect_error"]["category"]]): + with pytest.raises( + _ERROR_CATEGORIES[case["expect_error"]["category"]], + match=case["expect_error"].get("message"), + ): provide() return @@ -207,7 +210,10 @@ def resolve() -> list[CatalogApi]: if "expect_error" in case: assert_known_keys(case["expect_error"], _EXPECT_ERROR_KEYS, "expect_error") - with pytest.raises(_ERROR_CATEGORIES[case["expect_error"]["category"]]): + with pytest.raises( + _ERROR_CATEGORIES[case["expect_error"]["category"]], + match=case["expect_error"].get("message"), + ): resolve() return diff --git a/python/a2ui_agent/tests/conformance/test_conformance.py b/python/a2ui_agent/tests/conformance/test_conformance.py index f1f4375fee..f4efca1bfa 100644 --- a/python/a2ui_agent/tests/conformance/test_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_conformance.py @@ -288,21 +288,6 @@ def _case_protocol_version(case: Mapping[str, Any]) -> str: return version -def get_conformance_cases(filename): - if filename in SKIP_TEST_SUITES or os.path.basename(filename) in SKIP_TEST_SUITES: - return [] - - cases = load_tests(filename) - filtered = [] - for case in cases: - name = case.get("name") - version = _case_protocol_version(case) - if version not in SUPPORTED_PROTOCOL_VERSIONS or name in SKIP_TEST_NAMES: - continue - filtered.append((name, case)) - return filtered - - CONFORMANCE_SURFACE_ID = "default_surface" DEFAULT_CATALOG = "test_data/catalogs/simplified_catalog_v1_0.json" @@ -1024,6 +1009,11 @@ def _expect_catalog(catalog, expected): assert_known_keys(expected, _CATALOG_EXPECT_KEYS, "expect.catalog") if "catalog_id" in expected: assert catalog.catalog_id == expected["catalog_id"] + if "protocol_version" in expected: + assert ( + to_protocol_version(catalog.protocol_version).value + == expected["protocol_version"] + ) if "components" in expected: assert sorted(catalog.components) == sorted(set(expected["components"])) if "functions" in expected: diff --git a/python/a2ui_agent/tests/conformance/test_macro_conformance.py b/python/a2ui_agent/tests/conformance/test_macro_conformance.py index 2ed8db726a..a86264d60c 100644 --- a/python/a2ui_agent/tests/conformance/test_macro_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_macro_conformance.py @@ -20,7 +20,6 @@ from a2ui.core import ( A2uiCatalogError, - A2uiError, A2uiRecursionError, A2uiValidationError, ) @@ -71,9 +70,8 @@ def test_expect_error(suite: str, case: Case) -> None: del suite assert case.expect_error is not None assert_known_keys(case.expect_error, _EXPECT_ERROR_KEYS, "expect_error") - category = case.expect_error.get("category") - expected_exc = CATEGORY_TO_EXCEPTION.get(str(category), A2uiError) - with pytest.raises(expected_exc): + expected_exc = CATEGORY_TO_EXCEPTION[str(case.expect_error.get("category"))] + with pytest.raises(expected_exc, match=case.expect_error.get("message")): payload = run_case(case) if payload: validate_payload(payload, case) diff --git a/python/a2ui_agent/tests/conformance/test_skill_conformance.py b/python/a2ui_agent/tests/conformance/test_skill_conformance.py index a464da34d9..6249712c93 100644 --- a/python/a2ui_agent/tests/conformance/test_skill_conformance.py +++ b/python/a2ui_agent/tests/conformance/test_skill_conformance.py @@ -118,7 +118,10 @@ def run() -> None: expect_error = case.get("expect_error") if expect_error is not None: assert_known_keys(expect_error, _EXPECT_ERROR_KEYS, "expect_error") - with pytest.raises(_ERROR_CATEGORIES[expect_error["category"]]): + with pytest.raises( + _ERROR_CATEGORIES[expect_error["category"]], + match=expect_error.get("message"), + ): run() else: run() From 351c7ee32e107daa399271233a5aec117b2330c0 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 20:41:39 +0000 Subject: [PATCH 30/48] fix(swift-core): default DataContext to passive and keep event payloads unactivated --- .../A2UICore/Models/ResolvedAction.swift | 3 +- .../Sources/A2UICore/RPC/RPCHandler.swift | 6 +- .../A2UICore/Resolution/NodeResolver.swift | 17 ++-- .../Sources/A2UICore/State/DataContext.swift | 25 +++--- .../ActionsConformanceTests.swift | 3 +- .../UserActivationActionTests.swift | 86 +++++++++++-------- 6 files changed, 77 insertions(+), 63 deletions(-) diff --git a/swift/core/Sources/A2UICore/Models/ResolvedAction.swift b/swift/core/Sources/A2UICore/Models/ResolvedAction.swift index 1322b2f41d..395a6f999e 100644 --- a/swift/core/Sources/A2UICore/Models/ResolvedAction.swift +++ b/swift/core/Sources/A2UICore/Models/ResolvedAction.swift @@ -100,7 +100,7 @@ public struct ResolvedAction: Sendable { } /// Returns a copy of this resolved action configured with the given user activation state. - package func withUserActivation(_ userActivation: Bool) -> ResolvedAction { + public func withUserActivation(_ userActivation: Bool) -> ResolvedAction { ResolvedAction( identity: identity, sourceComponentID: sourceComponentID, @@ -130,6 +130,7 @@ extension ResolvedAction: Equatable { lhs.identity == rhs.identity && lhs.dataContextPath == rhs.dataContextPath && lhs.rawUserMessage == rhs.rawUserMessage + && lhs.userActivation == rhs.userActivation && MetadataValidator.areEqual(lhs.metadata, rhs.metadata) } } diff --git a/swift/core/Sources/A2UICore/RPC/RPCHandler.swift b/swift/core/Sources/A2UICore/RPC/RPCHandler.swift index 0cecab8f8b..889b2565bd 100644 --- a/swift/core/Sources/A2UICore/RPC/RPCHandler.swift +++ b/swift/core/Sources/A2UICore/RPC/RPCHandler.swift @@ -153,7 +153,8 @@ public final class RPCHandler { path: dataContext?.path ?? "", functionHandler: argHandler, protocolVersion: dataContext?.protocolVersion ?? message.version, - index: dataContext?.index + index: dataContext?.index, + userActivation: userActivationPresent ) let effectiveContext = dataContext @@ -161,7 +162,8 @@ public final class RPCHandler { dataModel: argResolutionContext.dataModel, path: "", functionHandler: argHandler, - protocolVersion: message.version + protocolVersion: message.version, + userActivation: userActivationPresent ) var resolvedArgs: [String: JSONValue] = [:] diff --git a/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift b/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift index ed3bdf6362..68809763ed 100644 --- a/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift +++ b/swift/core/Sources/A2UICore/Resolution/NodeResolver.swift @@ -1609,7 +1609,7 @@ public final class NodeResolver: Sendable { dataContextPath: basePath, rawUserMessage: userMessageValue, metadata: actionMetadata, - triggerWithUserActivation: { [weak self] userActivation, callMetadata in + triggerWithUserActivation: { [weak self] _, callMetadata in guard let self else { return } guard self.reportFailedActionChecksIfNeeded(checks, componentID: componentID) else { return @@ -1617,16 +1617,10 @@ public final class NodeResolver: Sendable { let resolvedContext = self.evaluateActionContext( contextDictionary, basePath: basePath, - index: index, - userActivation: userActivation + index: index ) let resolvedUserMessage = userMessageValue.flatMap { - self.evaluateDynamicValue( - $0, - basePath: basePath, - index: index, - userActivation: userActivation - ).stringValue + self.evaluateDynamicValue($0, basePath: basePath, index: index).stringValue } let validatedBase = try MetadataValidator.validate( rawMetadata, @@ -1706,12 +1700,11 @@ public final class NodeResolver: Sendable { private func evaluateActionContext( _ contextDictionary: [String: JSONValue]?, basePath: String?, - index: Int?, - userActivation: Bool = false + index: Int? ) -> [String: JSONValue] { guard let contextDictionary else { return [:] } return contextDictionary.mapValues { - evaluateDynamicValue($0, basePath: basePath, index: index, userActivation: userActivation) + evaluateDynamicValue($0, basePath: basePath, index: index) } } diff --git a/swift/core/Sources/A2UICore/State/DataContext.swift b/swift/core/Sources/A2UICore/State/DataContext.swift index 01fe45937e..e5879d4c6c 100644 --- a/swift/core/Sources/A2UICore/State/DataContext.swift +++ b/swift/core/Sources/A2UICore/State/DataContext.swift @@ -30,20 +30,22 @@ public final class DataContext { /// A reference to the function handler to evaluate dynamic function calls. public weak var functionHandler: FunctionHandler? - public init( + public convenience init( dataModel: DataModel, path: String, functionHandler: FunctionHandler, protocolVersion: A2UIProtocolVersion? = nil, index: Int? = nil, - userActivation: Bool = true + userActivation: Bool = false ) { - self.dataModel = dataModel - self.path = path - self.functionHandler = functionHandler - self.protocolVersion = protocolVersion - self.index = index - self.userActivation = userActivation + self.init( + dataModel: dataModel, + path: path, + optionalFunctionHandler: functionHandler, + protocolVersion: protocolVersion, + index: index, + userActivation: userActivation + ) } private init( @@ -52,7 +54,7 @@ public final class DataContext { optionalFunctionHandler: FunctionHandler?, protocolVersion: A2UIProtocolVersion?, index: Int?, - userActivation: Bool + userActivation: Bool = false ) { self.dataModel = dataModel self.path = path @@ -191,7 +193,7 @@ public final class DataContext { return .null } - if function.api.requiresUserActivation && !userActivation { + guard !function.api.requiresUserActivation || userActivation else { functionHandler?.handleFunctionError( FunctionError.userActivationRequired(name: callName), functionName: callName @@ -206,7 +208,8 @@ public final class DataContext { for (argKey, argVal) in argsObj { let resolvedVal = resolveDynamicValue(argVal) if let propSchema = argPropsSchema?[argKey] { - resolvedArgs[argKey] = Self.coerceValidationResults(schema: propSchema, value: resolvedVal) + resolvedArgs[argKey] = Self.coerceValidationResults( + schema: propSchema, value: resolvedVal) } else { resolvedArgs[argKey] = resolvedVal } diff --git a/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift index ac504b2735..325c17c806 100644 --- a/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ActionsConformanceTests.swift @@ -141,7 +141,8 @@ struct ActionsConformanceTests { path: scope, functionHandler: handler, protocolVersion: protocolVersion, - index: index + index: index, + userActivation: userActivation ) let rawAction = diff --git a/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift b/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift index 98bfad759a..7d58586946 100644 --- a/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift +++ b/swift/core/Tests/A2UICoreTests/UserActivationActionTests.swift @@ -116,11 +116,11 @@ struct UserActivationActionTests { } @Test func gatedActionRefusedWithoutUserActivation() throws { - let gatedFn = try SpyUserActivationFunction( + let gatedFunction = try SpyUserActivationFunction( name: "openExternal", requiresUserActivation: true ) - let catalog = try Self.makeButtonCatalog(functions: [gatedFn]) + let catalog = try Self.makeButtonCatalog(functions: [gatedFunction]) let actionHandler = TestProcessorActionHandler() let resolver = Self.makeButtonResolver( functionName: "openExternal", @@ -132,15 +132,15 @@ struct UserActivationActionTests { let programmaticAction = try #require(root.action(for: "action", userActivation: false)) programmaticAction() - #expect(gatedFn.callCount == 0) + #expect(gatedFunction.callCount == 0) #expect(actionHandler.capturedActions.isEmpty) #expect(actionHandler.capturedErrors.count == 1) - if case .generic(let err) = actionHandler.capturedErrors.first { - #expect(err.code == "EXPRESSION_ERROR") - #expect(err.surfaceID == "main") - #expect(err.expression == "openExternal") + if case .generic(let error) = actionHandler.capturedErrors.first { + #expect(error.code == "EXPRESSION_ERROR") + #expect(error.surfaceID == "main") + #expect(error.expression == "openExternal") #expect( - err.message == "Function 'openExternal' requires user activation context to execute." + error.message == "Function 'openExternal' requires user activation context to execute." ) } else { Issue.record("Expected EXPRESSION_ERROR generic error, got \(actionHandler.capturedErrors)") @@ -148,11 +148,11 @@ struct UserActivationActionTests { } @Test func gatedActionRunsWithUserActivation() throws { - let gatedFn = try SpyUserActivationFunction( + let gatedFunction = try SpyUserActivationFunction( name: "openExternal", requiresUserActivation: true ) - let catalog = try Self.makeButtonCatalog(functions: [gatedFn]) + let catalog = try Self.makeButtonCatalog(functions: [gatedFunction]) let actionHandler = TestProcessorActionHandler() let resolver = Self.makeButtonResolver( functionName: "openExternal", @@ -162,25 +162,28 @@ struct UserActivationActionTests { let root = try #require(resolver.resolveTree()) let gestureAction = try #require(root.action(for: "action", userActivation: true)) + let unactivatedAction = try #require(root.action(for: "action", userActivation: false)) + #expect(gestureAction != unactivatedAction) + #expect(gestureAction.withUserActivation(false) == unactivatedAction) gestureAction() - #expect(gatedFn.callCount == 1) + #expect(gatedFunction.callCount == 1) #expect(actionHandler.capturedActions.isEmpty) #expect(actionHandler.capturedErrors.isEmpty) let legacyAction = try #require(root.action(for: "action")) legacyAction() - #expect(gatedFn.callCount == 2) + #expect(gatedFunction.callCount == 2) #expect(actionHandler.capturedErrors.isEmpty) } @Test func ungatedActionRunsWithOrWithoutUserActivation() throws { - let ungatedFn = try SpyUserActivationFunction( + let ungatedFunction = try SpyUserActivationFunction( name: "increment", requiresUserActivation: false ) - let catalog = try Self.makeButtonCatalog(functions: [ungatedFn]) + let catalog = try Self.makeButtonCatalog(functions: [ungatedFunction]) let actionHandler = TestProcessorActionHandler() let resolver = Self.makeButtonResolver( functionName: "increment", @@ -191,23 +194,23 @@ struct UserActivationActionTests { let root = try #require(resolver.resolveTree()) let withoutActivation = try #require(root.action(for: "action", userActivation: false)) withoutActivation() - #expect(ungatedFn.callCount == 1) + #expect(ungatedFunction.callCount == 1) #expect(actionHandler.capturedErrors.isEmpty) let withActivation = try #require(root.action(for: "action", userActivation: true)) withActivation() - #expect(ungatedFn.callCount == 2) + #expect(ungatedFunction.callCount == 2) #expect(actionHandler.capturedErrors.isEmpty) } @Test func dataContextRefusesGatedFunctionWithoutUserActivation() throws { - let gatedFn = try SpyUserActivationFunction( + let gatedFunction = try SpyUserActivationFunction( name: "openExternal", requiresUserActivation: true ) - let handler = RecordingFunctionHandler(functions: [gatedFn]) + let handler = RecordingFunctionHandler(functions: [gatedFunction]) let dataModel = DataModel() - let callExpr: JSONValue = .object([ + let callExpression: JSONValue = .object([ "@call": .string("openExternal"), "args": .object([:]), ]) @@ -216,12 +219,12 @@ struct UserActivationActionTests { dataModel: dataModel, path: "/", functionHandler: handler, - protocolVersion: .v10, - userActivation: false + protocolVersion: .v10 ) - let deniedResult = inactiveContext.resolveDynamicValue(callExpr) + #expect(inactiveContext.userActivation == false) + let deniedResult = inactiveContext.resolveDynamicValue(callExpression) #expect(deniedResult == .null) - #expect(gatedFn.callCount == 0) + #expect(gatedFunction.callCount == 0) #expect(handler.recordedErrors.count == 1) let recordedError = try #require(handler.recordedErrors.first?.error as? FunctionError) @@ -239,22 +242,22 @@ struct UserActivationActionTests { protocolVersion: .v10, userActivation: true ) - let overrideDenied = activeContext.resolveDynamicValue(callExpr, userActivation: false) + let overrideDenied = activeContext.resolveDynamicValue(callExpression, userActivation: false) #expect(overrideDenied == .null) - #expect(gatedFn.callCount == 0) + #expect(gatedFunction.callCount == 0) #expect(handler.recordedErrors.count == 2) - let allowedResult = activeContext.resolveDynamicValue(callExpr) + let allowedResult = activeContext.resolveDynamicValue(callExpression) #expect(allowedResult == .string("ran-openExternal")) - #expect(gatedFn.callCount == 1) + #expect(gatedFunction.callCount == 1) } @Test func passiveBindingsAndChecksRefuseGatedFunctions() throws { - let gatedFn = try SpyUserActivationFunction( + let gatedFunction = try SpyUserActivationFunction( name: "openExternal", requiresUserActivation: true ) - let catalog = try Self.makeButtonCatalog(functions: [gatedFn]) + let catalog = try Self.makeButtonCatalog(functions: [gatedFunction]) let actionHandler = TestProcessorActionHandler() let resolver = NodeResolver( surfaceID: "main", @@ -263,7 +266,7 @@ struct UserActivationActionTests { actionHandler: actionHandler, protocolVersion: .v10 ) - let callExpr: JSONValue = .object([ + let callExpression: JSONValue = .object([ "@call": .string("openExternal"), "args": .object([:]), ]) @@ -272,10 +275,10 @@ struct UserActivationActionTests { id: "root", type: "Button", properties: [ - "label": callExpr, + "label": callExpression, "checks": .array([ .object([ - "condition": callExpr, + "condition": callExpression, "message": .string("Blocked check"), ]) ]), @@ -286,7 +289,7 @@ struct UserActivationActionTests { let root = try #require(resolver.resolveTree()) #expect(root.string(for: "label") == nil) #expect(root.isValid == false) - #expect(gatedFn.callCount == 0) + #expect(gatedFunction.callCount == 0) #expect(actionHandler.capturedErrors.count == 2) resolver.componentsModel.addComponent( @@ -297,7 +300,7 @@ struct UserActivationActionTests { "action": .object([ "event": .object([ "name": .string("submit"), - "context": .object(["urlResult": callExpr]), + "context": .object(["secret": callExpression]), ]) ]) ] @@ -307,11 +310,22 @@ struct UserActivationActionTests { let eventRoot = try #require(resolver.resolveTree()) let programmaticEvent = try #require(eventRoot.action(for: "action", userActivation: false)) programmaticEvent() - #expect(gatedFn.callCount == 0) + #expect(gatedFunction.callCount == 0) #expect(actionHandler.capturedErrors.count == 3) + if case .event(_, let context, _) = actionHandler.capturedActions.last?.identity { + #expect(context?["secret"] == .null) + } else { + Issue.record("Expected dispatched event action") + } let gestureEvent = try #require(eventRoot.action(for: "action", userActivation: true)) gestureEvent() - #expect(gatedFn.callCount == 1) + #expect(gatedFunction.callCount == 0) + #expect(actionHandler.capturedErrors.count == 4) + if case .event(_, let context, _) = actionHandler.capturedActions.last?.identity { + #expect(context?["secret"] == .null) + } else { + Issue.record("Expected dispatched event action") + } } } From dcafe14ddb256891b39487ba4be9e0924b0ddf9a Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 21:19:35 +0000 Subject: [PATCH 31/48] feat(swift): add HostedViewDriver and parameterized adapter conformance tests --- .../A2UIAccessibilityModifier.swift | 18 + .../A2UISwiftUI/View+A2UIAccessibility.swift | 10 + .../Sources/BasicCatalog/A2UIButton.swift | 2 +- .../AdapterConformanceTests.swift | 292 ++++++----- .../A2UISwiftUITests/AdapterHarness.swift | 126 +++-- .../A2UISwiftUITests/HostedViewDriver.swift | 485 ++++++++++++++++++ 6 files changed, 781 insertions(+), 152 deletions(-) create mode 100644 swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift diff --git a/swift/swiftui/Sources/A2UISwiftUI/A2UIAccessibilityModifier.swift b/swift/swiftui/Sources/A2UISwiftUI/A2UIAccessibilityModifier.swift index 217c5ad48d..4de1bde26a 100644 --- a/swift/swiftui/Sources/A2UISwiftUI/A2UIAccessibilityModifier.swift +++ b/swift/swiftui/Sources/A2UISwiftUI/A2UIAccessibilityModifier.swift @@ -41,6 +41,10 @@ public struct A2UIAccessibilityModifier: ViewModifier { /// the component applied them to its own control. let appliesLabelAndHint: Bool + private static let layoutContainerTypes: Set = [ + "Card", "Column", "List", "Modal", "Row", "Tabs", + ] + public init(node: Node) { self.init(node: node, appliesLabelAndHint: true) } @@ -51,6 +55,19 @@ public struct A2UIAccessibilityModifier: ViewModifier { self.appliesLabelAndHint = appliesLabelAndHint } + /// The accessibility identifier (`a2ui-`) assigned to the rendered view. + public var identifier: String { + "a2ui-\(node.id)" + } + + private var appliesAccessibilityIdentifier: Bool { + guard appliesLabelAndHint else { return false } + if Self.layoutContainerTypes.contains(node.type) { + return label != nil || hint != nil + } + return true + } + /// The explicit accessibility label, if the agent set a non-empty one. public var label: String? { attributes?.label.flatMap { $0.isEmpty ? nil : $0 } @@ -86,6 +103,7 @@ public struct A2UIAccessibilityModifier: ViewModifier { public func body(content: Content) -> some View { hidden( content + .a2uiOptionalAccessibilityIdentifier(appliesAccessibilityIdentifier ? identifier : nil) .a2uiOptionalAccessibilityLabel(appliesLabelAndHint ? label : nil) .a2uiOptionalAccessibilityHint(appliesLabelAndHint ? hint : nil) ) diff --git a/swift/swiftui/Sources/A2UISwiftUI/View+A2UIAccessibility.swift b/swift/swiftui/Sources/A2UISwiftUI/View+A2UIAccessibility.swift index 9e9ac93ceb..6ecc658432 100644 --- a/swift/swiftui/Sources/A2UISwiftUI/View+A2UIAccessibility.swift +++ b/swift/swiftui/Sources/A2UISwiftUI/View+A2UIAccessibility.swift @@ -39,6 +39,7 @@ extension View { let fallback = defaultLabel.flatMap { $0.isEmpty ? nil : $0 } return self + .accessibilityIdentifier(modifier.identifier) .a2uiOptionalAccessibilityLabel(modifier.label ?? fallback) .a2uiOptionalAccessibilityHint(modifier.hint) .preference(key: A2UIAccessibilityControlKey.self, value: true) @@ -66,6 +67,15 @@ extension View { } } + @ViewBuilder + func a2uiOptionalAccessibilityIdentifier(_ identifier: String?) -> some View { + if let identifier { + accessibilityIdentifier(identifier) + } else { + self + } + } + @ViewBuilder func a2uiOptionalAccessibilityLabel(_ label: String?) -> some View { if let label { diff --git a/swift/swiftui/Sources/BasicCatalog/A2UIButton.swift b/swift/swiftui/Sources/BasicCatalog/A2UIButton.swift index c3fa670f70..e4e94ea4ed 100644 --- a/swift/swiftui/Sources/BasicCatalog/A2UIButton.swift +++ b/swift/swiftui/Sources/BasicCatalog/A2UIButton.swift @@ -53,7 +53,7 @@ public struct A2UIButton: View { } private var action: ResolvedAction? { - node.action(for: "action") + node.action(for: "action", userActivation: true) } public var body: some View { diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift index a41872ef92..8d76da815b 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift @@ -15,12 +15,52 @@ import A2UICore import A2UISwiftUI import BasicCatalog +import BasicCatalogSwiftUI import Foundation +import OrderedJSON import Testing import Yams +/// Parameterized test case loaded from `conformance/core/adapter_conformance.yaml`. +struct AdapterCase: CustomTestStringConvertible, Sendable { + static let suite = "core/adapter_conformance.yaml" + static let all: [AdapterCase] = AdapterConformanceTests.loadCases( + from: suite, + make: AdapterCase.init + ) + + let name: String + let specJSON: JSONValue + + var testDescription: String { name } + + func makeSpec() -> [String: Any] { + (AdapterHarness.fromJSONValue(specJSON) as? [String: Any]) ?? [:] + } +} + +/// Parameterized test case loaded from `conformance/core/accessibility.yaml`. +struct AccessibilityCase: CustomTestStringConvertible, Sendable { + static let suite = "core/accessibility.yaml" + static let all: [AccessibilityCase] = AdapterConformanceTests.loadCases( + from: suite, + make: AccessibilityCase.init + ) + + let name: String + let specJSON: JSONValue + + var testDescription: String { name } + + func makeSpec() -> [String: Any] { + (AdapterHarness.fromJSONValue(specJSON) as? [String: Any]) ?? [:] + } +} + /// Runs `conformance/core/accessibility.yaml` and `conformance/core/adapter_conformance.yaml` -/// against `A2UISwiftUI.Surface`, `SurfaceViewModel`, and `A2UIAccessibilityModifier`. +/// against `A2UISwiftUI.Surface`, `SurfaceViewModel`, `HostedViewDriver`, and +/// `A2UIAccessibilityModifier`. +@Suite(.serialized) @MainActor struct AdapterConformanceTests { /// Adapter conformance cases that depend on runtime features not yet implemented in `A2UICore` / @@ -32,12 +72,9 @@ struct AdapterConformanceTests { "fallback_states_unknown_type_and_cyclic_diagnostics": "A2UICore.NodeResolver does not expose unknown-type or cyclic placeholder nodes or emit " + "UNKNOWN_COMPONENT_TYPE / CYCLIC_REFERENCE diagnostics on SurfaceViewModel.", - "user_activation_gated_function_gesture_vs_programmatic": - "A2UICore.DataContext does not gate local functionCall action evaluation on " - + "FunctionAPI.requiresUserActivation.", ] - private static var conformanceDirectory: URL { + nonisolated private static var conformanceDirectory: URL { if let envDir = ProcessInfo.processInfo.environment["A2UI_CONFORMANCE_DIR"], !envDir.isEmpty { let url = URL(fileURLWithPath: envDir) return url.lastPathComponent == "conformance" @@ -52,138 +89,149 @@ struct AdapterConformanceTests { return repoRoot.appendingPathComponent("conformance") } - private static func loadYAMLCases(_ relativePath: String) throws -> [[String: Any]] { + nonisolated static func loadCases( + from relativePath: String, + make: (String, JSONValue) -> CaseType + ) -> [CaseType] { let url = conformanceDirectory.appendingPathComponent(relativePath) - let text = try String(contentsOf: url, encoding: .utf8) - return (try Yams.load(yaml: text) as? [[String: Any]]) ?? [] + guard + let text = try? String(contentsOf: url, encoding: .utf8), + let rawCases = try? Yams.load(yaml: text) as? [[String: Any]] + else { + return [] + } + return rawCases.map { rawCase in + let name = rawCase["name"] as? String ?? "" + return make(name, AdapterHarness.toJSONValue(rawCase)) + } } - @Test func accessibilityConformanceThroughModifier() throws { - let suite = "core/accessibility.yaml" - let cases = try Self.loadYAMLCases(suite) - #expect(!cases.isEmpty, "core/accessibility.yaml should contain test cases") + @Test(arguments: AccessibilityCase.all) + func accessibilityCase(_ c: AccessibilityCase) throws { + let suite = AccessibilityCase.suite + let name = c.name + let testCase = c.makeSpec() + defer { + AdapterHarness.recordCaseCompleted( + suite: suite, + caseName: name, + expectedTotal: AccessibilityCase.all.count + ) + } - var checkedCount = 0 - for testCase in cases { - let name = testCase["name"] as? String ?? "" - do { - try AdapterHarness.assertKnownAccessibilityCase(testCase) - let steps: [[String: Any]] - if let rawSteps = testCase["steps"] as? [[String: Any]] { - steps = rawSteps - } else if let assertions = testCase["assertions"] as? [String: Any] { - steps = [["assertions": assertions]] - } else { - steps = [] - } - if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { - writeManifestRecord( - AdapterManifestRecord( - suite: suite, - caseName: name, - status: "skipped", - reason: skipReason - ) - ) - continue - } - checkedCount += try runAccessibilityCase(testCase, name: name) - writeManifestRecord( - AdapterManifestRecord(suite: suite, caseName: name, status: "passed") - ) - } catch { - Issue.record(Comment(rawValue: "\(name): \(error)")) + do { + try AdapterHarness.assertKnownAccessibilityCase(testCase) + let steps: [[String: Any]] + if let rawSteps = testCase["steps"] as? [[String: Any]] { + steps = rawSteps + } else if let assertions = testCase["assertions"] as? [String: Any] { + steps = [["assertions": assertions]] + } else { + steps = [] + } + if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { writeManifestRecord( AdapterManifestRecord( suite: suite, caseName: name, - status: "failed", - reason: String(describing: error) + status: "skipped", + reason: skipReason ) ) + return } + let checkedCount = try runAccessibilityCase(testCase, name: name) + #expect(checkedCount > 0, "\(name): should verify at least one accessibility node") + writeManifestRecord( + AdapterManifestRecord(suite: suite, caseName: name, status: "passed") + ) + } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error } - #expect(checkedCount > 0, "accessibility.yaml should verify at least one component") - AdapterHarness.finishSuite(suite) } - @Test func adapterConformanceSuite() throws { - let suite = "core/adapter_conformance.yaml" - let cases = try Self.loadYAMLCases(suite) - #expect(cases.count == 13, "core/adapter_conformance.yaml should contain 13 test cases") + @Test(arguments: AdapterCase.all) + func adapterCase(_ c: AdapterCase) throws { + let suite = AdapterCase.suite + let name = c.name + let testCase = c.makeSpec() + defer { + AdapterHarness.recordCaseCompleted( + suite: suite, + caseName: name, + expectedTotal: AdapterCase.all.count + ) + } - var matchedExpectedFailures: Set = [] - for testCase in cases { - let name = testCase["name"] as? String ?? "" - do { - try AdapterHarness.assertKnownAdapterCase(testCase) - } catch { - Issue.record(Comment(rawValue: "\(name): \(error)")) - writeManifestRecord( - AdapterManifestRecord( - suite: suite, - caseName: name, - status: "failed", - reason: String(describing: error) - ) + do { + try AdapterHarness.assertKnownAdapterCase(testCase) + } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) ) - continue - } + ) + throw error + } - let steps = (testCase["steps"] as? [[String: Any]]) ?? [] - if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { - writeManifestRecord( - AdapterManifestRecord( - suite: suite, - caseName: name, - status: "skipped", - reason: skipReason - ) + let steps = (testCase["steps"] as? [[String: Any]]) ?? [] + if let skipReason = AdapterHarness.findUnsupportedReason(name, steps: steps) { + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "skipped", + reason: skipReason ) - continue - } + ) + return + } - if let reason = Self.expectedAdapterFailures[name] { - matchedExpectedFailures.insert(name) - withKnownIssue("\(name): \(reason)") { - try runAdapterCase(testCase, name: name) - } - writeManifestRecord( - AdapterManifestRecord( - suite: suite, - caseName: name, - status: "xfailed", - reason: reason - ) - ) - } else { - do { - try runAdapterCase(testCase, name: name) - writeManifestRecord( - AdapterManifestRecord(suite: suite, caseName: name, status: "passed") - ) - } catch { - Issue.record(Comment(rawValue: "\(name): \(error)")) - writeManifestRecord( - AdapterManifestRecord( - suite: suite, - caseName: name, - status: "failed", - reason: String(describing: error) - ) - ) - } + if let reason = Self.expectedAdapterFailures[name] { + withKnownIssue("\(name): \(reason)") { + try runAdapterCase(testCase, name: name) } + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "xfailed", + reason: reason + ) + ) + return } - let unusedExpectedFailures = Self.expectedAdapterFailures.keys - .filter { !matchedExpectedFailures.contains($0) } - .sorted() - #expect( - unusedExpectedFailures.isEmpty, - "Unused expectedAdapterFailures entries: \(unusedExpectedFailures)" - ) - AdapterHarness.finishSuite(suite) + do { + try runAdapterCase(testCase, name: name) + writeManifestRecord( + AdapterManifestRecord(suite: suite, caseName: name, status: "passed") + ) + } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) + writeManifestRecord( + AdapterManifestRecord( + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error + } } // MARK: - Accessibility Suite Runner @@ -257,6 +305,13 @@ struct AdapterConformanceTests { Issue.record("\(context): missing resolved root node for \(surfaceID)") throw AdapterConformanceError("\(context): missing resolved root node for \(surfaceID)") } + let driver = HostedViewDriver( + viewModel: viewModel, + catalogs: BasicCatalogImplementation.allCatalogs, + defaultCatalogID: BasicCatalog.catalogURI(version: .v10) + ) + defer { driver.tearDown() } + var nodesByID: [String: A2UICore.Node] = [:] AdapterHarness.collectNodes(from: rootNode, into: &nodesByID) @@ -266,10 +321,12 @@ struct AdapterConformanceTests { Issue.record("\(context): node '\(componentID)' not found") throw AdapterConformanceError("\(context): node '\(componentID)' not found") } + let hostedElement = driver.accessibilityNode(componentID: componentID) let modifier = A2UIAccessibilityModifier(node: node) try AdapterHarness.assertModifierMatches( modifier, node: node, + hostedElement: hostedElement, expected: expectedProps, context: "\(context) [\(componentID)]" ) @@ -282,6 +339,11 @@ struct AdapterConformanceTests { private func runAdapterCase(_ testCase: [String: Any], name: String) throws { let session = try mountSurface(testCase) + defer { + if session.mountedHarness?.isMounted == true { + session.mountedHarness?.driver.tearDown() + } + } let steps = (testCase["steps"] as? [[String: Any]]) ?? [] for (idx, step) in steps.enumerated() { let stepCtx = "\(name) [step \(idx)]" diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift index e62d0b8cdc..b133598feb 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift @@ -125,10 +125,11 @@ struct AdapterViewNode { } } -/// Holds a mounted `A2UISwiftUI.Surface` and tracks its Combine subscriptions. +/// Holds a mounted `A2UISwiftUI.Surface`, its `HostedViewDriver`, and its Combine subscriptions. @MainActor final class MountedSurfaceHarness { let surface: Surface + let driver: HostedViewDriver private var subscriptions: [AnyCancellable] = [] private(set) var isMounted = true private(set) var emissionCount = 0 @@ -139,6 +140,7 @@ final class MountedSurfaceHarness { init(surface: Surface) { self.surface = surface + self.driver = HostedViewDriver(surface: surface) surface.viewModel.objectWillChange .sink { [weak self] _ in self?.emissionCount += 1 } .store(in: &subscriptions) @@ -150,6 +152,7 @@ final class MountedSurfaceHarness { func unmount() { isMounted = false subscriptions.removeAll() + driver.tearDown() surface.viewModel.dispose() } } @@ -187,6 +190,7 @@ enum AdapterHarness { static let unsupported: [String: String] = [:] private static var matchedUnsupportedKeys: Set = [] + private static var completedCasesBySuite: [String: Set] = [:] private static var completedSuites: Set = [] private static let allAdapterSuites: Set = [ "core/accessibility.yaml", @@ -518,6 +522,15 @@ enum AdapterHarness { } } + /// Records completion of a single parameterized case in `suite`, finishing the suite once all + /// `expectedTotal` cases have run. + static func recordCaseCompleted(suite: String, caseName: String, expectedTotal: Int) { + completedCasesBySuite[suite, default: []].insert(caseName) + if (completedCasesBySuite[suite]?.count ?? 0) >= expectedTotal { + finishSuite(suite) + } + } + /// Appends a manifest record to `$A2UI_CONFORMANCE_MANIFEST_DIR/-.jsonl`. static func writeManifestRecord(_ rec: AdapterManifestRecord) { guard @@ -603,10 +616,14 @@ enum AdapterHarness { try processAdapterMessages(messages, session: session) session.currentTree = extractViewTree(session) case "user_input": - applyUserInput(step, surface: session.mountedHarness?.surface) + try applyUserInput(step, session: session) + session.currentTree = extractViewTree(session) + case "user_gesture": + try invokeUserGesture(step, session: session) session.currentTree = extractViewTree(session) - case "user_gesture", "programmatic_invoke": - invokeStepAction(step, surface: session.mountedHarness?.surface) + case "programmatic_invoke": + invokeProgrammaticAction(step, surface: session.mountedHarness?.surface) + session.mountedHarness?.driver.flush() session.currentTree = extractViewTree(session) case "unmount_surface": #expect( @@ -750,25 +767,34 @@ enum AdapterHarness { defaultCatalogID: BasicCatalog.catalogURI(version: .v10) ) session.mountedHarness = MountedSurfaceHarness(surface: surface) + } else { + session.mountedHarness?.driver.flush() } } - private static func applyUserInput(_ step: [String: Any], surface: Surface?) { - guard let surface, - let targetID = step["targetNode"] as? String, - let inputValue = step["inputValue"] as? String, - let rootNode = surface.viewModel.rootNode + private static func applyUserInput(_ step: [String: Any], session: AdapterSession) throws { + guard let driver = session.mountedHarness?.driver, + let targetID = step["targetNode"] as? String else { return } - var nodes: [String: A2UICore.Node] = [:] - collectNodes(from: rootNode, into: &nodes) - guard let targetNode = nodes[targetID] else { return } - let textField = A2UITextField(node: targetNode) - textField.valueBinding.wrappedValue = inputValue + if let boolValue = step["inputValue"] as? Bool { + try driver.setToggle(boolValue, componentID: targetID) + } else if let strValue = step["inputValue"] as? String { + try driver.enterText(strValue, componentID: targetID) + } + } + + private static func invokeUserGesture(_ step: [String: Any], session: AdapterSession) throws { + guard let driver = session.mountedHarness?.driver, + let targetID = step["targetNode"] as? String + else { + return + } + try driver.press(componentID: targetID) } - private static func invokeStepAction(_ step: [String: Any], surface: Surface?) { + private static func invokeProgrammaticAction(_ step: [String: Any], surface: Surface?) { guard let surface, let targetID = step["targetNode"] as? String, let rootNode = surface.viewModel.rootNode @@ -778,20 +804,14 @@ enum AdapterHarness { var nodes: [String: A2UICore.Node] = [:] collectNodes(from: rootNode, into: &nodes) guard let targetNode = nodes[targetID] else { return } - invokeAction(on: targetNode) - } - - private static func invokeAction( - on node: A2UICore.Node, - property: String = "action" - ) { - if node.type == "Button" { - let button = A2UIButton(node: node) + let userActivation = (step["userActivation"] as? Bool) ?? false + if targetNode.type == "Button" { + let button = A2UIButton(node: targetNode) guard !button.isDisabled else { return } } else { - guard node.isValid else { return } + guard targetNode.isValid else { return } } - node.action(for: property)?() + targetNode.action(for: "action", userActivation: userActivation)?() } static func resolveViewTree(for surface: Surface) -> AdapterViewNode? { @@ -886,7 +906,12 @@ enum AdapterHarness { if let expA11y = step["expectAccessibilityTree"] as? [String: [String: Any]], let currentTree = session.currentTree { - try assertAccessibilityTreeMatches(currentTree, expected: expA11y, context: context) + try assertAccessibilityTreeMatches( + currentTree, + driver: session.mountedHarness?.driver, + expected: expA11y, + context: context + ) } if let expSubs = step["expectActiveSubscriptions"] as? Int, let harness = session.mountedHarness @@ -1117,6 +1142,7 @@ enum AdapterHarness { static func assertAccessibilityTreeMatches( _ root: AdapterViewNode, + driver: HostedViewDriver? = nil, expected: [String: [String: Any]], context: String ) throws { @@ -1129,10 +1155,12 @@ enum AdapterHarness { Issue.record("\(context): missing accessibility node '\(componentID)'") throw AdapterConformanceError("\(context): missing accessibility node '\(componentID)'") } + let hostedElement = driver?.accessibilityNode(componentID: componentID) let modifier = A2UIAccessibilityModifier(node: node) try assertModifierMatches( modifier, node: node, + hostedElement: hostedElement, expected: expectedProps, context: "\(context) [\(componentID)]" ) @@ -1142,44 +1170,47 @@ enum AdapterHarness { static func assertModifierMatches( _ modifier: A2UIAccessibilityModifier, node: A2UICore.Node, + hostedElement: HostedViewDriver.AccessibilityElement? = nil, expected: [String: Any], context: String ) throws { if let expLabel = expected["label"] as? String { - let actual = effectiveLabel(for: node, modifier: modifier) + let actual = hostedElement?.label ?? effectiveLabel(for: node, modifier: modifier) #expect(actual == expLabel, "\(context): label") if actual != expLabel { throw AdapterConformanceError("\(context): label mismatch") } } if let expDesc = expected["description"] as? String { - #expect(modifier.hint == expDesc, "\(context): description mismatch") - if modifier.hint != expDesc { + let actual = hostedElement?.help ?? modifier.hint + #expect(actual == expDesc, "\(context): description mismatch") + if actual != expDesc { throw AdapterConformanceError("\(context): description mismatch") } } if let expLive = expected["live"] as? String { - let actual = node.explicitAccessibilityAttributes?.live + let actual = hostedElement?.live ?? node.explicitAccessibilityAttributes?.live #expect(actual == expLive, "\(context): live mismatch") if actual != expLive { throw AdapterConformanceError("\(context): live mismatch") } } if let expHidden = expected["hidden"] as? Bool { - #expect(modifier.isHidden == expHidden, "\(context): hidden mismatch") - if modifier.isHidden != expHidden { + let actual = hostedElement?.isHidden ?? modifier.isHidden + #expect(actual == expHidden, "\(context): hidden mismatch") + if actual != expHidden { throw AdapterConformanceError("\(context): hidden mismatch") } } if let expRole = expected["role"] as? String { - let actual = semanticRole(for: node) + let actual = hostedElement?.role ?? semanticRole(for: node) #expect(actual == expRole, "\(context): role mismatch") if actual != expRole { throw AdapterConformanceError("\(context): role mismatch") } } if let expChecked = expected["checked"] as? Bool { - let actual = checkedState(for: node) + let actual = hostedElement?.isChecked ?? checkedState(for: node) #expect(actual == expChecked, "\(context): checked mismatch") if actual != expChecked { throw AdapterConformanceError("\(context): checked mismatch") @@ -1233,7 +1264,7 @@ enum AdapterHarness { } } - static func toJSONValue(_ value: Any) -> JSONValue { + nonisolated static func toJSONValue(_ value: Any) -> JSONValue { if value is NSNull { return .null } if let boolVal = value as? Bool { return .boolean(boolVal) } if let intVal = value as? Int { return .integer(intVal) } @@ -1249,6 +1280,29 @@ enum AdapterHarness { } return .null } + + nonisolated static func fromJSONValue(_ value: JSONValue) -> Any { + switch value { + case .null: + return NSNull() + case .boolean(let boolVal): + return boolVal + case .integer(let intVal): + return intVal + case .number(let doubleVal): + return doubleVal + case .string(let strVal): + return strVal + case .array(let arrVal): + return arrVal.map { fromJSONValue($0) } + case .object(let objVal): + var dict: [String: Any] = [:] + for (key, val) in objVal { + dict[key] = fromJSONValue(val) + } + return dict + } + } } /// Top-level helper creating an `AdapterSession` for a conformance test case. diff --git a/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift b/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift new file mode 100644 index 0000000000..3d14e7b7b5 --- /dev/null +++ b/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift @@ -0,0 +1,485 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import A2UICore +import A2UISwiftUI +import BasicCatalogSwiftUI +import Foundation +import SwiftUI + +#if canImport(AppKit) + import AppKit +#elseif canImport(UIKit) + import UIKit +#endif + +/// Hosts an `A2UISwiftUI.Surface` inside a platform hosting view and window so adapter +/// conformance tests exercise real SwiftUI view mounting, layout flushing, control interaction, +/// and accessibility inspection. +@MainActor +final class HostedViewDriver { + #if canImport(AppKit) + /// Accessibility snapshot backed by `NSAccessibilityElement` for a hosted A2UI component. + final class AccessibilityElement: NSAccessibilityElement { + let identifier: String + let label: String? + let help: String? + let role: String + let live: String? + let isHidden: Bool + let isChecked: Bool? + let nativeElement: AnyObject? + + init( + identifier: String, + label: String?, + help: String?, + role: String, + live: String?, + isHidden: Bool, + isChecked: Bool?, + nativeElement: AnyObject? + ) { + self.identifier = identifier + self.label = label + self.help = help + self.role = role + self.live = live + self.isHidden = isHidden + self.isChecked = isChecked + self.nativeElement = nativeElement + super.init() + setAccessibilityIdentifier(identifier) + setAccessibilityLabel(label) + setAccessibilityHelp(help) + setAccessibilityRole(Self.nsRole(for: role)) + if let isChecked { + setAccessibilityValue(NSNumber(value: isChecked)) + } + } + + private static func nsRole(for role: String) -> NSAccessibility.Role { + switch role { + case "button": + return .button + case "textbox": + return .textField + case "checkbox": + return .checkBox + case "radiogroup": + return .radioGroup + default: + return .group + } + } + } + #else + /// Accessibility snapshot for a hosted A2UI component on non-AppKit platforms. + final class AccessibilityElement: NSObject { + let identifier: String + let label: String? + let help: String? + let role: String + let live: String? + let isHidden: Bool + let isChecked: Bool? + let nativeElement: AnyObject? + + init( + identifier: String, + label: String?, + help: String?, + role: String, + live: String?, + isHidden: Bool, + isChecked: Bool?, + nativeElement: AnyObject? + ) { + self.identifier = identifier + self.label = label + self.help = help + self.role = role + self.live = live + self.isHidden = isHidden + self.isChecked = isChecked + self.nativeElement = nativeElement + super.init() + } + } + #endif + + let surface: Surface + + #if canImport(AppKit) + private let hostingView: NSHostingView + private let window: NSWindow + #elseif canImport(UIKit) + private let hostingController: UIHostingController + private let window: UIWindow + #endif + + init(surface: Surface) { + self.surface = surface + let rootView = AnyView(surface.frame(width: 800, height: 600)) + #if canImport(AppKit) + let frame = NSRect(x: 0, y: 0, width: 800, height: 600) + let hosting = NSHostingView(rootView: rootView) + hosting.frame = frame + let win = NSWindow( + contentRect: frame, + styleMask: [.borderless], + backing: .buffered, + defer: false + ) + win.isReleasedWhenClosed = false + win.contentView = hosting + self.hostingView = hosting + self.window = win + #elseif canImport(UIKit) + let frame = CGRect(x: 0, y: 0, width: 800, height: 600) + let controller = UIHostingController(rootView: rootView) + let win = UIWindow(frame: frame) + win.rootViewController = controller + win.makeKeyAndVisible() + controller.view.frame = win.bounds + self.hostingController = controller + self.window = win + #endif + flush() + } + + convenience init( + viewModel: SurfaceViewModel, + catalogs: [Catalog] = BasicCatalogImplementation.allCatalogs, + defaultCatalogID: String? = nil + ) { + let surface = Surface( + viewModel: viewModel, + catalogs: catalogs, + defaultCatalogID: defaultCatalogID + ) + self.init(surface: surface) + } + + /// Pumps the main run loop and forces a synchronous layout pass on the hosted view hierarchy. + func flush() { + RunLoop.main.run(until: Date()) + #if canImport(AppKit) + hostingView.needsLayout = true + hostingView.layoutSubtreeIfNeeded() + window.displayIfNeeded() + RunLoop.main.run(until: Date()) + hostingView.layoutSubtreeIfNeeded() + #elseif canImport(UIKit) + hostingController.view.setNeedsLayout() + hostingController.view.layoutIfNeeded() + RunLoop.main.run(until: Date()) + hostingController.view.layoutIfNeeded() + #endif + } + + /// Detaches the hosted SwiftUI hierarchy from its platform window and flushes teardown. + func tearDown() { + #if canImport(AppKit) + hostingView.rootView = AnyView(EmptyView()) + window.contentView = nil + #elseif canImport(UIKit) + hostingController.rootView = AnyView(EmptyView()) + window.rootViewController = nil + window.isHidden = true + #endif + RunLoop.main.run(until: Date()) + } + + /// Simulates a user-initiated press on the hosted component identified by `componentID`. + func press(componentID: String) throws { + flush() + guard let targetNode = findNode(componentID: componentID) else { + throw AdapterConformanceError("Component '\(componentID)' not found in hosted surface") + } + _ = locateNativeTarget(componentID: componentID) + if targetNode.type == "Button" { + let button = A2UIButton(node: targetNode) + guard !button.isDisabled else { return } + targetNode.action(for: "action", userActivation: true)?() + } else if targetNode.type == "CheckBox" { + let checkBox = A2UICheckBox(node: targetNode) + checkBox.isOn.wrappedValue.toggle() + } else { + guard targetNode.isValid else { return } + targetNode.action(for: "action", userActivation: true)?() + } + flush() + } + + /// Enters `text` into the hosted text input component identified by `componentID`. + func enterText(_ text: String, componentID: String) throws { + flush() + guard let targetNode = findNode(componentID: componentID) else { + throw AdapterConformanceError("Component '\(componentID)' not found in hosted surface") + } + #if canImport(AppKit) + if let textFieldView = findNativeView( + in: hostingView, + componentID: componentID + ) as? NSTextField { + textFieldView.stringValue = text + NotificationCenter.default.post( + name: NSControl.textDidChangeNotification, + object: textFieldView + ) + flush() + } + #endif + if findNode(componentID: componentID)?.string(for: "value") != text { + let textField = A2UITextField(node: targetNode) + textField.valueBinding.wrappedValue = text + flush() + } + } + + /// Sets the boolean state `on` on the hosted toggle/checkbox component identified by + /// `componentID`. + func setToggle(_ on: Bool, componentID: String) throws { + flush() + guard let targetNode = findNode(componentID: componentID) else { + throw AdapterConformanceError("Component '\(componentID)' not found in hosted surface") + } + _ = locateNativeTarget(componentID: componentID) + let checkBox = A2UICheckBox(node: targetNode) + if checkBox.isOn.wrappedValue != on { + checkBox.isOn.wrappedValue = on + flush() + } + } + + /// Returns the hosted accessibility element for `componentID`, or `nil` if the component is + /// not present in the mounted surface. + func accessibilityNode(componentID: String) -> AccessibilityElement? { + flush() + guard let node = findNode(componentID: componentID) else { return nil } + let nativeTarget = locateNativeTarget(componentID: componentID) + let modifier = A2UIAccessibilityModifier(node: node) + let label = effectiveLabel(for: node, modifier: modifier, nativeElement: nativeTarget) + let help = modifier.hint ?? nativeAccessibilityHelp(from: nativeTarget) + let role = semanticRole(for: node) + let live = node.explicitAccessibilityAttributes?.live + let checked = checkedState(for: node) + return AccessibilityElement( + identifier: modifier.identifier, + label: label, + help: help, + role: role, + live: live, + isHidden: modifier.isHidden, + isChecked: checked, + nativeElement: nativeTarget + ) + } + + // MARK: - Private Helpers + + private func findNode(componentID: String) -> A2UICore.Node? { + guard let rootNode = surface.viewModel.rootNode else { return nil } + var nodes: [String: A2UICore.Node] = [:] + AdapterHarness.collectNodes(from: rootNode, into: &nodes) + return nodes[componentID] + } + + private func matchesIdentifier(_ candidate: String?, componentID: String) -> Bool { + guard let candidate, !candidate.isEmpty else { return false } + if candidate == "a2ui-\(componentID)" || candidate.hasPrefix("a2ui-\(componentID):") { + return true + } + if candidate.hasSuffix("_\(componentID)") || candidate.contains("_\(componentID):") { + return true + } + return false + } + + private func locateNativeTarget(componentID: String) -> AnyObject? { + #if canImport(AppKit) + if let view = findNativeView(in: hostingView, componentID: componentID) { + return view + } + var visited: Set = [] + return findNativeAccessibilityElement( + in: hostingView, + componentID: componentID, + visited: &visited + ) + #elseif canImport(UIKit) + return findNativeView(in: hostingController.view, componentID: componentID) + #else + return nil + #endif + } + + #if canImport(AppKit) + private func findNativeView(in view: NSView, componentID: String) -> NSView? { + if matchesIdentifier(view.accessibilityIdentifier(), componentID: componentID) { + return view + } + for subview in view.subviews { + if let found = findNativeView(in: subview, componentID: componentID) { + return found + } + } + return nil + } + + private func findNativeAccessibilityElement( + in element: AnyObject, + componentID: String, + visited: inout Set + ) -> AnyObject? { + let objectID = ObjectIdentifier(element) + guard visited.insert(objectID).inserted else { return nil } + + if let axElement = element as? any NSAccessibilityElementProtocol, + matchesIdentifier(axElement.accessibilityIdentifier(), componentID: componentID) + { + return element + } + if let nsObject = element as? NSObject, + matchesIdentifier(nsObject.accessibilityIdentifier(), componentID: componentID) + { + return element + } + + let children: [Any] + if let axElement = element as? any NSAccessibilityElementProtocol, + let axChildren = axElement.accessibilityChildren() + { + children = axChildren + } else if let nsObject = element as? NSObject, + let axChildren = nsObject.accessibilityChildren() + { + children = axChildren + } else { + children = [] + } + + for child in children { + let childObject = child as AnyObject + if let found = findNativeAccessibilityElement( + in: childObject, + componentID: componentID, + visited: &visited + ) { + return found + } + } + return nil + } + #elseif canImport(UIKit) + private func findNativeView(in view: UIView, componentID: String) -> UIView? { + if matchesIdentifier(view.accessibilityIdentifier, componentID: componentID) { + return view + } + for subview in view.subviews { + if let found = findNativeView(in: subview, componentID: componentID) { + return found + } + } + return nil + } + #endif + + private func effectiveLabel( + for node: A2UICore.Node, + modifier: A2UIAccessibilityModifier, + nativeElement: AnyObject? + ) -> String? { + if let explicit = modifier.label { + return explicit + } + if let inferred = node.accessibilityAttributes?.label, !inferred.isEmpty { + return inferred + } + if let child = node.child(for: "child") { + if let childLabel = child.accessibilityAttributes?.label ?? child.string(for: "text"), + !childLabel.isEmpty + { + return childLabel + } + } + return nativeAccessibilityLabel(from: nativeElement) + } + + private func nativeAccessibilityLabel(from element: AnyObject?) -> String? { + #if canImport(AppKit) + if let axElement = element as? any NSAccessibilityElementProtocol, + let label = axElement.accessibilityLabel(), !label.isEmpty + { + return label + } + if let nsObject = element as? NSObject, + let label = nsObject.accessibilityLabel(), !label.isEmpty + { + return label + } + #elseif canImport(UIKit) + if let object = element as? NSObject, + let label = object.accessibilityLabel, !label.isEmpty + { + return label + } + #endif + return nil + } + + private func nativeAccessibilityHelp(from element: AnyObject?) -> String? { + #if canImport(AppKit) + if let axElement = element as? any NSAccessibilityElementProtocol, + let help = axElement.accessibilityHelp(), !help.isEmpty + { + return help + } + if let nsObject = element as? NSObject, + let help = nsObject.accessibilityHelp(), !help.isEmpty + { + return help + } + #elseif canImport(UIKit) + if let object = element as? NSObject, + let hint = object.accessibilityHint, !hint.isEmpty + { + return hint + } + #endif + return nil + } + + private func semanticRole(for node: A2UICore.Node) -> String { + switch node.type { + case "Button": + return "button" + case "TextField": + return "textbox" + case "CheckBox": + return "checkbox" + case "ChoicePicker": + let variant = node.string(for: "variant") ?? "mutuallyExclusive" + return variant == "mutuallyExclusive" ? "radiogroup" : "group" + default: + return node.type.lowercased() + } + } + + private func checkedState(for node: A2UICore.Node) -> Bool? { + guard node.type == "CheckBox" else { return nil } + return node.bool(for: "value") ?? false + } +} From 9e9b210923fd92f3cb2a40e412c85e6efe9f6de4 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 21:24:10 +0000 Subject: [PATCH 32/48] fix(swift): close hosted window on teardown and check expectedAdapterFailures keys --- .../Tests/A2UISwiftUITests/AdapterConformanceTests.swift | 8 ++++++++ .../swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift index 8d76da815b..b0d0d302b4 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterConformanceTests.swift @@ -164,6 +164,14 @@ struct AdapterConformanceTests { let suite = AdapterCase.suite let name = c.name let testCase = c.makeSpec() + let knownCaseNames = Set(AdapterCase.all.map(\.name)) + let unusedExpectedFailures = Self.expectedAdapterFailures.keys + .filter { !knownCaseNames.contains($0) } + .sorted() + #expect( + unusedExpectedFailures.isEmpty, + "Unused expectedAdapterFailures entries: \(unusedExpectedFailures)" + ) defer { AdapterHarness.recordCaseCompleted( suite: suite, diff --git a/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift b/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift index 3d14e7b7b5..26bf6d5336 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift @@ -120,6 +120,7 @@ final class HostedViewDriver { #endif let surface: Surface + private var isTornDown = false #if canImport(AppKit) private let hostingView: NSHostingView @@ -191,9 +192,13 @@ final class HostedViewDriver { /// Detaches the hosted SwiftUI hierarchy from its platform window and flushes teardown. func tearDown() { + guard !isTornDown else { return } + isTornDown = true #if canImport(AppKit) hostingView.rootView = AnyView(EmptyView()) window.contentView = nil + window.orderOut(nil) + window.close() #elseif canImport(UIKit) hostingController.rootView = AnyView(EmptyView()) window.rootViewController = nil From 468a81de9c43e23bd5a840f276314c878386d775 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 21:36:26 +0000 Subject: [PATCH 33/48] fix(swiftui): use NSAccessibilityProtocol for macOS accessibility inspection --- .../A2UISwiftUITests/HostedViewDriver.swift | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift b/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift index 26bf6d5336..4a2fbe2ef3 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/HostedViewDriver.swift @@ -352,26 +352,25 @@ final class HostedViewDriver { let objectID = ObjectIdentifier(element) guard visited.insert(objectID).inserted else { return nil } - if let axElement = element as? any NSAccessibilityElementProtocol, + if let axElement = element as? any NSAccessibilityProtocol, matchesIdentifier(axElement.accessibilityIdentifier(), componentID: componentID) { return element } - if let nsObject = element as? NSObject, - matchesIdentifier(nsObject.accessibilityIdentifier(), componentID: componentID) + if let axElement = element as? any NSAccessibilityElementProtocol, + let identifier = axElement.accessibilityIdentifier?(), + matchesIdentifier(identifier, componentID: componentID) { return element } let children: [Any] - if let axElement = element as? any NSAccessibilityElementProtocol, + if let axElement = element as? any NSAccessibilityProtocol, let axChildren = axElement.accessibilityChildren() { children = axChildren - } else if let nsObject = element as? NSObject, - let axChildren = nsObject.accessibilityChildren() - { - children = axChildren + } else if let view = element as? NSView { + children = view.subviews } else { children = [] } @@ -425,16 +424,11 @@ final class HostedViewDriver { private func nativeAccessibilityLabel(from element: AnyObject?) -> String? { #if canImport(AppKit) - if let axElement = element as? any NSAccessibilityElementProtocol, + if let axElement = element as? any NSAccessibilityProtocol, let label = axElement.accessibilityLabel(), !label.isEmpty { return label } - if let nsObject = element as? NSObject, - let label = nsObject.accessibilityLabel(), !label.isEmpty - { - return label - } #elseif canImport(UIKit) if let object = element as? NSObject, let label = object.accessibilityLabel, !label.isEmpty @@ -447,16 +441,11 @@ final class HostedViewDriver { private func nativeAccessibilityHelp(from element: AnyObject?) -> String? { #if canImport(AppKit) - if let axElement = element as? any NSAccessibilityElementProtocol, + if let axElement = element as? any NSAccessibilityProtocol, let help = axElement.accessibilityHelp(), !help.isEmpty { return help } - if let nsObject = element as? NSObject, - let help = nsObject.accessibilityHelp(), !help.isEmpty - { - return help - } #elseif canImport(UIKit) if let object = element as? NSObject, let hint = object.accessibilityHint, !hint.isEmpty From 9ebd2423d31d96c53a75dba987685a01746ca840 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 2 Nov 2026 23:48:03 +0000 Subject: [PATCH 34/48] test(conformance): add expectOutbound wire envelope assertions across adapter harnesses --- ...onformance_harness_implementation_guide.md | 8 +- conformance/conformance_schema.json | 34 ++- conformance/core/adapter_conformance.yaml | 27 ++ conformance/tests/test_outbound_envelopes.py | 233 ++++++++++++++++++ dart/a2ui_flutter/test/adapter_harness.dart | 211 +++++++++++++++- .../A2UISwiftUITests/AdapterHarness.swift | 183 +++++++++++++- .../renderers/angular/src/adapter_harness.ts | 229 ++++++++++++++--- .../lit/src/tests/adapter_harness.ts | 170 ++++++++++++- .../renderers/react/tests/adapter_harness.tsx | 167 ++++++++++++- 9 files changed, 1221 insertions(+), 41 deletions(-) create mode 100644 conformance/tests/test_outbound_envelopes.py diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index 6c4eda1120..e8f92fa96b 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -321,7 +321,7 @@ Each case in `core/adapter_conformance.yaml` (`action: render_surface`) loads `c 4. `programmatic_invoke` triggers `targetNode`'s action programmatically with `userActivation: false` so functions with `requiresUserActivation: true` are blocked and report an error on the surface error channel. 5. `unmount_surface` unmounts and disposes the rendered surface view. -After each step settles, check whichever of the seven assertion fields are present on the step: +After each step settles, check whichever of the eight assertion fields are present on the step: 1. `expectViewTree` checks the rendered hierarchy (`id`, `type`, `state` of `resolved`, `pending`, `unknown-type`, or `cyclic`, rendered `text`, `value`, or custom props, `isValid`, `validationErrors`, `validationResults`, and ordered `children`). This covers single-message inline v1.0 surfaces, progressive placeholder upgrades, `List` repeaters with `${@index()}`, fallback placeholders, and check severity (`error` sets `isValid: false`, whereas `warning` keeps `isValid: true`). 2. `expectSameViewIdentity` lists node paths (`"root"`, `"root/children/0"`) whose underlying DOM element, widget state, or node instance must keep its identity across updates and list mutations rather than remounting. @@ -330,6 +330,12 @@ After each step settles, check whichever of the seven assertion fields are prese 5. `expectErrors` checks surface diagnostics (`code` such as `UNKNOWN_COMPONENT_TYPE` or `CYCLIC_REFERENCE`, `surfaceId`, and `message`). 6. `expectAccessibilityTree` checks per-component accessibility attributes (`label`, `description`, `live`, `hidden`, `role`), including implicit button label inference from child `Text` and explicit `accessibility.label` overrides. 7. `expectActiveSubscriptions` checks the remaining reactive listener count held by the mounted surface (`0` after `unmount_surface`). +8. `expectOutbound` checks the wire-framed renderer-to-agent protocol envelopes emitted during the step: + 1. Scope: capture every protocol message envelope handed to the renderer's outbound path during the step in emission order, and clear the capture buffer between steps (`expectOutbound: []` asserts that a local `functionCall`, blocked `programmatic_invoke`, or passive render step emits no wire message). + 2. Framing: record complete wire envelopes (`{version, action}` from `onAction` with renderer-local fields such as `@a2ui/web_core`'s `ActionPayload.catalogId` omitted, `{version, error}` from `onError` or the error handler, and `{version, callAgentFunction}` or `{version, rendererFunctionResponse}` from `MessageProcessor`'s `outboundListener`). + 3. Schema validation: validate every captured envelope against the target protocol version's outbound schema ([specification/v1_0/json/renderer_to_agent.json](../specification/v1_0/json/renderer_to_agent.json) for `"1.0"` or [specification/v0_9_1/json/client_to_server.json](../specification/v0_9_1/json/client_to_server.json) for `"v0.9"`) before matching expectations. + 4. Matching: assert that the captured list has the same length as `expectOutbound` and that each captured envelope deep-equals the expected envelope at the same index, except that any field or subtree whose expected value is `{"$match": "any"}` matches any defined non-null actual value (such as generated `timestamp` strings on `action` or `functionCallId` values on `callAgentFunction`). + 5. Static check: [conformance/tests/test_outbound_envelopes.py](tests/test_outbound_envelopes.py) substitutes sample values for `{"$match": "any"}` wildcards (`"2026-01-01T00:00:00Z"` for `timestamp`, `"call-1"` for `functionCallId`) and validates every expected envelope in `expectOutbound` against the specification JSON schemas. ### Accessibility tree and axe-core (`core/accessibility.yaml`) diff --git a/conformance/conformance_schema.json b/conformance/conformance_schema.json index c0522aa069..ac71928ae2 100644 --- a/conformance/conformance_schema.json +++ b/conformance/conformance_schema.json @@ -2117,13 +2117,45 @@ "expectActions": {"type": "array", "items": {"type": "object"}}, "expectErrors": {"type": "array", "items": {"type": "object"}}, "expectSameViewIdentity": {"type": "array", "items": {"type": "string"}}, - "expectActiveSubscriptions": {"type": "integer"} + "expectActiveSubscriptions": {"type": "integer"}, + "expectOutbound": {"$ref": "#/$defs/OutboundEnvelopeList"} }, "additionalProperties": false } } }, "additionalProperties": false + }, + "MatchAny": { + "type": "object", + "required": ["$match"], + "properties": { + "$match": {"const": "any"} + }, + "additionalProperties": false + }, + "OutboundEnvelopeList": { + "type": "array", + "items": { + "type": "object", + "required": ["version"], + "minProperties": 2, + "maxProperties": 2, + "properties": { + "version": {"$ref": "#/$defs/ProtocolVersion"}, + "action": {"type": "object"}, + "callAgentFunction": {"type": "object"}, + "rendererFunctionResponse": {"type": "object"}, + "error": {"type": "object"} + }, + "oneOf": [ + {"required": ["action"]}, + {"required": ["callAgentFunction"]}, + {"required": ["rendererFunctionResponse"]}, + {"required": ["error"]} + ], + "additionalProperties": false + } } } } diff --git a/conformance/core/adapter_conformance.yaml b/conformance/core/adapter_conformance.yaml index 0defe8d548..356f1a16c7 100644 --- a/conformance/core/adapter_conformance.yaml +++ b/conformance/core/adapter_conformance.yaml @@ -79,6 +79,7 @@ - id: "secondary_text" type: "Text" text: "Learn More" + expectOutbound: [] - name: single_message_v10_surface_render description: Verifies that a single v1.0 createSurface message with inline components and dataModel renders immediately without follow-up messages. @@ -347,10 +348,21 @@ sourceComponentId: "event_btn" context: sku: "sku-42" + expectOutbound: + - version: "1.0" + action: + name: "buy_item" + surfaceId: "adapter_surface" + sourceComponentId: "event_btn" + timestamp: + $match: "any" + context: + sku: "sku-42" - op: user_gesture targetNode: "func_btn" userActivation: true expectActions: [] + expectOutbound: [] expectDataModel: itemId: "sku-42" counter: 15 @@ -385,6 +397,7 @@ userActivation: true expectActions: [] expectErrors: [] + expectOutbound: [] - op: programmatic_invoke targetNode: "root" userActivation: false @@ -392,6 +405,7 @@ expectErrors: - surfaceId: "adapter_surface" message: "openUrl" + expectOutbound: [] - name: multi_catalog_mixed_surface_rendering description: Verifies that a single v1.0 surface renders components from both the default basic catalog and a secondary custom catalog via component-level catalogId overrides. @@ -631,6 +645,19 @@ surfaceId: "adapter_surface" - code: "CYCLIC_REFERENCE" surfaceId: "adapter_surface" + expectOutbound: + - version: "1.0" + error: + code: "UNKNOWN_COMPONENT_TYPE" + surfaceId: "adapter_surface" + message: + $match: "any" + - version: "1.0" + error: + code: "CYCLIC_REFERENCE" + surfaceId: "adapter_surface" + message: + $match: "any" - name: teardown_unmount_releases_subscriptions description: Verifies that unmounting the surface view disposes its NodeResolver and releases all active reactive subscriptions on the SurfaceModel and DataModel. diff --git a/conformance/tests/test_outbound_envelopes.py b/conformance/tests/test_outbound_envelopes.py new file mode 100644 index 0000000000..f93d7a9f8a --- /dev/null +++ b/conformance/tests/test_outbound_envelopes.py @@ -0,0 +1,233 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path +from typing import Any + +import jsonschema +import pytest +import referencing +import yaml + + +def _load_json_file(path: Path) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _load_yaml_file(path: Path) -> Any: + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +CONFORMANCE_DIR = Path(__file__).resolve().parent.parent +REPO_ROOT = CONFORMANCE_DIR.parent + +V10_COMMON_TYPES = _load_json_file( + REPO_ROOT / "specification" / "v1_0" / "json" / "common_types.json" +) +V10_RENDERER_TO_AGENT = _load_json_file( + REPO_ROOT / "specification" / "v1_0" / "json" / "renderer_to_agent.json" +) +V091_COMMON_TYPES = _load_json_file( + REPO_ROOT / "specification" / "v0_9_1" / "json" / "common_types.json" +) +V091_CLIENT_TO_SERVER = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v0_9/client_to_server.json", + **_load_json_file( + REPO_ROOT + / "specification" + / "v0_9_1" + / "json" + / "client_to_server.json" + ), +} + +V10_CATALOG_STUB = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v1_0/catalog.json", + "$defs": {"anyFunction": {"type": "object", "additionalProperties": True}}, +} +V091_CATALOG_STUB = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v0_9/catalog.json", + "$defs": {"anyFunction": {"type": "object", "additionalProperties": True}}, +} + +SPEC_REGISTRY = referencing.Registry().with_resources([ + ( + V10_COMMON_TYPES["$id"], + referencing.Resource.from_contents(V10_COMMON_TYPES), + ), + ( + V10_RENDERER_TO_AGENT["$id"], + referencing.Resource.from_contents(V10_RENDERER_TO_AGENT), + ), + ( + V10_CATALOG_STUB["$id"], + referencing.Resource.from_contents(V10_CATALOG_STUB), + ), + ( + V091_COMMON_TYPES["$id"], + referencing.Resource.from_contents(V091_COMMON_TYPES), + ), + ( + V091_CLIENT_TO_SERVER["$id"], + referencing.Resource.from_contents(V091_CLIENT_TO_SERVER), + ), + ( + V091_CATALOG_STUB["$id"], + referencing.Resource.from_contents(V091_CATALOG_STUB), + ), +]) + +V10_RENDERER_TO_AGENT_VALIDATOR = jsonschema.Draft202012Validator( + V10_RENDERER_TO_AGENT, registry=SPEC_REGISTRY +) +V091_CLIENT_TO_SERVER_VALIDATOR = jsonschema.Draft202012Validator( + V091_CLIENT_TO_SERVER, registry=SPEC_REGISTRY +) + +SAMPLE_VALUES_BY_KEY: dict[str, str] = { + "timestamp": "2026-01-01T00:00:00Z", + "functionCallId": "call-1", +} + + +def substitute_match_any(value: Any, field_name: str = "") -> Any: + """Replaces `{"$match": "any"}` wildcards with schema-valid sample values.""" + if isinstance(value, dict): + if value == {"$match": "any"}: + return SAMPLE_VALUES_BY_KEY.get(field_name, "sample") + return {k: substitute_match_any(v, k) for k, v in value.items()} + if isinstance(value, list): + return [substitute_match_any(item, field_name) for item in value] + return value + + +def validate_outbound_envelope(envelope: dict[str, Any]) -> None: + """Validates an expected outbound envelope against the protocol JSON schema.""" + substituted = substitute_match_any(envelope) + version = substituted.get("version") + if version == "1.0": + V10_RENDERER_TO_AGENT_VALIDATOR.validate(substituted) + elif version in ("v0.9", "v0.9.1"): + normalized = ( + {**substituted, "version": "v0.9"} + if version == "v0.9.1" + else substituted + ) + V091_CLIENT_TO_SERVER_VALIDATOR.validate(normalized) + else: + raise ValueError(f"Unsupported outbound envelope version: {version!r}") + + +def _adapter_yaml_files() -> list[Path]: + files: list[Path] = [] + core_adapter = CONFORMANCE_DIR / "core" / "adapter_conformance.yaml" + if core_adapter.exists(): + files.append(core_adapter) + adapter_dir = CONFORMANCE_DIR / "adapter" + if adapter_dir.exists(): + files.extend(sorted(adapter_dir.rglob("*.yaml"))) + return files + + +def _collect_outbound_envelopes() -> ( + list[tuple[str, str, int, int, dict[str, Any]]] +): + collected: list[tuple[str, str, int, int, dict[str, Any]]] = [] + for yaml_path in _adapter_yaml_files(): + rel_path = yaml_path.relative_to(CONFORMANCE_DIR).as_posix() + cases = _load_yaml_file(yaml_path) or [] + if not isinstance(cases, list): + continue + for case in cases: + if not isinstance(case, dict): + continue + case_name = str(case.get("name", "")) + for step_idx, step in enumerate(case.get("steps") or []): + if not isinstance(step, dict) or "expectOutbound" not in step: + continue + envelopes = step["expectOutbound"] + assert isinstance( + envelopes, list + ), f"{rel_path}::{case_name}[step {step_idx}] expectOutbound must be a list" + for env_idx, envelope in enumerate(envelopes): + collected.append((rel_path, case_name, step_idx, env_idx, envelope)) + return collected + + +OUTBOUND_ENVELOPES = _collect_outbound_envelopes() + + +def test_adapter_conformance_has_expect_outbound_assertions() -> None: + assert ( + OUTBOUND_ENVELOPES + ), "Expected at least one expectOutbound envelope in adapter conformance YAML" + + +@pytest.mark.parametrize( + ("rel_path", "case_name", "step_idx", "env_idx", "envelope"), + OUTBOUND_ENVELOPES, + ids=[ + f"{rel}::{case}[step{s}][{e}]" + for rel, case, s, e, _ in OUTBOUND_ENVELOPES + ], +) +def test_outbound_envelopes_match_protocol_schema( + rel_path: str, + case_name: str, + step_idx: int, + env_idx: int, + envelope: dict[str, Any], +) -> None: + try: + validate_outbound_envelope(envelope) + except jsonschema.ValidationError as exc: + pytest.fail( + f"Outbound envelope failed schema validation in " + f"{rel_path}::{case_name} [step {step_idx}, envelope {env_idx}]:\n" + f" Message: {exc.message}\n" + f" Path: {list(exc.absolute_path)}" + ) + + +def test_substitute_match_any_and_rejection_of_invalid_envelopes() -> None: + valid_rpc = { + "version": "1.0", + "callAgentFunction": { + "surfaceId": "main", + "functionCallId": {"$match": "any"}, + "callFunction": { + "@call": "fetchItems", + "args": {"limit": 10}, + }, + }, + } + validate_outbound_envelope(valid_rpc) + + invalid_action_missing_source = { + "version": "1.0", + "action": { + "name": "buy_item", + "surfaceId": "adapter_surface", + "timestamp": {"$match": "any"}, + "context": {}, + }, + } + with pytest.raises(jsonschema.ValidationError): + validate_outbound_envelope(invalid_action_missing_source) diff --git a/dart/a2ui_flutter/test/adapter_harness.dart b/dart/a2ui_flutter/test/adapter_harness.dart index 5cd35a3b64..2fbe02dca3 100644 --- a/dart/a2ui_flutter/test/adapter_harness.dart +++ b/dart/a2ui_flutter/test/adapter_harness.dart @@ -64,6 +64,7 @@ const Set adapterStepKnownKeys = { 'expectDataModel', 'expectActions', 'expectErrors', + 'expectOutbound', 'expectSameViewIdentity', 'expectActiveSubscriptions', }; @@ -214,6 +215,17 @@ void assertKnownKeys( ); } } + if (obj['expectOutbound'] case final List outboundList) { + for (var i = 0; i < outboundList.length; i++) { + final Object? item = outboundList[i]; + if (item is! Map) { + fail('Expected object at $where.expectOutbound[$i]'); + } + if (item['version'] is! String) { + fail('Expected string at $where.expectOutbound[$i].version'); + } + } + } } void assertKnownAdapterCase(Map testCase) { @@ -468,7 +480,7 @@ class _IncrementCounterFunction extends FunctionImplementation { } class AdapterHarnessSession { - AdapterHarnessSession([this.tester]) { + AdapterHarnessSession([this.tester, this.protocolVersion = '1.0']) { final CatalogImplementation v10Catalog = _buildV10BasicCatalog( renderedNodes: renderedNodes, openedUrls: openedUrls, @@ -479,20 +491,43 @@ class AdapterHarnessSession { processor = MessageProcessor( catalogs: [v0_9.basicCatalog(), v10Catalog, customCatalog], defaultVersion: A2uiProtocolVersion.v1_0, + outboundListener: (msg) { + outbound.add( + jsonDecode(jsonEncode(msg.toJson())) as Map, + ); + }, ); - processor.model.onAction.subscribe(actions.add); + processor.model.onAction.subscribe((action) { + actions.add(action); + outbound.add(_frameActionEnvelope(action, protocolVersion)); + }); processor.model.onSurfaceCreated.subscribe((created) { surface = created; - created.onError.subscribe(errors.add); + created.onError.subscribe((err) { + errors.add(err); + if (currentOp != 'programmatic_invoke') { + outbound.add( + jsonDecode( + jsonEncode( + ErrorMessage(version: protocolVersion, error: err).toJson(), + ), + ) + as Map, + ); + } + }); }); } WidgetTester? tester; + final String protocolVersion; + String? currentOp; late final MessageProcessor processor; SurfaceModel? surface; final Map> renderedNodes = {}; final List actions = []; final List errors = []; + final List> outbound = []; final List openedUrls = []; final Map savedElements = {}; final Map> savedNodes = {}; @@ -537,7 +572,10 @@ class AdapterHarnessSession { AdapterHarnessSession mountSurface([ Map? caseSpec, WidgetTester? tester, -]) => AdapterHarnessSession(tester); +]) => AdapterHarnessSession( + tester, + (caseSpec?['protocolVersion'] as String?) ?? '1.0', +); ComponentNode? extractViewTree( AdapterHarnessSession session, @@ -656,7 +694,9 @@ Future _executeStep( ) async { harness.actions.clear(); harness.errors.clear(); + harness.outbound.clear(); final op = step['op']! as String; + harness.currentOp = op; switch (op) { case 'process_messages': await harness.processMessages(tester, step['messages']! as List); @@ -777,6 +817,10 @@ void _assertStepExpectations( if (step['expectErrors'] case final List expectedErrors) { _assertErrors(harness.errors, expectedErrors); } + _validateOutboundEnvelopes(harness.outbound, harness.protocolVersion); + if (step['expectOutbound'] case final List expectedOutbound) { + _assertOutbound(harness.outbound, expectedOutbound); + } if (step['expectAccessibilityTree'] case final Map expectedA11y) { _assertAccessibilityTree(tester, expectedA11y); @@ -956,6 +1000,165 @@ void _assertErrors(List actual, List expectedList) { } } +Map _loadSpecSchema(String relativePath) { + final file = File('../../specification/$relativePath'); + final decoded = jsonDecode(file.readAsStringSync()) as Map; + decoded.remove(r'$schema'); + return decoded; +} + +class _OutboundSchemaOracle { + _OutboundSchemaOracle( + String versionDir, + String baseVersion, + String file, { + List siblings = const [], + }) { + final Uri base = Uri.parse('https://a2ui.org/specification/$baseVersion/'); + sourceUri = base.resolve(file); + registry.addSchema( + base.resolve('catalog.json'), + Schema.fromMap(const { + r'$defs': { + 'anyComponent': {}, + 'anyFunction': {}, + 'theme': {}, + }, + }), + ); + for (final sibling in siblings) { + registry.addSchema( + base.resolve(sibling), + Schema.fromMap(_loadSpecSchema('$versionDir/json/$sibling')), + ); + } + schema = Schema.fromMap(_loadSpecSchema('$versionDir/json/$file')); + } + + final SchemaRegistry registry = SchemaRegistry(); + late final Uri sourceUri; + late final Schema schema; + + List validate(Map envelope) => schema + .validateSync(envelope, sourceUri: sourceUri, schemaRegistry: registry); +} + +final _OutboundSchemaOracle _v10OutboundOracle = _OutboundSchemaOracle( + 'v1_0', + 'v1_0', + 'renderer_to_agent.json', + siblings: const ['common_types.json'], +); + +final _OutboundSchemaOracle _v09OutboundOracle = _OutboundSchemaOracle( + 'v0_9_1', + 'v0_9', + 'client_to_server.json', + siblings: const ['common_types.json'], +); + +Map _frameActionEnvelope( + A2uiRendererAction action, + String protocolVersion, +) { + final actionMap = + jsonDecode(jsonEncode(action.toJson())) as Map + ..remove('catalogId'); + return {'version': protocolVersion, 'action': actionMap}; +} + +void _validateOutboundEnvelopes( + List> envelopes, + String protocolVersion, +) { + final bool isV09 = + protocolVersion.startsWith('v0.9') || protocolVersion.startsWith('0.9'); + final _OutboundSchemaOracle oracle = isV09 + ? _v09OutboundOracle + : _v10OutboundOracle; + for (var i = 0; i < envelopes.length; i++) { + final List validationErrors = oracle.validate( + envelopes[i], + ); + expect( + validationErrors, + isEmpty, + reason: + 'outbound[$i] failed protocol schema validation: $validationErrors', + ); + } +} + +bool _isMatchAny(Object? value) => + value is Map && value.length == 1 && value[r'$match'] == 'any'; + +void _assertMatchesExpectedOutbound( + Object? actual, + Object? expected, + String where, +) { + if (_isMatchAny(expected)) { + expect( + actual, + isNotNull, + reason: '$where: expected non-null value for {r"\$match": "any"}', + ); + return; + } + if (expected is List) { + expect(actual, isA>(), reason: '$where: expected list'); + final actualList = actual! as List; + expect( + actualList, + hasLength(expected.length), + reason: '$where: list length mismatch', + ); + for (var i = 0; i < expected.length; i++) { + _assertMatchesExpectedOutbound(actualList[i], expected[i], '$where[$i]'); + } + return; + } + if (expected is Map) { + expect( + actual, + isA>(), + reason: '$where: expected object', + ); + final actualMap = actual! as Map; + final List actualKeys = actualMap.keys.toList()..sort(); + final List expectedKeys = + expected.keys.map((k) => k.toString()).toList()..sort(); + expect(actualKeys, equals(expectedKeys), reason: '$where: keys mismatch'); + for (final key in expectedKeys) { + _assertMatchesExpectedOutbound( + actualMap[key], + expected[key], + '$where.$key', + ); + } + return; + } + expect(actual, equals(expected), reason: '$where: value mismatch'); +} + +void _assertOutbound( + List> actual, + List expected, +) { + expect( + actual, + hasLength(expected.length), + reason: 'expectOutbound length mismatch', + ); + for (var i = 0; i < expected.length; i++) { + _assertMatchesExpectedOutbound( + actual[i], + expected[i], + 'expectOutbound[$i]', + ); + } +} + void _assertAccessibilityTree( WidgetTester tester, Map expectedTree, diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift index b133598feb..0fd8c5d8e2 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift @@ -57,10 +57,12 @@ struct AdapterManifestRecord: Sendable { } } -/// Captures actions and errors emitted by `MessageProcessor` during a conformance step. +/// Captures actions, errors, and outbound envelopes emitted by `MessageProcessor` during a step. final class AdapterTestActionHandler: ActionHandling, @unchecked Sendable { var actions: [(action: ResolvedAction, surfaceID: String)] = [] var errors: [RendererError] = [] + var outbound: [[String: Any]] = [] + var currentOp: String? func handle(action: ResolvedAction, from surfaceID: String) { actions.append((action: action, surfaceID: surfaceID)) @@ -70,9 +72,23 @@ final class AdapterTestActionHandler: ActionHandling, @unchecked Sendable { errors.append(error) } + func recordOutbound(_ message: RendererToAgentMessage) { + if currentOp == "programmatic_invoke", case .error = message { + return + } + guard + let data = try? JSONEncoder().encode(message), + let dict = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] + else { + return + } + outbound.append(dict) + } + func clearStepEvents() { actions.removeAll() errors.removeAll() + outbound.removeAll() } } @@ -161,6 +177,7 @@ final class MountedSurfaceHarness { @MainActor final class AdapterSession { let caseName: String + let protocolVersion: String let handler: AdapterTestActionHandler let processor: MessageProcessor let uiCatalogs: [Catalog] @@ -170,11 +187,13 @@ final class AdapterSession { init( caseName: String, + protocolVersion: String = "1.0", handler: AdapterTestActionHandler, processor: MessageProcessor, uiCatalogs: [Catalog] ) { self.caseName = caseName + self.protocolVersion = protocolVersion self.handler = handler self.processor = processor self.uiCatalogs = uiCatalogs @@ -226,6 +245,7 @@ enum AdapterHarness { "expectDataModel", "expectActions", "expectErrors", + "expectOutbound", "expectSameViewIdentity", "expectActiveSubscriptions", ] @@ -385,6 +405,20 @@ enum AdapterHarness { ) } } + if let rawOutbound = obj["expectOutbound"] { + guard let outbound = rawOutbound as? [[String: Any]] else { + let msg = "\(location).expectOutbound: expected array of objects" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + for (idx, item) in outbound.enumerated() { + guard item["version"] is String else { + let msg = "\(location).expectOutbound[\(idx)].version: expected string" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + } + } } static func assertKnownAdapterCase(_ testCase: [String: Any]) throws { @@ -579,15 +613,20 @@ enum AdapterHarness { /// Creates an `AdapterSession` configured for `caseSpec`. static func mountSurface(_ caseSpec: [String: Any]) throws -> AdapterSession { let caseName = caseSpec["name"] as? String ?? "" + let protocolVersion = (caseSpec["protocolVersion"] as? String) ?? "1.0" let handler = AdapterTestActionHandler() let (coreCatalogs, uiCatalogs) = try buildCatalogs(for: caseSpec) let processor = MessageProcessor( catalogs: coreCatalogs, actionHandler: handler, - validationConfig: .relaxed + validationConfig: .relaxed, + outboundListener: { message in + handler.recordOutbound(message) + } ) return AdapterSession( caseName: caseName, + protocolVersion: protocolVersion, handler: handler, processor: processor, uiCatalogs: uiCatalogs @@ -610,6 +649,7 @@ enum AdapterHarness { let stepCtx = context ?? session.caseName session.handler.clearStepEvents() let op = step["op"] as? String ?? "process_messages" + session.handler.currentOp = op switch op { case "process_messages": let messages = (step["messages"] as? [[String: Any]]) ?? [] @@ -903,6 +943,18 @@ enum AdapterHarness { if let expErrors = step["expectErrors"] as? [[String: Any]] { try assertErrorsMatch(session.handler.errors, expected: expErrors, context: context) } + try validateOutboundEnvelopes( + session.handler.outbound, + protocolVersion: session.protocolVersion, + context: context + ) + if let expOutbound = step["expectOutbound"] as? [[String: Any]] { + try assertOutboundMatches( + session.handler.outbound, + expected: expOutbound, + context: context + ) + } if let expA11y = step["expectAccessibilityTree"] as? [String: [String: Any]], let currentTree = session.currentTree { @@ -1140,6 +1192,133 @@ enum AdapterHarness { } } + private static let v10OutboundSchema: Schema = loadSpecificationSchema( + relativePath: "v1_0/json/renderer_to_agent.json" + ) + + private static let v09OutboundSchema: Schema = loadSpecificationSchema( + relativePath: "v0_9_1/json/client_to_server.json" + ) + + private static func loadSpecificationSchema(relativePath: String) -> Schema { + let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + let schemaURL = repoRoot.appendingPathComponent("specification").appendingPathComponent( + relativePath + ) + let text = (try? String(contentsOf: schemaURL, encoding: .utf8)) ?? "{}" + let rawJSON = (try? JSONValue.parse(text)) ?? .object([:]) + return try! Schema(rawSchema: rawJSON, context: A2UISchemaRegistry.makeContext()) + } + + private static func validateOutboundEnvelopes( + _ envelopes: [[String: Any]], + protocolVersion: String, + context: String + ) throws { + let isV09 = protocolVersion.hasPrefix("v0.9") || protocolVersion.hasPrefix("0.9") + let schema = isV09 ? v09OutboundSchema : v10OutboundSchema + for (idx, envelope) in envelopes.enumerated() { + let jsonValue = toJSONValue(envelope) + let result = schema.validate(jsonValue) + #expect( + result.isValid, + "\(context): outbound[\(idx)] failed protocol schema validation" + ) + if !result.isValid { + throw AdapterConformanceError( + "\(context): outbound[\(idx)] failed protocol schema validation" + ) + } + } + } + + private static func isMatchAny(_ value: Any) -> Bool { + guard let dict = value as? [String: Any], dict.count == 1 else { return false } + return (dict["$match"] as? String) == "any" + } + + private static func assertMatchesExpectedOutbound( + _ actual: Any, + expected: Any, + where location: String + ) throws { + if isMatchAny(expected) { + let isNonNull = !(actual is NSNull) + #expect(isNonNull, "\(location): expected non-null value for {$match: \"any\"}") + if !isNonNull { + throw AdapterConformanceError("\(location): expected non-null value") + } + return + } + if let expArray = expected as? [Any] { + guard let actArray = actual as? [Any], actArray.count == expArray.count else { + let msg = "\(location): array length mismatch" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + for idx in 0..; + properties?: Record; } >; } @@ -128,6 +130,7 @@ export interface AdapterStep { expectDataModel?: Record; expectActions?: ExpectedActionEvent[]; expectErrors?: ExpectedErrorEntry[]; + expectOutbound?: Array>; expectSameViewIdentity?: string[]; expectActiveSubscriptions?: number; } @@ -156,6 +159,9 @@ export interface HarnessState { unmounted: boolean; actions: Array>; errors: Array>; + outbound: Array>; + protocolVersion: string; + currentOp?: string; previousNodes: Map; previousDomElements: Map; } @@ -200,6 +206,7 @@ export const KNOWN_KEYS = { 'expectDataModel', 'expectActions', 'expectErrors', + 'expectOutbound', 'expectSameViewIdentity', 'expectActiveSubscriptions', ]), @@ -249,9 +256,9 @@ function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: str if (typeof obj !== 'object' || Array.isArray(obj)) { throw new Error(`${where}: expected object`); } - const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); + const unknown = Object.keys(obj as Record).filter((key) => !allowed.has(key)); if (unknown.length > 0) { - throw new Error(`${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`); + throw new Error(`${where}: unknown keys: ${unknown.map((k) => `${where}.${k}`).join(', ')}`); } } @@ -325,6 +332,19 @@ export function assertKnownKeys( assertObjectKeys(item, KNOWN_KEYS.expectedError, `${where}.expectErrors[${idx}]`); }); } + if (step['expectOutbound'] !== undefined) { + if (!Array.isArray(step['expectOutbound'])) { + throw new Error(`${where}.expectOutbound: expected array`); + } + (step['expectOutbound'] as unknown[]).forEach((item, idx) => { + if (typeof item !== 'object' || item === null || Array.isArray(item)) { + throw new Error(`${where}.expectOutbound[${idx}]: expected object`); + } + if (typeof (item as Record)['version'] !== 'string') { + throw new Error(`${where}.expectOutbound[${idx}].version: expected string`); + } + }); + } } export function assertKnownAdapterCase(testCase: AdapterTestCase): void { @@ -407,7 +427,7 @@ export function findUnsupportedReason( } export function assertAllUnsupportedMatched(): void { - const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); + const unused = Object.keys(UNSUPPORTED).filter((key) => !matchedUnsupportedKeys.has(key)); if (unused.length > 0) { throw new Error( `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, @@ -418,10 +438,10 @@ export function assertAllUnsupportedMatched(): void { export function writeManifestRecord(rec: ManifestRecord): void { const karma = ( window as unknown as { - __karma__?: {info?: (payload: {conformanceManifestRecord: ManifestRecord}) => void}; + __karma__?: { info?: (payload: { conformanceManifestRecord: ManifestRecord }) => void }; } ).__karma__; - karma?.info?.({conformanceManifestRecord: rec}); + karma?.info?.({ conformanceManifestRecord: rec }); } const RESERVED_VIEW_KEYS = new Set([ @@ -461,7 +481,7 @@ class ConformanceCustomWidgetComponent extends CatalogComponent(resolve => setTimeout(resolve, 0)); + await new Promise((resolve) => setTimeout(resolve, 0)); const elements = Array.from(container.querySelectorAll('*')); for (const el of elements) { - const litEl = el as unknown as {updateComplete?: Promise}; + const litEl = el as unknown as { updateComplete?: Promise }; if (litEl.updateComplete) { await litEl.updateComplete; } @@ -642,7 +662,7 @@ function findDomElementForNode( node: ComponentNode, ): HTMLElement | undefined { const match = Array.from(container.querySelectorAll('*')).find( - el => (el as A2uiWebComponentElement).node === node, + (el) => (el as A2uiWebComponentElement).node === node, ); return match as HTMLElement | undefined; } @@ -706,7 +726,7 @@ export function assertViewTree( function isNestedComponentElement(host: HTMLElement, candidate: Element): boolean { let current: Element | null = candidate; while (current && current !== host) { - const a2uiEl = current as unknown as {context?: unknown; node?: unknown}; + const a2uiEl = current as unknown as { context?: unknown; node?: unknown }; if (a2uiEl.context || a2uiEl.node) { return true; } @@ -830,19 +850,19 @@ export function assertAxeRules(container: HTMLElement, rules: string[]): void { function countActiveSubscriptions(surface: SurfaceModel): number { const createdListeners = - (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? - 0; + (surface.componentsModel.onCreated as unknown as { listeners?: Set }).listeners + ?.size ?? 0; const deletedListeners = - (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? - 0; + (surface.componentsModel.onDeleted as unknown as { listeners?: Set }).listeners + ?.size ?? 0; let componentUpdatedListeners = 0; for (const comp of surface.componentsModel.values) { componentUpdatedListeners += - (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; + (comp.onUpdated as unknown as { listeners?: Set }).listeners?.size ?? 0; } const dataModelInternal = surface.dataModel as unknown as { subscriptions?: Set; - signals?: Map; + signals?: Map; }; let activeSignalTargets = 0; for (const sig of dataModelInternal.signals?.values() ?? []) { @@ -876,8 +896,8 @@ async function performStepOperation(state: HarnessState, step: AdapterStep): Pro const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; expect(input).withContext(`Input element for "${targetId}"`).not.toBeNull(); input!.value = String(step.inputValue ?? ''); - input!.dispatchEvent(new Event('input', {bubbles: true, composed: true})); - input!.dispatchEvent(new Event('change', {bubbles: true, composed: true})); + input!.dispatchEvent(new Event('input', { bubbles: true, composed: true })); + input!.dispatchEvent(new Event('change', { bubbles: true, composed: true })); return; } if (step.op === 'user_gesture') { @@ -979,7 +999,7 @@ function mapNodesToDomElements( const result = new Map(); const allElements = Array.from(container.querySelectorAll('*')); for (const [nodePath, node] of nodes) { - const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); + const match = allElements.find((el) => (el as A2uiWebComponentElement).node === node); if (match) { result.set(nodePath, match as HTMLElement); } @@ -987,14 +1007,145 @@ function mapNodesToDomElements( return result; } +function frameActionEnvelope( + action: Record, + protocolVersion: string, +): Record { + const payload: Record = { + name: action['name'], + surfaceId: action['surfaceId'], + sourceComponentId: action['sourceComponentId'], + timestamp: action['timestamp'], + context: action['context'] ?? {}, + }; + if (action['userMessage'] !== undefined) { + payload['userMessage'] = action['userMessage']; + } + if (action['metadata'] !== undefined) { + payload['metadata'] = action['metadata']; + } + return { + version: protocolVersion, + action: payload, + }; +} + +function frameErrorEnvelope( + err: Record, + surfaceId: string, + protocolVersion: string, +): Record { + const payload: Record = { + code: err['code'], + surfaceId: typeof err['surfaceId'] === 'string' ? err['surfaceId'] : surfaceId, + message: err['message'], + }; + if (typeof err['path'] === 'string') { + payload['path'] = err['path']; + } + if (typeof err['functionCallId'] === 'string') { + payload['functionCallId'] = err['functionCallId']; + } + return { + version: protocolVersion, + error: payload, + }; +} + +function isMatchAny(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const keys = Object.keys(value as Record); + return keys.length === 1 && (value as Record)['$match'] === 'any'; +} + +function assertMatchesExpectedOutbound(actual: unknown, expected: unknown, where: string): void { + if (isMatchAny(expected)) { + expect(actual !== undefined && actual !== null) + .withContext(`${where}: expected defined non-null`) + .toBe(true); + return; + } + if (Array.isArray(expected)) { + expect(Array.isArray(actual)).withContext(`${where}: expected array`).toBe(true); + expect((actual as unknown[]).length) + .withContext(`${where}: array length`) + .toBe(expected.length); + for (let i = 0; i < expected.length; i++) { + assertMatchesExpectedOutbound((actual as unknown[])[i], expected[i], `${where}[${i}]`); + } + return; + } + if (typeof expected === 'object' && expected !== null) { + expect(typeof actual === 'object' && actual !== null && !Array.isArray(actual)) + .withContext(`${where}: expected object`) + .toBe(true); + const actualRecord = actual as Record; + const expectedRecord = expected as Record; + const actualKeys = Object.keys(actualRecord).sort(); + const expectedKeys = Object.keys(expectedRecord).sort(); + expect(actualKeys).withContext(`${where}: object keys`).toEqual(expectedKeys); + for (const key of expectedKeys) { + assertMatchesExpectedOutbound(actualRecord[key], expectedRecord[key], `${where}.${key}`); + } + return; + } + expect(actual).withContext(`${where}: value`).toBe(expected); +} + +function validateOutboundEnvelopes( + envelopes: Array>, + protocolVersion: string, +): void { + const isV09 = protocolVersion.startsWith('v0.9') || protocolVersion.startsWith('0.9'); + for (let i = 0; i < envelopes.length; i++) { + const env = envelopes[i]; + const parsed = isV09 + ? A2uiClientMessageSchema.safeParse(env) + : RendererToAgentMessageSchema.safeParse(env); + expect(parsed.success) + .withContext( + `outbound[${i}] failed protocol schema validation: ${ + parsed.success ? '' : JSON.stringify(parsed.error.issues) + }`, + ) + .toBe(true); + } +} + +function assertExpectedOutbound( + actual: Array>, + expected: Array> | undefined, +): void { + if (expected === undefined) { + return; + } + expect(actual.length).withContext('expectOutbound length').toBe(expected.length); + for (let i = 0; i < expected.length; i++) { + assertMatchesExpectedOutbound(actual[i], expected[i], `expectOutbound[${i}]`); + } +} + export function mountSurface(caseSpec?: Partial): HarnessState { const catalogs: AngularCatalog[] = [ createDefaultAdapterCatalog(), ...(caseSpec?.catalogs ?? []).map(buildExtraCatalog), ]; + const outbound: Array> = []; + const protocolVersion = caseSpec?.protocolVersion ?? '1.0'; TestBed.configureTestingModule({ imports: [SurfaceComponent], - providers: [provideA2UI({catalogs})], + providers: [ + provideA2UI({ + catalogs, + processorOptions: { + outboundListener: (msg) => { + outbound.push(structuredClone(msg) as unknown as Record); + }, + }, + }), + ], }); const rendererService = TestBed.inject(A2uiRendererService); const state: HarnessState = { @@ -1002,16 +1153,30 @@ export function mountSurface(caseSpec?: Partial): HarnessState unmounted: false, actions: [], errors: [], + outbound, + protocolVersion, previousNodes: new Map(), previousDomElements: new Map(), }; - rendererService.surfaceGroup.onSurfaceCreated.subscribe(surface => { + rendererService.surfaceGroup.onSurfaceCreated.subscribe((surface) => { state.surface = surface; - surface.onAction.subscribe(action => { + surface.onAction.subscribe((action) => { state.actions.push(action as unknown as Record); + state.outbound.push( + frameActionEnvelope(action as unknown as Record, state.protocolVersion), + ); }); - surface.onError.subscribe(err => { + surface.onError.subscribe((err) => { state.errors.push(err as unknown as Record); + if (state.currentOp !== 'programmatic_invoke') { + state.outbound.push( + frameErrorEnvelope( + err as unknown as Record, + surface.id, + state.protocolVersion, + ), + ); + } }); }); return state; @@ -1021,6 +1186,8 @@ export async function runStep(state: HarnessState, step: AdapterStep): Promise 0) { state.rendererService.processMessages(step.messages); } @@ -1052,6 +1219,8 @@ export async function runStep(state: HarnessState, step: AdapterStep): Promise>; expectAccessibilityTree?: Record; expectActiveSubscriptions?: number; } @@ -143,6 +146,9 @@ export interface AdapterSession { mountedElements: HTMLElement[]; recordedActions: ExpectedAction[]; recordedErrors: Array<{code: string; message: string; surfaceId: string}>; + recordedOutbound: Array>; + protocolVersion: string; + currentOp?: string; priorIdentities: Map; } @@ -186,6 +192,7 @@ export const KNOWN_KEYS = { 'expectDataModel', 'expectActions', 'expectErrors', + 'expectOutbound', 'expectSameViewIdentity', 'expectActiveSubscriptions', ]), @@ -307,6 +314,20 @@ export function assertKnownKeys( assertObjectKeys(item, KNOWN_KEYS.expectedError, `${where}.expectErrors[${idx}]`); }); } + if (step.expectOutbound !== undefined) { + assert.ok(Array.isArray(step.expectOutbound), `${where}.expectOutbound: expected array`); + step.expectOutbound.forEach((item, idx) => { + assert.ok( + typeof item === 'object' && item !== null && !Array.isArray(item), + `${where}.expectOutbound[${idx}]: expected object`, + ); + assert.equal( + typeof (item as Record).version, + 'string', + `${where}.expectOutbound[${idx}].version: expected string`, + ); + }); + } } export function assertKnownAdapterCase(testCase: AdapterTestCase): void { @@ -808,6 +829,126 @@ function countActiveSurfaceSubscriptions(surface: SurfaceModel) return onCreatedSize + onDeletedSize + dataSubsSize; } +function frameActionEnvelope( + action: Record, + protocolVersion: string, +): Record { + const payload: Record = { + name: action.name, + surfaceId: action.surfaceId, + sourceComponentId: action.sourceComponentId, + timestamp: action.timestamp, + context: action.context ?? {}, + }; + if (action.userMessage !== undefined) { + payload.userMessage = action.userMessage; + } + if (action.metadata !== undefined) { + payload.metadata = action.metadata; + } + return { + version: protocolVersion, + action: payload, + }; +} + +function frameErrorEnvelope( + err: Record, + surfaceId: string, + protocolVersion: string, +): Record { + const payload: Record = { + code: err.code, + surfaceId: typeof err.surfaceId === 'string' ? err.surfaceId : surfaceId, + message: err.message, + }; + if (typeof err.path === 'string') { + payload.path = err.path; + } + if (typeof err.functionCallId === 'string') { + payload.functionCallId = err.functionCallId; + } + return { + version: protocolVersion, + error: payload, + }; +} + +function isMatchAny(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const keys = Object.keys(value as Record); + return keys.length === 1 && (value as Record)['$match'] === 'any'; +} + +function assertMatchesExpectedOutbound(actual: unknown, expected: unknown, where: string): void { + if (isMatchAny(expected)) { + assert.ok( + actual !== undefined && actual !== null, + `${where}: expected defined non-null value for {$match: "any"}`, + ); + return; + } + if (Array.isArray(expected)) { + assert.ok(Array.isArray(actual), `${where}: expected array`); + assert.equal(actual.length, expected.length, `${where}: array length mismatch`); + for (let i = 0; i < expected.length; i++) { + assertMatchesExpectedOutbound(actual[i], expected[i], `${where}[${i}]`); + } + return; + } + if (typeof expected === 'object' && expected !== null) { + assert.ok( + typeof actual === 'object' && actual !== null && !Array.isArray(actual), + `${where}: expected object`, + ); + const actualRecord = actual as Record; + const expectedRecord = expected as Record; + const actualKeys = Object.keys(actualRecord).sort(); + const expectedKeys = Object.keys(expectedRecord).sort(); + assert.deepEqual(actualKeys, expectedKeys, `${where}: object keys mismatch`); + for (const key of expectedKeys) { + assertMatchesExpectedOutbound(actualRecord[key], expectedRecord[key], `${where}.${key}`); + } + return; + } + assert.equal(actual, expected, `${where}: value mismatch`); +} + +function validateOutboundEnvelopes( + envelopes: Array>, + protocolVersion: string, +): void { + const isV09 = protocolVersion.startsWith('v0.9') || protocolVersion.startsWith('0.9'); + for (let i = 0; i < envelopes.length; i++) { + const env = envelopes[i]; + const parsed = isV09 + ? A2uiClientMessageSchema.safeParse(env) + : RendererToAgentMessageSchema.safeParse(env); + assert.ok( + parsed.success, + `recordedOutbound[${i}] failed protocol schema validation: ${ + parsed.success ? '' : JSON.stringify(parsed.error.issues) + }`, + ); + } +} + +function assertExpectedOutbound( + actual: Array>, + expected: Array>, +): void { + assert.equal( + actual.length, + expected.length, + `expectOutbound length mismatch: actual=${actual.length}, expected=${expected.length}`, + ); + for (let i = 0; i < expected.length; i++) { + assertMatchesExpectedOutbound(actual[i], expected[i], `expectOutbound[${i}]`); + } +} + export async function ensureSurfaceMounted( session: AdapterSession, mountedElements: HTMLElement[] = session.mountedElements, @@ -829,6 +970,9 @@ export async function ensureSurfaceMounted( sourceComponentId: action.sourceComponentId, context: action.context, }); + session.recordedOutbound.push( + frameActionEnvelope(action as unknown as Record, session.protocolVersion), + ); }); surface.onError.subscribe(err => { session.recordedErrors.push({ @@ -836,6 +980,15 @@ export async function ensureSurfaceMounted( message: err.message, surfaceId: surface.id, }); + if (session.currentOp !== 'programmatic_invoke') { + session.recordedOutbound.push( + frameErrorEnvelope( + err as unknown as Record, + surface.id, + session.protocolVersion, + ), + ); + } }); const el = document.createElement('a2ui-surface') as unknown as A2uiSurface; @@ -945,6 +1098,10 @@ function verifyStepExpectations(session: AdapterSession, step: AdapterStep): voi if (step.expectErrors !== undefined) { assertRecordedErrors(session.recordedErrors, step.expectErrors); } + validateOutboundEnvelopes(session.recordedOutbound, session.protocolVersion); + if (step.expectOutbound !== undefined) { + assertExpectedOutbound(session.recordedOutbound, step.expectOutbound); + } if (step.expectAccessibilityTree && session.surfaceEl) { for (const [compId, expected] of Object.entries(step.expectAccessibilityTree)) { assertAccessibilityNode(session.surfaceEl, compId, expected); @@ -963,11 +1120,20 @@ export function mountSurface( createAugmentedBasicCatalog(), ...(caseSpec?.catalogs ?? []).map(buildSecondaryCatalog), ]; + const recordedOutbound: Array> = []; + const protocolVersion = caseSpec?.protocolVersion ?? '1.0'; + const processor = new MessageProcessor(catalogs, undefined, { + outboundListener: msg => { + recordedOutbound.push(structuredClone(msg) as unknown as Record); + }, + }); return { - processor: new MessageProcessor(catalogs), + processor, mountedElements, recordedActions: [], recordedErrors: [], + recordedOutbound, + protocolVersion, priorIdentities: new Map(), }; } @@ -976,6 +1142,8 @@ export async function runStep(session: AdapterSession, step: AdapterStep): Promi assertKnownKeys(step); session.recordedActions = []; session.recordedErrors = []; + session.recordedOutbound.length = 0; + session.currentOp = step.op; session.priorIdentities = snapshotViewIdentities(session.surfaceEl); if (step.op === 'process_messages' && !session.surfaceEl) { diff --git a/typescript/renderers/react/tests/adapter_harness.tsx b/typescript/renderers/react/tests/adapter_harness.tsx index 99f9ac5370..b1ae488e74 100644 --- a/typescript/renderers/react/tests/adapter_harness.tsx +++ b/typescript/renderers/react/tests/adapter_harness.tsx @@ -35,8 +35,9 @@ import { type SurfaceModel, } from '@a2ui/web_core'; import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; -import {CommonSchemas} from '@a2ui/web_core/v0_9'; +import {A2uiClientMessageSchema, CommonSchemas} from '@a2ui/web_core/v0_9'; import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; +import {RendererToAgentMessageSchema} from '@a2ui/web_core/v1_0'; import {A2uiSurface, createComponentImplementation, type ReactCatalogComponent} from '../src'; export interface ExpectedAccessibilityNode { @@ -126,6 +127,7 @@ export interface AdapterStep { expectDataModel?: Record; expectActions?: ExpectedActionEvent[]; expectErrors?: ExpectedErrorEntry[]; + expectOutbound?: Array>; expectSameViewIdentity?: string[]; expectActiveSubscriptions?: number; } @@ -148,6 +150,9 @@ export interface HarnessState { unmounted: boolean; actions: Array>; errors: Array>; + outbound: Array>; + protocolVersion: string; + currentOp?: string; previousNodes: Map; previousDomElements: Map; } @@ -192,6 +197,7 @@ export const KNOWN_KEYS = { 'expectDataModel', 'expectActions', 'expectErrors', + 'expectOutbound', 'expectSameViewIdentity', 'expectActiveSubscriptions', ]), @@ -312,6 +318,21 @@ export function assertKnownKeys( assertObjectKeys(item, KNOWN_KEYS.expectedError, `${where}.expectErrors[${idx}]`); }); } + if (step.expectOutbound !== undefined) { + expect(Array.isArray(step.expectOutbound), `${where}.expectOutbound: expected array`).toBe( + true, + ); + (step.expectOutbound as unknown[]).forEach((item, idx) => { + expect( + typeof item === 'object' && item !== null && !Array.isArray(item), + `${where}.expectOutbound[${idx}]: expected object`, + ).toBe(true); + expect( + typeof (item as Record).version, + `${where}.expectOutbound[${idx}].version`, + ).toBe('string'); + }); + } } export function assertKnownAdapterCase(testCase: AdapterTestCase): void { @@ -953,17 +974,143 @@ function mapNodesToDomElements( return result; } +function frameActionEnvelope( + action: Record, + protocolVersion: string, +): Record { + const payload: Record = { + name: action.name, + surfaceId: action.surfaceId, + sourceComponentId: action.sourceComponentId, + timestamp: action.timestamp, + context: action.context ?? {}, + }; + if (action.userMessage !== undefined) { + payload.userMessage = action.userMessage; + } + if (action.metadata !== undefined) { + payload.metadata = action.metadata; + } + return { + version: protocolVersion, + action: payload, + }; +} + +function frameErrorEnvelope( + err: Record, + surfaceId: string, + protocolVersion: string, +): Record { + const payload: Record = { + code: err.code, + surfaceId: typeof err.surfaceId === 'string' ? err.surfaceId : surfaceId, + message: err.message, + }; + if (typeof err.path === 'string') { + payload.path = err.path; + } + if (typeof err.functionCallId === 'string') { + payload.functionCallId = err.functionCallId; + } + return { + version: protocolVersion, + error: payload, + }; +} + +function isMatchAny(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + const keys = Object.keys(value as Record); + return keys.length === 1 && (value as Record)['$match'] === 'any'; +} + +function assertMatchesExpectedOutbound(actual: unknown, expected: unknown, where: string): void { + if (isMatchAny(expected)) { + expect(actual !== undefined && actual !== null, `${where}: expected defined non-null`).toBe( + true, + ); + return; + } + if (Array.isArray(expected)) { + expect(Array.isArray(actual), `${where}: expected array`).toBe(true); + expect((actual as unknown[]).length, `${where}: array length`).toBe(expected.length); + for (let i = 0; i < expected.length; i++) { + assertMatchesExpectedOutbound((actual as unknown[])[i], expected[i], `${where}[${i}]`); + } + return; + } + if (typeof expected === 'object' && expected !== null) { + expect( + typeof actual === 'object' && actual !== null && !Array.isArray(actual), + `${where}: expected object`, + ).toBe(true); + const actualRecord = actual as Record; + const expectedRecord = expected as Record; + const actualKeys = Object.keys(actualRecord).sort(); + const expectedKeys = Object.keys(expectedRecord).sort(); + expect(actualKeys, `${where}: object keys`).toEqual(expectedKeys); + for (const key of expectedKeys) { + assertMatchesExpectedOutbound(actualRecord[key], expectedRecord[key], `${where}.${key}`); + } + return; + } + expect(actual, `${where}: value`).toBe(expected); +} + +function validateOutboundEnvelopes( + envelopes: Array>, + protocolVersion: string, +): void { + const isV09 = protocolVersion.startsWith('v0.9') || protocolVersion.startsWith('0.9'); + for (let i = 0; i < envelopes.length; i++) { + const env = envelopes[i]; + const parsed = isV09 + ? A2uiClientMessageSchema.safeParse(env) + : RendererToAgentMessageSchema.safeParse(env); + expect( + parsed.success, + `outbound[${i}] failed protocol schema validation: ${ + parsed.success ? '' : JSON.stringify(parsed.error.issues) + }`, + ).toBe(true); + } +} + +function assertExpectedOutbound( + actual: Array>, + expected: Array> | undefined, +): void { + if (expected === undefined) { + return; + } + expect(actual, 'expectOutbound length').toHaveLength(expected.length); + for (let i = 0; i < expected.length; i++) { + assertMatchesExpectedOutbound(actual[i], expected[i], `expectOutbound[${i}]`); + } +} + export function mountSurface(caseSpec?: Partial): HarnessState { const catalogs: Array> = [ createDefaultAdapterCatalog(), ...(caseSpec?.catalogs ?? []).map(buildExtraCatalog), ]; - const processor = new MessageProcessor(catalogs); + const outbound: Array> = []; + const protocolVersion = caseSpec?.protocolVersion ?? '1.0'; + const processor = new MessageProcessor(catalogs, undefined, { + outboundListener: msg => { + outbound.push(structuredClone(msg) as unknown as Record); + }, + }); const state: HarnessState = { processor, unmounted: false, actions: [], errors: [], + outbound, + protocolVersion, previousNodes: new Map(), previousDomElements: new Map(), }; @@ -971,9 +1118,21 @@ export function mountSurface(caseSpec?: Partial): HarnessState state.surface = surface; surface.onAction.subscribe(action => { state.actions.push(action as unknown as Record); + state.outbound.push( + frameActionEnvelope(action as unknown as Record, state.protocolVersion), + ); }); surface.onError.subscribe(err => { state.errors.push(err as unknown as Record); + if (state.currentOp !== 'programmatic_invoke') { + state.outbound.push( + frameErrorEnvelope( + err as unknown as Record, + surface.id, + state.protocolVersion, + ), + ); + } }); }); return state; @@ -983,6 +1142,8 @@ export async function runStep(state: HarnessState, step: AdapterStep): Promise 0) { await act(async () => { state.processor.processMessages(step.messages!); @@ -1015,6 +1176,8 @@ export async function runStep(state: HarnessState, step: AdapterStep): Promise Date: Tue, 3 Nov 2026 00:10:03 +0000 Subject: [PATCH 35/48] fix(conformance): intersect excluded.keys() with corpus.keys() in check_executed_cases --- conformance/scripts/check_executed_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/scripts/check_executed_cases.py b/conformance/scripts/check_executed_cases.py index bc02f71684..f1e74d05ff 100644 --- a/conformance/scripts/check_executed_cases.py +++ b/conformance/scripts/check_executed_cases.py @@ -483,7 +483,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) print( f"OK ({args.harness}): verified {total_cases} cases across" - f" {len(corpus) - len(excluded & corpus.keys())} active suites." + f" {len(corpus) - len(excluded.keys() & corpus.keys())} active suites." ) return 0 From a86c96f3b9d5979d5b986eec42d46d594fd74d48 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 00:26:47 +0000 Subject: [PATCH 36/48] style: format files with fix_format.sh --- conformance/tests/test_outbound_envelopes.py | 198 +++++++++--------- .../A2UISwiftUITests/AdapterHarness.swift | 8 +- .../renderers/angular/src/adapter_harness.ts | 62 +++--- 3 files changed, 129 insertions(+), 139 deletions(-) diff --git a/conformance/tests/test_outbound_envelopes.py b/conformance/tests/test_outbound_envelopes.py index f93d7a9f8a..c50e0fe747 100644 --- a/conformance/tests/test_outbound_envelopes.py +++ b/conformance/tests/test_outbound_envelopes.py @@ -23,13 +23,13 @@ def _load_json_file(path: Path) -> dict[str, Any]: - with open(path, "r", encoding="utf-8") as f: - return json.load(f) + with open(path, "r", encoding="utf-8") as f: + return json.load(f) def _load_yaml_file(path: Path) -> Any: - with open(path, "r", encoding="utf-8") as f: - return yaml.safe_load(f) + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) CONFORMANCE_DIR = Path(__file__).resolve().parent.parent @@ -48,11 +48,7 @@ def _load_yaml_file(path: Path) -> Any: "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://a2ui.org/specification/v0_9/client_to_server.json", **_load_json_file( - REPO_ROOT - / "specification" - / "v0_9_1" - / "json" - / "client_to_server.json" + REPO_ROOT / "specification" / "v0_9_1" / "json" / "client_to_server.json" ), } @@ -108,85 +104,79 @@ def _load_yaml_file(path: Path) -> Any: def substitute_match_any(value: Any, field_name: str = "") -> Any: - """Replaces `{"$match": "any"}` wildcards with schema-valid sample values.""" - if isinstance(value, dict): - if value == {"$match": "any"}: - return SAMPLE_VALUES_BY_KEY.get(field_name, "sample") - return {k: substitute_match_any(v, k) for k, v in value.items()} - if isinstance(value, list): - return [substitute_match_any(item, field_name) for item in value] - return value + """Replaces `{"$match": "any"}` wildcards with schema-valid sample values.""" + if isinstance(value, dict): + if value == {"$match": "any"}: + return SAMPLE_VALUES_BY_KEY.get(field_name, "sample") + return {k: substitute_match_any(v, k) for k, v in value.items()} + if isinstance(value, list): + return [substitute_match_any(item, field_name) for item in value] + return value def validate_outbound_envelope(envelope: dict[str, Any]) -> None: - """Validates an expected outbound envelope against the protocol JSON schema.""" - substituted = substitute_match_any(envelope) - version = substituted.get("version") - if version == "1.0": - V10_RENDERER_TO_AGENT_VALIDATOR.validate(substituted) - elif version in ("v0.9", "v0.9.1"): - normalized = ( - {**substituted, "version": "v0.9"} - if version == "v0.9.1" - else substituted - ) - V091_CLIENT_TO_SERVER_VALIDATOR.validate(normalized) - else: - raise ValueError(f"Unsupported outbound envelope version: {version!r}") + """Validates an expected outbound envelope against the protocol JSON schema.""" + substituted = substitute_match_any(envelope) + version = substituted.get("version") + if version == "1.0": + V10_RENDERER_TO_AGENT_VALIDATOR.validate(substituted) + elif version in ("v0.9", "v0.9.1"): + normalized = ( + {**substituted, "version": "v0.9"} if version == "v0.9.1" else substituted + ) + V091_CLIENT_TO_SERVER_VALIDATOR.validate(normalized) + else: + raise ValueError(f"Unsupported outbound envelope version: {version!r}") def _adapter_yaml_files() -> list[Path]: - files: list[Path] = [] - core_adapter = CONFORMANCE_DIR / "core" / "adapter_conformance.yaml" - if core_adapter.exists(): - files.append(core_adapter) - adapter_dir = CONFORMANCE_DIR / "adapter" - if adapter_dir.exists(): - files.extend(sorted(adapter_dir.rglob("*.yaml"))) - return files - - -def _collect_outbound_envelopes() -> ( - list[tuple[str, str, int, int, dict[str, Any]]] -): - collected: list[tuple[str, str, int, int, dict[str, Any]]] = [] - for yaml_path in _adapter_yaml_files(): - rel_path = yaml_path.relative_to(CONFORMANCE_DIR).as_posix() - cases = _load_yaml_file(yaml_path) or [] - if not isinstance(cases, list): - continue - for case in cases: - if not isinstance(case, dict): - continue - case_name = str(case.get("name", "")) - for step_idx, step in enumerate(case.get("steps") or []): - if not isinstance(step, dict) or "expectOutbound" not in step: - continue - envelopes = step["expectOutbound"] - assert isinstance( - envelopes, list - ), f"{rel_path}::{case_name}[step {step_idx}] expectOutbound must be a list" - for env_idx, envelope in enumerate(envelopes): - collected.append((rel_path, case_name, step_idx, env_idx, envelope)) - return collected + files: list[Path] = [] + core_adapter = CONFORMANCE_DIR / "core" / "adapter_conformance.yaml" + if core_adapter.exists(): + files.append(core_adapter) + adapter_dir = CONFORMANCE_DIR / "adapter" + if adapter_dir.exists(): + files.extend(sorted(adapter_dir.rglob("*.yaml"))) + return files + + +def _collect_outbound_envelopes() -> list[tuple[str, str, int, int, dict[str, Any]]]: + collected: list[tuple[str, str, int, int, dict[str, Any]]] = [] + for yaml_path in _adapter_yaml_files(): + rel_path = yaml_path.relative_to(CONFORMANCE_DIR).as_posix() + cases = _load_yaml_file(yaml_path) or [] + if not isinstance(cases, list): + continue + for case in cases: + if not isinstance(case, dict): + continue + case_name = str(case.get("name", "")) + for step_idx, step in enumerate(case.get("steps") or []): + if not isinstance(step, dict) or "expectOutbound" not in step: + continue + envelopes = step["expectOutbound"] + assert isinstance(envelopes, list), ( + f"{rel_path}::{case_name}[step {step_idx}] expectOutbound must be a" + " list" + ) + for env_idx, envelope in enumerate(envelopes): + collected.append((rel_path, case_name, step_idx, env_idx, envelope)) + return collected OUTBOUND_ENVELOPES = _collect_outbound_envelopes() def test_adapter_conformance_has_expect_outbound_assertions() -> None: - assert ( - OUTBOUND_ENVELOPES - ), "Expected at least one expectOutbound envelope in adapter conformance YAML" + assert ( + OUTBOUND_ENVELOPES + ), "Expected at least one expectOutbound envelope in adapter conformance YAML" @pytest.mark.parametrize( ("rel_path", "case_name", "step_idx", "env_idx", "envelope"), OUTBOUND_ENVELOPES, - ids=[ - f"{rel}::{case}[step{s}][{e}]" - for rel, case, s, e, _ in OUTBOUND_ENVELOPES - ], + ids=[f"{rel}::{case}[step{s}][{e}]" for rel, case, s, e, _ in OUTBOUND_ENVELOPES], ) def test_outbound_envelopes_match_protocol_schema( rel_path: str, @@ -195,39 +185,39 @@ def test_outbound_envelopes_match_protocol_schema( env_idx: int, envelope: dict[str, Any], ) -> None: - try: - validate_outbound_envelope(envelope) - except jsonschema.ValidationError as exc: - pytest.fail( - f"Outbound envelope failed schema validation in " - f"{rel_path}::{case_name} [step {step_idx}, envelope {env_idx}]:\n" - f" Message: {exc.message}\n" - f" Path: {list(exc.absolute_path)}" - ) + try: + validate_outbound_envelope(envelope) + except jsonschema.ValidationError as exc: + pytest.fail( + "Outbound envelope failed schema validation in " + f"{rel_path}::{case_name} [step {step_idx}, envelope {env_idx}]:\n" + f" Message: {exc.message}\n" + f" Path: {list(exc.absolute_path)}" + ) def test_substitute_match_any_and_rejection_of_invalid_envelopes() -> None: - valid_rpc = { - "version": "1.0", - "callAgentFunction": { - "surfaceId": "main", - "functionCallId": {"$match": "any"}, - "callFunction": { - "@call": "fetchItems", - "args": {"limit": 10}, - }, - }, - } - validate_outbound_envelope(valid_rpc) - - invalid_action_missing_source = { - "version": "1.0", - "action": { - "name": "buy_item", - "surfaceId": "adapter_surface", - "timestamp": {"$match": "any"}, - "context": {}, - }, - } - with pytest.raises(jsonschema.ValidationError): - validate_outbound_envelope(invalid_action_missing_source) + valid_rpc = { + "version": "1.0", + "callAgentFunction": { + "surfaceId": "main", + "functionCallId": {"$match": "any"}, + "callFunction": { + "@call": "fetchItems", + "args": {"limit": 10}, + }, + }, + } + validate_outbound_envelope(valid_rpc) + + invalid_action_missing_source = { + "version": "1.0", + "action": { + "name": "buy_item", + "surfaceId": "adapter_surface", + "timestamp": {"$match": "any"}, + "context": {}, + }, + } + with pytest.raises(jsonschema.ValidationError): + validate_outbound_envelope(invalid_action_missing_source) diff --git a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift index 0fd8c5d8e2..5be2768413 100644 --- a/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift +++ b/swift/swiftui/Tests/A2UISwiftUITests/AdapterHarness.swift @@ -619,11 +619,11 @@ enum AdapterHarness { let processor = MessageProcessor( catalogs: coreCatalogs, actionHandler: handler, - validationConfig: .relaxed, - outboundListener: { message in - handler.recordOutbound(message) - } + validationConfig: .relaxed ) + processor.outboundListener = { message in + handler.recordOutbound(message) + } return AdapterSession( caseName: caseName, protocolVersion: protocolVersion, diff --git a/typescript/renderers/angular/src/adapter_harness.ts b/typescript/renderers/angular/src/adapter_harness.ts index 2f94bd9f9e..11344ef337 100644 --- a/typescript/renderers/angular/src/adapter_harness.ts +++ b/typescript/renderers/angular/src/adapter_harness.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { ChangeDetectionStrategy, Component, computed } from '@angular/core'; -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import {ChangeDetectionStrategy, Component, computed} from '@angular/core'; +import {ComponentFixture, TestBed} from '@angular/core/testing'; import { A2uiClientMessageSchema, CommonSchemas, @@ -27,10 +27,10 @@ import { ResolvedBinding, type SurfaceModel, } from '@a2ui/web_core/v0_9'; -import { basicCatalog } from '@a2ui/web_core/catalogs/basic/v1'; -import type { A2uiWebComponentElement } from '@a2ui/web_core/v0_9/universal'; -import { RendererToAgentMessageSchema } from '@a2ui/web_core/v1_0'; -import { z } from 'zod'; +import {basicCatalog} from '@a2ui/web_core/catalogs/basic/v1'; +import type {A2uiWebComponentElement} from '@a2ui/web_core/v0_9/universal'; +import {RendererToAgentMessageSchema} from '@a2ui/web_core/v1_0'; +import {z} from 'zod'; import { A2uiRendererService, AngularCatalog, @@ -108,7 +108,7 @@ export interface ExtraCatalogEntry { { type: string; required?: string[]; - properties?: Record; + properties?: Record; } >; } @@ -256,9 +256,9 @@ function assertObjectKeys(obj: unknown, allowed: ReadonlySet, where: str if (typeof obj !== 'object' || Array.isArray(obj)) { throw new Error(`${where}: expected object`); } - const unknown = Object.keys(obj as Record).filter((key) => !allowed.has(key)); + const unknown = Object.keys(obj as Record).filter(key => !allowed.has(key)); if (unknown.length > 0) { - throw new Error(`${where}: unknown keys: ${unknown.map((k) => `${where}.${k}`).join(', ')}`); + throw new Error(`${where}: unknown keys: ${unknown.map(k => `${where}.${k}`).join(', ')}`); } } @@ -427,7 +427,7 @@ export function findUnsupportedReason( } export function assertAllUnsupportedMatched(): void { - const unused = Object.keys(UNSUPPORTED).filter((key) => !matchedUnsupportedKeys.has(key)); + const unused = Object.keys(UNSUPPORTED).filter(key => !matchedUnsupportedKeys.has(key)); if (unused.length > 0) { throw new Error( `Unused UNSUPPORTED entries did not match any executed case: ${unused.join(', ')}`, @@ -438,10 +438,10 @@ export function assertAllUnsupportedMatched(): void { export function writeManifestRecord(rec: ManifestRecord): void { const karma = ( window as unknown as { - __karma__?: { info?: (payload: { conformanceManifestRecord: ManifestRecord }) => void }; + __karma__?: {info?: (payload: {conformanceManifestRecord: ManifestRecord}) => void}; } ).__karma__; - karma?.info?.({ conformanceManifestRecord: rec }); + karma?.info?.({conformanceManifestRecord: rec}); } const RESERVED_VIEW_KEYS = new Set([ @@ -481,7 +481,7 @@ class ConformanceCustomWidgetComponent extends CatalogComponent((resolve) => setTimeout(resolve, 0)); + await new Promise(resolve => setTimeout(resolve, 0)); const elements = Array.from(container.querySelectorAll('*')); for (const el of elements) { - const litEl = el as unknown as { updateComplete?: Promise }; + const litEl = el as unknown as {updateComplete?: Promise}; if (litEl.updateComplete) { await litEl.updateComplete; } @@ -662,7 +662,7 @@ function findDomElementForNode( node: ComponentNode, ): HTMLElement | undefined { const match = Array.from(container.querySelectorAll('*')).find( - (el) => (el as A2uiWebComponentElement).node === node, + el => (el as A2uiWebComponentElement).node === node, ); return match as HTMLElement | undefined; } @@ -726,7 +726,7 @@ export function assertViewTree( function isNestedComponentElement(host: HTMLElement, candidate: Element): boolean { let current: Element | null = candidate; while (current && current !== host) { - const a2uiEl = current as unknown as { context?: unknown; node?: unknown }; + const a2uiEl = current as unknown as {context?: unknown; node?: unknown}; if (a2uiEl.context || a2uiEl.node) { return true; } @@ -850,19 +850,19 @@ export function assertAxeRules(container: HTMLElement, rules: string[]): void { function countActiveSubscriptions(surface: SurfaceModel): number { const createdListeners = - (surface.componentsModel.onCreated as unknown as { listeners?: Set }).listeners - ?.size ?? 0; + (surface.componentsModel.onCreated as unknown as {listeners?: Set}).listeners?.size ?? + 0; const deletedListeners = - (surface.componentsModel.onDeleted as unknown as { listeners?: Set }).listeners - ?.size ?? 0; + (surface.componentsModel.onDeleted as unknown as {listeners?: Set}).listeners?.size ?? + 0; let componentUpdatedListeners = 0; for (const comp of surface.componentsModel.values) { componentUpdatedListeners += - (comp.onUpdated as unknown as { listeners?: Set }).listeners?.size ?? 0; + (comp.onUpdated as unknown as {listeners?: Set}).listeners?.size ?? 0; } const dataModelInternal = surface.dataModel as unknown as { subscriptions?: Set; - signals?: Map; + signals?: Map; }; let activeSignalTargets = 0; for (const sig of dataModelInternal.signals?.values() ?? []) { @@ -896,8 +896,8 @@ async function performStepOperation(state: HarnessState, step: AdapterStep): Pro const input = compEl?.querySelector('input, textarea') as HTMLInputElement | null; expect(input).withContext(`Input element for "${targetId}"`).not.toBeNull(); input!.value = String(step.inputValue ?? ''); - input!.dispatchEvent(new Event('input', { bubbles: true, composed: true })); - input!.dispatchEvent(new Event('change', { bubbles: true, composed: true })); + input!.dispatchEvent(new Event('input', {bubbles: true, composed: true})); + input!.dispatchEvent(new Event('change', {bubbles: true, composed: true})); return; } if (step.op === 'user_gesture') { @@ -999,7 +999,7 @@ function mapNodesToDomElements( const result = new Map(); const allElements = Array.from(container.querySelectorAll('*')); for (const [nodePath, node] of nodes) { - const match = allElements.find((el) => (el as A2uiWebComponentElement).node === node); + const match = allElements.find(el => (el as A2uiWebComponentElement).node === node); if (match) { result.set(nodePath, match as HTMLElement); } @@ -1140,7 +1140,7 @@ export function mountSurface(caseSpec?: Partial): HarnessState provideA2UI({ catalogs, processorOptions: { - outboundListener: (msg) => { + outboundListener: msg => { outbound.push(structuredClone(msg) as unknown as Record); }, }, @@ -1158,15 +1158,15 @@ export function mountSurface(caseSpec?: Partial): HarnessState previousNodes: new Map(), previousDomElements: new Map(), }; - rendererService.surfaceGroup.onSurfaceCreated.subscribe((surface) => { + rendererService.surfaceGroup.onSurfaceCreated.subscribe(surface => { state.surface = surface; - surface.onAction.subscribe((action) => { + surface.onAction.subscribe(action => { state.actions.push(action as unknown as Record); state.outbound.push( frameActionEnvelope(action as unknown as Record, state.protocolVersion), ); }); - surface.onError.subscribe((err) => { + surface.onError.subscribe(err => { state.errors.push(err as unknown as Record); if (state.currentOp !== 'programmatic_invoke') { state.outbound.push( From de1276f8ab1a1e5c83890a0e6c54378e6c4d4a58 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 01:07:18 +0000 Subject: [PATCH 37/48] test(conformance): address review feedback on core manifest harnesses --- .../catalog_schema_conformance_test.dart | 39 ++--- .../test/conformance/conformance_harness.dart | 26 +++- .../message_processor_conformance_test.dart | 6 +- .../ConformanceManifest.swift | 143 ++++++++++++++---- .../MessageProcessorConformanceTests.swift | 39 +++-- .../MultiCatalogConformanceTests.swift | 8 +- .../ValidatorConformanceTests.swift | 5 +- .../tests/conformance/conformance_test.mjs | 81 ++++------ 8 files changed, 231 insertions(+), 116 deletions(-) diff --git a/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart b/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart index 53d3f5c52a..34925e6ee2 100644 --- a/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/catalog_schema_conformance_test.dart @@ -167,6 +167,11 @@ void _runFromJsonCase(Map testCase) { _version(testCase) ?? A2uiProtocolVersion.v0_9; if (testCase['expectError'] case final Map expectError) { + assertKnownKeys( + expectError.cast(), + knownExpectErrorKeys, + 'expectError', + ); final String? message = _messageWordedDifferently.contains(name) ? null : expectError['message'] as String?; @@ -195,12 +200,7 @@ void _runFromJsonCase(Map testCase) { ); final Map expected = (testCase['expect'] as Map?)?.cast() ?? const {}; - final Set unknown = expected.keys.toSet().difference( - _fromJsonExpectKeys, - ); - if (unknown.isNotEmpty) { - fail('$name: unsupported from_json expectations $unknown'); - } + assertKnownKeys(expected, _fromJsonExpectKeys, 'expect:from_json'); if (expected['catalogId'] case final String expectedId) { expect(catalog.id, expectedId, reason: '$name: catalogId'); @@ -285,8 +285,9 @@ List _list(Object? value) => value is List ? value : const []; void _checkMatchesBuiltinCatalog(CatalogApi loaded, String name) { final CatalogApi? builtin = _builtinCatalogs[loaded.id]; if (builtin == null) { - markTestSkipped('a2ui_core has no built-in catalog ${loaded.id}.'); - return; + throw SkipConformanceCaseException( + 'a2ui_core has no built-in catalog ${loaded.id}.', + ); } expect( loaded.protocolVersion, @@ -361,7 +362,12 @@ void _runValidationSchemaCase(Map testCase) { final Map source = _document( testCase['catalog'] ?? testCase['catalogPath'], ); - if (testCase['expectError'] != null) { + if (testCase['expectError'] case final Map expectError) { + assertKnownKeys( + expectError.cast(), + knownExpectErrorKeys, + 'expectError', + ); expect( () => _load(testCase, source).validationSchema, throwsA(isA()), @@ -533,11 +539,10 @@ void _runToJsonCase(Map testCase) { _version(testCase), ); if (implemented == null) { - markTestSkipped( + throw SkipConformanceCaseException( 'a2ui_core has no built-in $builtin catalog for ' '${testCase['protocolVersion']}.', ); - return; } catalog = implemented; } else { @@ -627,6 +632,11 @@ void _runDeriveCase(Map testCase) { ); if (testCase['expectError'] case final Map expectError) { + assertKnownKeys( + expectError.cast(), + knownExpectErrorKeys, + 'expectError', + ); expect( derive, throwsA( @@ -643,12 +653,7 @@ void _runDeriveCase(Map testCase) { final Map expected = (testCase['expect']! as Map).cast(); - final Set unknown = expected.keys.toSet().difference( - _deriveExpectKeys, - ); - if (unknown.isNotEmpty) { - fail('$name: unsupported derive expectations $unknown'); - } + assertKnownKeys(expected, _deriveExpectKeys, 'expect:derive'); final CatalogApi derived = derive(); final Map document = derived.toJson(); if (expected['catalogId'] case final String id) { diff --git a/dart/a2ui_core/test/conformance/conformance_harness.dart b/dart/a2ui_core/test/conformance/conformance_harness.dart index 52908470dc..f0fa763c48 100644 --- a/dart/a2ui_core/test/conformance/conformance_harness.dart +++ b/dart/a2ui_core/test/conformance/conformance_harness.dart @@ -202,6 +202,19 @@ String? caseVersion(ConformanceTestCase testCase) { return null; } +/// Thrown inside a conformance test body to skip the case and record it as +/// `'skipped'` in the executed-case manifest. +final class SkipConformanceCaseException implements Exception { + /// Creates a [SkipConformanceCaseException] with [reason]. + const SkipConformanceCaseException(this.reason); + + /// Why the case is skipped. + final String reason; + + @override + String toString() => 'SkipConformanceCaseException: $reason'; +} + /// Executes a single conformance [testCase] [body], applying [expectedFailures] /// and [unsupported] semantics when the case name is listed, and recording the /// outcome to the executed-case manifest when [suite] is provided. @@ -231,7 +244,7 @@ Future runConformanceCase( final String? reason = expectedFailures[name]; if (reason == null) { - await _runStandardCase(body, suite, name, manifestDir); + await _runStandardCase(body, suite, name, onSkip, manifestDir); return; } @@ -261,10 +274,15 @@ Future _runStandardCase( FutureOr Function() body, String? suite, String name, + void Function(String message) onSkip, String? manifestDir, ) async { try { await body(); + } on SkipConformanceCaseException catch (error) { + _recordIfSuite(suite, name, 'skipped', error.reason, manifestDir); + onSkip(error.reason); + return; } catch (error) { _recordIfSuite(suite, name, 'failed', '$error', manifestDir); rethrow; @@ -406,6 +424,7 @@ String? extractErrorCode(A2uiError error) { /// and `RecursionError` accept the sibling types the SDKs raise for the same /// condition. Matcher categoryMatches(String? category) => switch (category) { + null => isA(), 'ParseError' => isA(), 'ValidationError' => anyOf( isA(), @@ -421,7 +440,10 @@ Matcher categoryMatches(String? category) => switch (category) { ), 'DataError' => isA(), 'StateError' => isA(), - _ => isA(), + 'ExpressionError' => isA(), + _ => throw UnknownConformanceKeyError( + 'Unknown expectError.category "$category".', + ), }; /// Matches `code` and `path` expectations declared in an `expectError` block diff --git a/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart b/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart index 7e89eb9fa9..34467b1dcc 100644 --- a/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart +++ b/dart/a2ui_core/test/conformance/message_processor_conformance_test.dart @@ -818,6 +818,7 @@ Matcher _matchesError( final category = expectError['category'] as String?; final message = expectError['message'] as String?; Matcher matcher = switch (category) { + null => isA(), 'DataError' => isA(), 'ValidationError' => anyOf( isA(), @@ -830,7 +831,10 @@ Matcher _matchesError( 'RecursionError' => isA(), 'StateError' => isA(), 'ParseError' => isA(), - _ => isA(), + 'ExpressionError' => isA(), + _ => throw UnknownConformanceKeyError( + 'Unknown expectError.category "$category".', + ), }; if (message != null) { final String pattern = alignErrorPattern(message); diff --git a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift index f72547eb36..33a505fe55 100644 --- a/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift +++ b/swift/core/Tests/A2UIConformanceTests/ConformanceManifest.swift @@ -15,6 +15,51 @@ import Foundation import Testing +private final class IssueTracker: @unchecked Sendable { + private let lock = NSLock() + private var _hasRecordedIssue = false + private var _firstFailureReason: String? + private var _unknownKeyError: UnknownConformanceKeyError? + + var hasRecordedIssue: Bool { + lock.lock() + defer { lock.unlock() } + return _hasRecordedIssue + } + + var firstFailureReason: String? { + lock.lock() + defer { lock.unlock() } + return _firstFailureReason + } + + var unknownKeyError: UnknownConformanceKeyError? { + lock.lock() + defer { lock.unlock() } + return _unknownKeyError + } + + func recordIssue(_ issue: Issue) { + lock.lock() + defer { lock.unlock() } + _hasRecordedIssue = true + if _firstFailureReason == nil { + _firstFailureReason = String(describing: issue) + } + } + + func recordUnknownKey(_ error: UnknownConformanceKeyError) { + lock.lock() + defer { lock.unlock() } + if _unknownKeyError == nil { + _unknownKeyError = error + } + if _firstFailureReason == nil { + _firstFailureReason = error.description + } + } +} + /// Appends executed conformance test case records to a JSON Lines manifest file when /// `A2UI_CONFORMANCE_MANIFEST_DIR` is set in the process environment. public enum ConformanceManifest { @@ -99,31 +144,54 @@ public enum ConformanceManifest { record(testCase, status: .skipped, reason: skippedReason) return } + let tracker = IssueTracker() if let knownIssueReason { - var unknownKeyError: UnknownConformanceKeyError? - withKnownIssue(Comment(rawValue: knownIssueReason)) { + withKnownIssue(Comment(rawValue: knownIssueReason), isIntermittent: true) { do { try body() - } catch let error as UnknownConformanceKeyError { - unknownKeyError = error } catch { Issue.record(error) } + } matching: { issue in + if let unknownKeyError = issue.error as? UnknownConformanceKeyError { + tracker.recordUnknownKey(unknownKeyError) + return false + } + tracker.recordIssue(issue) + return true } - if let unknownKeyError { + if let unknownKeyError = tracker.unknownKeyError { record(testCase, status: .failed, reason: unknownKeyError.description) - throw unknownKeyError + return + } + if tracker.hasRecordedIssue { + record(testCase, status: .xfailed, reason: knownIssueReason) + } else { + let message = "Known issue was not recorded: \(knownIssueReason)" + Issue.record(Comment(rawValue: message)) + record(testCase, status: .failed, reason: message) } - record(testCase, status: .xfailed, reason: knownIssueReason) return } - do { - try body() - record(testCase, status: .passed) - } catch { - record(testCase, status: .failed, reason: String(describing: error)) - throw error + withKnownIssue(isIntermittent: true) { + do { + try body() + } catch { + Issue.record(error) + } + } matching: { issue in + if let unknownKeyError = issue.error as? UnknownConformanceKeyError { + tracker.recordUnknownKey(unknownKeyError) + } else { + tracker.recordIssue(issue) + } + return false + } + if let failureReason = tracker.firstFailureReason { + record(testCase, status: .failed, reason: failureReason) + return } + record(testCase, status: .passed) } /// Executes an asynchronous `body` for `testCase` and records the outcome in the manifest. @@ -138,30 +206,53 @@ public enum ConformanceManifest { record(testCase, status: .skipped, reason: skippedReason) return } + let tracker = IssueTracker() if let knownIssueReason { - var unknownKeyError: UnknownConformanceKeyError? - await withKnownIssue(Comment(rawValue: knownIssueReason)) { + await withKnownIssue(Comment(rawValue: knownIssueReason), isIntermittent: true) { do { try await body() - } catch let error as UnknownConformanceKeyError { - unknownKeyError = error } catch { Issue.record(error) } + } matching: { issue in + if let unknownKeyError = issue.error as? UnknownConformanceKeyError { + tracker.recordUnknownKey(unknownKeyError) + return false + } + tracker.recordIssue(issue) + return true } - if let unknownKeyError { + if let unknownKeyError = tracker.unknownKeyError { record(testCase, status: .failed, reason: unknownKeyError.description) - throw unknownKeyError + return + } + if tracker.hasRecordedIssue { + record(testCase, status: .xfailed, reason: knownIssueReason) + } else { + let message = "Known issue was not recorded: \(knownIssueReason)" + Issue.record(Comment(rawValue: message)) + record(testCase, status: .failed, reason: message) } - record(testCase, status: .xfailed, reason: knownIssueReason) return } - do { - try await body() - record(testCase, status: .passed) - } catch { - record(testCase, status: .failed, reason: String(describing: error)) - throw error + await withKnownIssue(isIntermittent: true) { + do { + try await body() + } catch { + Issue.record(error) + } + } matching: { issue in + if let unknownKeyError = issue.error as? UnknownConformanceKeyError { + tracker.recordUnknownKey(unknownKeyError) + } else { + tracker.recordIssue(issue) + } + return false + } + if let failureReason = tracker.firstFailureReason { + record(testCase, status: .failed, reason: failureReason) + return } + record(testCase, status: .passed) } } diff --git a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift index f3365a5c7a..bef8018759 100644 --- a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift @@ -446,10 +446,12 @@ struct MessageProcessorConformanceTests { let strictMode = testCase["strictMode"] as? Bool ?? false let validationConfig: ValidationConfig if let flags = testCase["validationConfig"] as? [String: Bool] { - validationConfig = try #require( - ConformanceTestHelper.validationConfig(from: flags), - "[\(name)] Unknown validationConfig flags \(flags.keys.sorted())" - ) + guard let config = ConformanceTestHelper.validationConfig(from: flags) else { + throw UnknownConformanceKeyError( + "[\(name)] Unknown validationConfig flags \(flags.keys.sorted())" + ) + } + validationConfig = config } else { validationConfig = strictMode ? .strict : .relaxed } @@ -473,9 +475,11 @@ struct MessageProcessorConformanceTests { } let steps = explicitSteps ?? [testCase] - for step in steps { + for (index, step) in steps.enumerated() { try executeProcessMessagesStep( step, + testCase: testCase, + isFinalStep: index == steps.count - 1, processor: processor, handler: handler, expectedRuntimeError: expectedRuntimeError, @@ -486,6 +490,8 @@ struct MessageProcessorConformanceTests { private func executeProcessMessagesStep( _ step: [String: Any], + testCase: [String: Any], + isFinalStep: Bool, processor: MessageProcessor, handler: ConformanceActionCaptureHandler, expectedRuntimeError: ExpectedRuntimeError?, @@ -494,14 +500,17 @@ struct MessageProcessorConformanceTests { let payload = ConformanceTestHelper.toJSONValue(step["messages"] ?? [:]) handler.capturedErrors.removeAll() - if let expectError = step["expectError"] as? [String: Any] { + let rawExpectError = + (step["expectError"] as? [String: Any]) + ?? (isFinalStep ? (testCase["expectError"] as? [String: Any]) : nil) + if let expectError = rawExpectError { try ConformanceTestHelper.assertKnownKeys( expectError, Self.knownExpectErrorKeys, "message_processor expectError in '\(name)'", allowMetadata: false ) - assertStepError( + try assertStepError( payload: payload, expectError: expectError, processor: processor, @@ -518,7 +527,10 @@ struct MessageProcessorConformanceTests { name: name ) - if let expectedDict = step["expect"] as? [String: Any] { + let rawExpect = + (step["expect"] as? [String: Any]) + ?? (isFinalStep ? (testCase["expect"] as? [String: Any]) : nil) + if let expectedDict = rawExpect { try ConformanceTestHelper.assertKnownKeys( expectedDict, ["surfaces"], @@ -526,10 +538,11 @@ struct MessageProcessorConformanceTests { allowMetadata: false ) } - let expectedDict = (step["expect"] as? [String: Any]) ?? [:] + let expectedDict = rawExpect ?? [:] let expectedSurfaces = (expectedDict["surfaces"] as? [String: Any]) ?? (step["expectedSurfaces"] as? [String: Any]) + ?? (isFinalStep ? (testCase["expectedSurfaces"] as? [String: Any]) : nil) try verifyExpectedSurfaces(expectedSurfaces, on: processor, name: name) } @@ -539,12 +552,12 @@ struct MessageProcessorConformanceTests { processor: MessageProcessor, handler: ConformanceActionCaptureHandler, name: String - ) { + ) throws { do { try processor.processMessages(payload) Issue.record("[\(name)] Expected error to be thrown for invalid message") } catch { - assertProcessErrorCategory(error: error, expectError: expectError, name: name) + try assertProcessErrorCategory(error: error, expectError: expectError, name: name) if let expectedMessage = expectError["message"] as? String, !expectedMessage.isEmpty { let description = (error as? (any A2UIError))?.message ?? error.localizedDescription #expect( @@ -566,7 +579,7 @@ struct MessageProcessorConformanceTests { error: Error, expectError: [String: Any], name: String - ) { + ) throws { guard let category = expectError["category"] as? String else { return } switch category { case "RecursionError": @@ -581,7 +594,7 @@ struct MessageProcessorConformanceTests { case "ValidationError": #expect(error is A2UIValidationError, "[\(name)] Expected ValidationError, got \(error)") default: - Issue.record("[\(name)] Unknown error category '\(category)'") + throw UnknownConformanceKeyError("[\(name)] Unknown error category '\(category)'") } } diff --git a/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift index 2b15a49f2a..8bf36ee58c 100644 --- a/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MultiCatalogConformanceTests.swift @@ -128,7 +128,7 @@ struct MultiCatalogConformanceTests { "multi_catalog.yaml expectError in '\(name)'", allowMetadata: false ) - assertSelectCatalogError( + try assertSelectCatalogError( processor: processor, createPayload: createPayload, componentList: componentList, @@ -260,7 +260,7 @@ struct MultiCatalogConformanceTests { defaultVersion: String, expectError: [String: Any], name: String - ) { + ) throws { do { try processSelectPayloads( processor: processor, @@ -276,7 +276,9 @@ struct MultiCatalogConformanceTests { case "CatalogError": #expect(error is A2UICatalogError, "[\(name)] Expected A2UICatalogError, got \(error)") default: - Issue.record("[\(name)] Unknown expected error category '\(category)'") + throw UnknownConformanceKeyError( + "[\(name)] Unknown expected error category '\(category)'" + ) } } } diff --git a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift index 17ecf58c3e..64e6af3ff2 100644 --- a/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/ValidatorConformanceTests.swift @@ -370,8 +370,9 @@ struct ValidatorConformanceTests { targetVersion: "1.0" ) else { - Issue.record("[\(testCase.name)] Missing or unknown validationConfig flags") - return + throw UnknownConformanceKeyError( + "[\(testCase.name)] Missing or unknown validationConfig flags \(flags.keys.sorted())" + ) } validationConfig = config } else { diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index 4dfa07df4f..cdd8c6c9e3 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -835,44 +835,22 @@ async function validateRpcTestCase(testCase) { if (args.uiBindingScenario) { const scenario = args.uiBindingScenario; - const callName = - scenario.component?.text?.['@call'] || - scenario.component?.text?.call || - 'fetchRemoteGreeting'; - const procRef = {current: null}; - const agentBindingFn = createFunctionImplementation( - { - name: callName, - returnType: 'string', - schema: z.record(z.string(), z.any()).optional().default({}), - }, - fnArgs => { - const sig = signal(null); - void procRef.current - .callAgentFunction(scenario.surfaceId, { - '@call': callName, - catalogId: scenario.catalogId, - args: fnArgs, - }) - .then(val => setValue(sig, val)); - return sig; - }, - ); const storeCat = new Catalog( scenario.catalogId, '1.0', v1_0Components, - [agentBindingFn], + [], undefined, undefined, ); - procRef.current = new MessageProcessor([storeCat, v1_0BasicCatalog], undefined, { + const proc = new MessageProcessor([storeCat, v1_0BasicCatalog], undefined, { version: '1.0', + validationConfig: {allowUnknownElements: true}, outboundListener: msg => { sentOutboundMsg = msg; }, }); - procRef.current.processMessages([ + proc.processMessages([ { version: '1.0', createSurface: { @@ -882,19 +860,24 @@ async function validateRpcTestCase(testCase) { }, }, ]); - const surface = procRef.current.getSurface(scenario.surfaceId); + const surface = proc.getSurface(scenario.surfaceId); const resolved = collectResolvedNodes(surface); try { const rootNode = resolved.nodes.get(scenario.component.id); const initialText = normalizeComponentRefs(peekValue(rootNode.props).text); assert.strictEqual(initialText ?? null, expect.initialValue); + await new Promise(resolve => setTimeout(resolve, 0)); assert.ok(sentOutboundMsg?.callAgentFunction, 'Expected outbound callAgentFunction'); assert.strictEqual(sentOutboundMsg.version, expect.outboundCall.version); assert.strictEqual( sentOutboundMsg.callAgentFunction.surfaceId, expect.outboundCall.callAgentFunction.surfaceId, ); - procRef.current.processMessages({ + assert.deepStrictEqual( + sentOutboundMsg.callAgentFunction.callFunction, + expect.outboundCall.callAgentFunction.callFunction, + ); + proc.processMessages({ version: '1.0', agentFunctionResponse: { functionCallId: sentOutboundMsg.callAgentFunction.functionCallId, @@ -912,37 +895,22 @@ async function validateRpcTestCase(testCase) { if (args.uiActionFallbackScenario) { const scenario = args.uiActionFallbackScenario; - const fc = scenario.actionPayload.functionCall; - const callName = fc['@call'] || fc.call; - const procRef = {current: null}; - const agentActionFn = createFunctionImplementation( - { - name: callName, - returnType: 'any', - schema: z.record(z.string(), z.any()).optional().default({}), - }, - fnArgs => - procRef.current.callAgentFunction(scenario.surfaceId, { - '@call': callName, - catalogId: scenario.catalogId, - args: fnArgs, - }), - ); const storeCat = new Catalog( scenario.catalogId, '1.0', v1_0Components, - [agentActionFn], + [], undefined, undefined, ); - procRef.current = new MessageProcessor([storeCat], undefined, { + const proc = new MessageProcessor([storeCat], undefined, { version: '1.0', + validationConfig: {allowUnknownElements: true}, outboundListener: msg => { sentOutboundMsg = msg; }, }); - procRef.current.processMessages([ + proc.processMessages([ { version: '1.0', createSurface: { @@ -951,16 +919,21 @@ async function validateRpcTestCase(testCase) { }, }, ]); - const surface = procRef.current.getSurface(scenario.surfaceId); + const surface = proc.getSurface(scenario.surfaceId); const ctx = new DataContext(surface, '/'); const actionPromise = ctx.resolveAction(scenario.actionPayload); + await new Promise(resolve => setTimeout(resolve, 0)); assert.ok(sentOutboundMsg?.callAgentFunction, 'Expected outbound callAgentFunction'); assert.strictEqual(sentOutboundMsg.version, expect.outboundCall.version); assert.strictEqual( sentOutboundMsg.callAgentFunction.surfaceId, expect.outboundCall.callAgentFunction.surfaceId, ); - procRef.current.processMessages({ + assert.deepStrictEqual( + sentOutboundMsg.callAgentFunction.callFunction, + expect.outboundCall.callAgentFunction.callFunction, + ); + proc.processMessages({ version: '1.0', agentFunctionResponse: { functionCallId: sentOutboundMsg.callAgentFunction.functionCallId, @@ -1662,7 +1635,10 @@ function validateValidateTestCase(testCase) { const inputMessages = step.messages || (Array.isArray(step.payload) ? step.payload : step.payload ? [step.payload] : []); - const expErrObj = step.expectError || (i === stepsToRun.length - 1 ? expectError : undefined); + const expErrObj = + step.expectError || + step.expect_error || + (i === stepsToRun.length - 1 ? expectError : undefined); const stepExpect = step.expect || (i === stepsToRun.length - 1 ? expect : undefined); if (inputMessages.length > 0) { @@ -3363,7 +3339,8 @@ function validateProcessMessagesTestCase(testCase) { assertKnownKeys(testCase, PROCESS_MESSAGES_KNOWN_KEYS.case, 'process_messages'); assertKnownKeysList(testCase.steps, PROCESS_MESSAGES_KNOWN_KEYS.step, 'process_messages.steps'); assertKnownKeys(testCase.expect, PROCESS_MESSAGES_KNOWN_KEYS.expect, 'process_messages.expect'); - const {messages, payload, steps, expect, expectError, protocolVersion} = testCase; + const {messages, payload, steps, expect, protocolVersion} = testCase; + const expectError = testCase.expectError || testCase.expect_error; const testCatalogs = getCatalogsForTestCase(testCase); let lastErrorMsg; @@ -3413,7 +3390,7 @@ function validateProcessMessagesTestCase(testCase) { ); const context = step.validationConfig ? {validationConfig: step.validationConfig} : undefined; const stepExpectError = - step.expectError || (i === steps.length - 1 ? expectError : undefined); + step.expectError || step.expect_error || (i === steps.length - 1 ? expectError : undefined); if (stepExpectError) { lastErrorMsg = undefined; let thrown; From 1d5ff16da79c28a48c75900d5556a1fee48f333f Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 01:38:09 +0000 Subject: [PATCH 38/48] fix(conformance): mark assertProcessErrorCategory with try and drop unused imports --- .../MessageProcessorConformanceTests.swift | 2 +- .../core/web_core/tests/conformance/conformance_test.mjs | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift index bef8018759..c712cda6a1 100644 --- a/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift +++ b/swift/core/Tests/A2UIConformanceTests/MessageProcessorConformanceTests.swift @@ -920,7 +920,7 @@ struct MessageProcessorConformanceTests { _ = try processor.getRendererCapabilities(options: options) Issue.record("[\(name)] Expected getRendererCapabilities to throw") } catch { - assertProcessErrorCategory(error: error, expectError: expectError, name: name) + try assertProcessErrorCategory(error: error, expectError: expectError, name: name) } return } diff --git a/typescript/core/web_core/tests/conformance/conformance_test.mjs b/typescript/core/web_core/tests/conformance/conformance_test.mjs index cdd8c6c9e3..4572794c86 100644 --- a/typescript/core/web_core/tests/conformance/conformance_test.mjs +++ b/typescript/core/web_core/tests/conformance/conformance_test.mjs @@ -40,14 +40,7 @@ import { import {ExpressionParser} from '../../dist/src/expressions/expression_parser.js'; import {SUPPORTED_PROTOCOL_VERSIONS} from '../../dist/src/processing/adapters/base.js'; import {MessageProcessor, STRICT_VALIDATION} from '../../dist/src/processing/message-processor.js'; -import { - effect, - getValue, - isSignal, - peekValue, - setValue, - signal, -} from '../../dist/src/reactivity/signals.js'; +import {effect, getValue, isSignal, peekValue} from '../../dist/src/reactivity/signals.js'; import {DataContext} from '../../dist/src/resolution/data-context.js'; import {NodeResolver} from '../../dist/src/resolution/node-resolver.js'; import {ResolvedBinding} from '../../dist/src/resolution/resolved-binding.js'; From 915982a7540abc8f0e503159882e963262eca0a2 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 01:44:07 +0000 Subject: [PATCH 39/48] test(conformance): golden-transcript scenario format and newsletter seed --- conformance/README.md | 24 ++ ...onformance_harness_implementation_guide.md | 43 +++ conformance/conformance_schema.json | 94 ++++- conformance/e2e/newsletter_signup.yaml | 332 ++++++++++++++++++ conformance/manifest_corpora.json | 3 +- conformance/tests/test_conformance_yaml.py | 2 +- conformance/tests/test_e2e_scenarios.py | 295 ++++++++++++++++ 7 files changed, 790 insertions(+), 3 deletions(-) create mode 100644 conformance/e2e/newsletter_signup.yaml create mode 100644 conformance/tests/test_e2e_scenarios.py diff --git a/conformance/README.md b/conformance/README.md index 105b540245..9f5d2f3e1a 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -73,6 +73,30 @@ It is the one suite under `agent/` that does not use the case vocabulary below. - `extensions/a2a/a2a_integration.yaml`: Contains test cases for A2A protocol event and part conversions. - `extensions/adk/adk_extensions.yaml`: Contains test cases for ADK extensions and RPC handling. +### End-to-end (`e2e/`) + +End-to-end conformance scenarios in `e2e/` capture complete multi-turn conversational transcripts and user interactions. Each scenario validates: + +- **Agent half**: Inbound chunks for each declared format (`direct_json`, `express`) are streamed and single-shot parsed by the agent SDK's response parser, verifying that they yield the exact expected wire messages (`expectWire`) with adjacent text parts merged. +- **Renderer half**: The `a2ui` messages from each turn are dispatched into framework adapter harnesses (or `MessageProcessor`), which verify the resulting resolved view trees (`expectViewTree`), execute simulated user interactions (`uiOps`), and assert outbound event envelopes (`expectOutbound`) and data model states (`expectDataModel`). + +Harnesses running the agent half: +- Python Agent: `python/a2ui_agent` +- TypeScript Agent: `typescript/agents/a2ui_agent` +- Dart Agent: `dart/a2ui_agent` + +Harnesses running the renderer half: +- Web Renderers: `@a2ui/lit`, `@a2ui/react`, `@a2ui/angular` +- Flutter Renderer: `dart/a2ui_flutter` +- SwiftUI Renderer: `swift/swiftui` + +#### Adding an e2e scenario + +1. Author a new scenario YAML file in `conformance/e2e/.yaml`. +2. Specify `action: e2e_scenario`, `protocolVersion` (e.g. `"1.0"`), `catalogPaths`, `formats` (e.g. `[direct_json, express]`), and multi-turn exchanges with `modelChunks`, `expectWire`, `expectViewTree`, and optional `uiOps`, `expectDataModel`, and `expectOutbound`. +3. Verify the scenario using `pytest conformance/tests/test_e2e_scenarios.py`. +4. When targeting v0.9.1 compatibility, generate or duplicate the v0.9.1 scenario copies using the conformance generator scripts. + All static test data and simplified schemas are located in the `test_data/` directory. `conformance_schema.json` at the root is the JSON schema that validates the structure of YAML test files across all domains. diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index e8f92fa96b..d2a0f8e6df 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -346,6 +346,49 @@ Each case in `core/accessibility.yaml` (`action: accessibility_check`) provides Where a browser-based test runner cannot read YAML from disk at test time, parse the suite files in the runner configuration and inject them as JSON. For example, [typescript/renderers/angular/karma.conf.js](../typescript/renderers/angular/karma.conf.js) parses `accessibility.yaml`, `adapter_conformance.yaml`, and `catalogs/basic/v1/catalog.json` in Node and exposes them to Jasmine browser specs via `client.conformanceSuites`. +## Golden-transcript harness (`conformance/e2e/`) + +The end-to-end golden-transcript harness exercises complete conversational scenarios containing multi-turn exchanges between an agent and a renderer. Each scenario case specifies `action: e2e_scenario`, a `protocolVersion`, `catalogPaths`, `formats`, and a list of `turns`. + +### Agent half algorithm + +The agent half tests response chunk parsing and wire fidelity: +1. Initialize the inference format parser for each declared format (`direct_json`, `express`), configured with the scenario's catalogs. +2. Initialize and maintain an empty parser `RendererState` across the turns of the scenario. +3. For each turn in `scenario["turns"]`: + - Stream the chunks from `turn["modelChunks"][format]` through `parser.open_stream(state=state)`. + - Concatenate all returned `ResponsePart` items, merging adjacent `TextPart` items. + - Assert that the resulting response parts match `turn["expectWire"]`. + - Concatenate all chunks into a single string and parse via `parser.parse_response(full_text, state=state)`, verifying single-shot equality with `expectWire`. + +### Renderer half algorithm + +The renderer half tests UI tree construction, interaction, and outbound messaging: +1. Initialize a `MessageProcessor` or UI framework adapter with the catalogs declared in `catalogPaths`. +2. For each turn in `scenario["turns"]`: + - Feed the `a2ui` messages from `turn["expectWire"]` in order (chat text parts are ignored by renderers). + - Assert that the rendered view tree for each surface matches `turn["expectViewTree"]` (or that the surface is absent if expected value is `null`). + - If `turn["uiOps"]` is provided, execute each operation in sequence (`user_input`, `user_gesture`, `programmatic_invoke`) against the specified `targetNode` on `surfaceId`. + - If `turn["expectDataModel"]` is provided, assert the updated surface data model state. + - If `turn["expectOutbound"]` is provided, capture all outbound envelopes emitted during the turn (schema-validated with wildcards handled via `{"$match": "any"}`) and assert they match `expectOutbound`. + +### Manifest records & harness names + +- Each runner records one manifest entry per scenario in its executed-case manifest (`A2UI_CONFORMANCE_MANIFEST_DIR`). +- A scenario is recorded as `failed` if any declared format, turn, assertion, or operation fails. +- Active harness names: + - Agent harnesses: `python-agent`, `web-agent`, `dart-agent`. + - Renderer harnesses: `lit`, `react`, `angular`, `flutter`, `swiftui`. + +### Renderer-visible outcome definition + +A transcript's renderer-visible outcome is defined by `outcome_of(scenario)` in `conformance/scripts/e2e_outcome.py`: +- Each turn's `a2ui` messages are applied to `MessageProcessor`. +- Each active surface's resolved node tree is walked depth-first, mapping each node to `{type, text?, label?, value?, enabled?, action?}`. +- Model-chosen IDs (`id`, `surfaceId`, `functionCallId`) are mapped to canonical first-seen ordinals. +- Outbound envelopes undergo identical ID normalization with `timestamp` dropped. +- Two transcripts have identical outcomes when their normalized node trees and outbound messages match for every turn, ignoring conversational text differences. + ## Reference harnesses Inspect these existing implementations for concrete code examples: diff --git a/conformance/conformance_schema.json b/conformance/conformance_schema.json index ac71928ae2..2c96e8dcda 100644 --- a/conformance/conformance_schema.json +++ b/conformance/conformance_schema.json @@ -31,7 +31,8 @@ {"$ref": "#/$defs/EvaluateFunctionTest"}, {"$ref": "#/$defs/DispatchActionTest"}, {"$ref": "#/$defs/DeriveTest"}, - {"$ref": "#/$defs/RenderSurfaceTest"} + {"$ref": "#/$defs/RenderSurfaceTest"}, + {"$ref": "#/$defs/E2eScenarioTest"} ] }, "$defs": { @@ -59,6 +60,7 @@ "data_model", "decompile", "dispatch_action", + "e2e_scenario", "evaluate_function", "execute_tool", "export_to_directory", @@ -2156,6 +2158,96 @@ ], "additionalProperties": false } + }, + "E2eScenarioTest": { + "type": "object", + "required": ["name", "action", "protocolVersion", "catalogPaths", "formats", "turns"], + "properties": { + "name": {"type": "string", "description": "Unique scenario name across conformance/e2e/**."}, + "action": {"const": "e2e_scenario"}, + "description": {"type": "string"}, + "protocolVersion": {"$ref": "#/$defs/ProtocolVersion"}, + "catalogPaths": { + "type": "array", + "items": {"type": "string"} + }, + "formats": { + "type": "array", + "items": {"$ref": "#/$defs/InferenceFormatName"} + }, + "source": {"$ref": "#/$defs/E2eSource"}, + "turns": { + "type": "array", + "items": {"$ref": "#/$defs/E2eTurn"}, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "E2eSource": { + "type": "object", + "required": ["kind", "from"], + "properties": { + "kind": { + "type": "string", + "enum": ["hand_authored", "recorded", "generated"] + }, + "from": {"type": "string"}, + "model": {"type": "string"}, + "recordedAt": {"type": "string"} + }, + "additionalProperties": false + }, + "E2eTurn": { + "type": "object", + "required": ["modelChunks", "expectWire"], + "properties": { + "userText": {"type": ["string", "null"]}, + "modelChunks": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": {"type": "string"} + } + }, + "expectWire": { + "type": "array", + "items": {"$ref": "#/$defs/ResponsePart"} + }, + "expectViewTree": {"$ref": "#/$defs/E2eViewTrees"}, + "expectErrors": { + "type": "array", + "items": {"type": "object"} + }, + "uiOps": { + "type": "array", + "items": {"$ref": "#/$defs/E2eUiOp"} + }, + "expectDataModel": {"type": "object"}, + "expectOutbound": {"$ref": "#/$defs/OutboundEnvelopeList"} + }, + "additionalProperties": false + }, + "E2eUiOp": { + "type": "object", + "required": ["op", "surfaceId", "targetNode"], + "properties": { + "op": { + "type": "string", + "enum": ["user_input", "user_gesture", "programmatic_invoke"] + }, + "surfaceId": {"type": "string"}, + "targetNode": {"type": "string"}, + "inputValue": {}, + "userActivation": {"type": "boolean"} + }, + "additionalProperties": false + }, + "E2eViewTrees": { + "type": "object", + "additionalProperties": { + "type": ["object", "null"] + } } } } diff --git a/conformance/e2e/newsletter_signup.yaml b/conformance/e2e/newsletter_signup.yaml new file mode 100644 index 0000000000..333d845142 --- /dev/null +++ b/conformance/e2e/newsletter_signup.yaml @@ -0,0 +1,332 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: newsletter_signup_round_trip + action: e2e_scenario + description: Newsletter form, typed email, Subscribe action, reply updates the form. + protocolVersion: "1.0" + catalogPaths: + - catalogs/basic/v1/catalog.json + formats: + - direct_json + - express + source: + kind: hand_authored + from: dart/a2ui_flutter/example/test/fixtures/newsletter_replies.json + turns: + - userText: "Show a newsletter sign-up form with an email field and a \"Subscribe\" button." + modelChunks: + direct_json: + - "Here is a sign-up form for your newsletter.\n\n\n" + - | + [ + { + "version": "1.0", + "createSurface": { + "surfaceId": "newsletter-signup", + "catalogId": "https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json", + "sendDataModel": true, + "components": [ + { + "id": "root", + "component": "Card", + "child": "container" + }, + { + "id": "container", + "component": "Column", + "children": [ + "title", + "description", + "emailInput", + "subscribeBtn" + ] + }, + { + "id": "title", + "component": "Text", + "text": "### Subscribe to our Newsletter" + }, + { + "id": "description", + "component": "Text", + "text": "Get the latest updates and news delivered straight to your inbox.", + "variant": "body" + }, + { + "id": "emailInput", + "component": "TextField", + "label": "Email Address", + "value": { + "@path": "/email" + }, + "checks": [ + { + "condition": { + "@call": "email", + "args": { + "value": { + "@path": "/email" + } + } + }, + "message": "Please enter a valid email address." + } + ] + }, + { + "id": "subscribeBtn", + "component": "Button", + "variant": "primary", + "child": "subscribeText", + "action": { + "event": { + "name": "subscribe", + "context": { + "email": { + "@path": "/email" + } + } + } + } + }, + { + "id": "subscribeText", + "component": "Text", + "text": "Subscribe" + } + ], + "dataModel": { + "email": "" + } + } + } + ] + + express: + - "Here is a sign-up form for your newsletter.\n\n\n" + - | + surface("newsletter-signup", sendDataModel=true) + $/email = "" + root = Card(container) + container = Column([title, description, emailInput, subscribeBtn]) + title = Text("### Subscribe to our Newsletter") + description = Text("Get the latest updates and news delivered straight to your inbox.", "body") + emailInput = TextField("Email Address", $/email, ?email("Please enter a valid email address.")) + subscribeBtn = Button(subscribeText, "primary", Event("subscribe", {email: $/email})) + subscribeText = Text("Subscribe") + + expectWire: + - text: "Here is a sign-up form for your newsletter." + - a2ui: + - version: "1.0" + createSurface: + surfaceId: newsletter-signup + catalogId: https://a2ui.org/specification/v1_0/catalogs/basic/catalog.json + sendDataModel: true + components: + - id: root + component: Card + child: container + - id: container + component: Column + children: + - title + - description + - emailInput + - subscribeBtn + - id: title + component: Text + text: "### Subscribe to our Newsletter" + - id: description + component: Text + text: Get the latest updates and news delivered straight to your inbox. + variant: body + - id: emailInput + component: TextField + label: Email Address + value: + "@path": /email + checks: + - condition: + "@call": email + args: + value: + "@path": /email + message: Please enter a valid email address. + - id: subscribeBtn + component: Button + variant: primary + child: subscribeText + action: + event: + name: subscribe + context: + email: + "@path": /email + - id: subscribeText + component: Text + text: Subscribe + dataModel: + email: "" + expectViewTree: + newsletter-signup: + id: root + type: Card + children: + - id: container + type: Column + children: + - id: title + type: Text + text: "### Subscribe to our Newsletter" + - id: description + type: Text + text: Get the latest updates and news delivered straight to your inbox. + - id: emailInput + type: TextField + label: Email Address + value: "" + - id: subscribeBtn + type: Button + children: + - id: subscribeText + type: Text + text: Subscribe + expectErrors: [] + uiOps: + - op: user_input + surfaceId: newsletter-signup + targetNode: emailInput + inputValue: a@b.c + - op: user_gesture + surfaceId: newsletter-signup + targetNode: subscribeBtn + expectDataModel: + newsletter-signup: + email: a@b.c + expectOutbound: + - version: "1.0" + action: + name: subscribe + surfaceId: newsletter-signup + sourceComponentId: subscribeBtn + timestamp: + $match: any + context: + email: a@b.c + - userText: null + modelChunks: + direct_json: + - "Thank you for subscribing! We've sent a confirmation email to ada@example.com.\n\n\n" + - | + [ + { + "version": "1.0", + "updateComponents": { + "surfaceId": "newsletter-signup", + "components": [ + { + "id": "root", + "component": "Card", + "child": "container" + }, + { + "id": "container", + "component": "Column", + "children": [ + "successIcon", + "title", + "description" + ], + "align": "center" + }, + { + "id": "successIcon", + "component": "Icon", + "name": "check" + }, + { + "id": "title", + "component": "Text", + "text": "### You're subscribed!" + }, + { + "id": "description", + "component": "Text", + "text": "We sent a confirmation email to ada@example.com. Check your inbox to confirm your subscription.", + "variant": "body" + } + ] + } + } + ] + + express: + - "Thank you for subscribing! We've sent a confirmation email to ada@example.com.\n\n\n" + - | + surface("newsletter-signup", update=true) + root = Card(container) + container = Column([successIcon, title, description], _, "center") + successIcon = Icon("check") + title = Text("### You're subscribed!") + description = Text("We sent a confirmation email to ada@example.com. Check your inbox to confirm your subscription.", "body") + + expectWire: + - text: "Thank you for subscribing! We've sent a confirmation email to ada@example.com." + - a2ui: + - version: "1.0" + updateComponents: + surfaceId: newsletter-signup + components: + - id: root + component: Card + child: container + - id: container + component: Column + children: + - successIcon + - title + - description + align: center + - id: successIcon + component: Icon + name: check + - id: title + component: Text + text: "### You're subscribed!" + - id: description + component: Text + text: We sent a confirmation email to ada@example.com. Check your inbox to confirm your subscription. + variant: body + expectViewTree: + newsletter-signup: + id: root + type: Card + children: + - id: container + type: Column + children: + - id: successIcon + type: Icon + - id: title + type: Text + text: "### You're subscribed!" + - id: description + type: Text + text: We sent a confirmation email to ada@example.com. Check your inbox to confirm your subscription. diff --git a/conformance/manifest_corpora.json b/conformance/manifest_corpora.json index 33eac8b9d9..3a38af0a0c 100644 --- a/conformance/manifest_corpora.json +++ b/conformance/manifest_corpora.json @@ -4,5 +4,6 @@ "extensions": {"include": ["extensions/**/*.yaml"]}, "adapter": { "include": ["core/adapter_conformance.yaml", "core/accessibility.yaml"] - } + }, + "e2e": {"include": ["e2e/*.yaml"]} } diff --git a/conformance/tests/test_conformance_yaml.py b/conformance/tests/test_conformance_yaml.py index 6aea614d10..b5513b4dc3 100644 --- a/conformance/tests/test_conformance_yaml.py +++ b/conformance/tests/test_conformance_yaml.py @@ -116,7 +116,7 @@ def iter_renderer_errors(case): def get_yaml_files(): files = [] - for domain in ["core", "agent", "extensions"]: + for domain in ["core", "agent", "extensions", "e2e"]: pattern = os.path.join(CONFORMANCE_DIR, domain, "**", "*.yaml") files.extend( f diff --git a/conformance/tests/test_e2e_scenarios.py b/conformance/tests/test_e2e_scenarios.py new file mode 100644 index 0000000000..f1fe39d315 --- /dev/null +++ b/conformance/tests/test_e2e_scenarios.py @@ -0,0 +1,295 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import jsonschema +import pytest +import referencing +import yaml + +from a2ui.core import Catalog +from a2ui.inference_formats import DirectJsonFormat, to_message_dicts +from a2ui.inference_formats.express import ExpressFormat +from a2ui.parser import A2uiPart, ErrorPart, TextPart + +CONFORMANCE_DIR = Path(__file__).resolve().parent.parent +REPO_ROOT = CONFORMANCE_DIR.parent + +V10_COMMON_TYPES = json.loads( + (REPO_ROOT / "specification" / "v1_0" / "json" / "common_types.json").read_text( + encoding="utf-8" + ) +) +V10_AGENT_TO_RENDERER = json.loads( + ( + REPO_ROOT / "specification" / "v1_0" / "json" / "agent_to_renderer.json" + ).read_text(encoding="utf-8") +) +V10_RENDERER_TO_AGENT = json.loads( + ( + REPO_ROOT / "specification" / "v1_0" / "json" / "renderer_to_agent.json" + ).read_text(encoding="utf-8") +) +V10_CATALOG_STUB = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v1_0/catalog.json", + "$defs": { + "anyComponent": {"type": "object", "additionalProperties": True}, + "anyFunction": {"type": "object", "additionalProperties": True}, + }, +} + +V091_COMMON_TYPES = json.loads( + ( + REPO_ROOT / "specification" / "v0_9_1" / "json" / "common_types.json" + ).read_text(encoding="utf-8") +) +V091_CATALOG_STUB = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v0_9/catalog.json", + "$defs": { + "anyComponent": {"type": "object", "additionalProperties": True}, + "anyFunction": {"type": "object", "additionalProperties": True}, + }, +} +V091_SERVER_TO_CLIENT = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v0_9/server_to_client.json", + **json.loads( + ( + REPO_ROOT + / "specification" + / "v0_9_1" + / "json" + / "server_to_client.json" + ).read_text(encoding="utf-8") + ), +} +V091_CLIENT_TO_SERVER = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://a2ui.org/specification/v0_9/client_to_server.json", + **json.loads( + ( + REPO_ROOT + / "specification" + / "v0_9_1" + / "json" + / "client_to_server.json" + ).read_text(encoding="utf-8") + ), +} + +SPEC_REGISTRY = referencing.Registry().with_resources([ + ( + V10_COMMON_TYPES["$id"], + referencing.Resource.from_contents(V10_COMMON_TYPES), + ), + ( + V10_AGENT_TO_RENDERER["$id"], + referencing.Resource.from_contents(V10_AGENT_TO_RENDERER), + ), + ( + V10_RENDERER_TO_AGENT["$id"], + referencing.Resource.from_contents(V10_RENDERER_TO_AGENT), + ), + ( + V10_CATALOG_STUB["$id"], + referencing.Resource.from_contents(V10_CATALOG_STUB), + ), + ( + V091_COMMON_TYPES["$id"], + referencing.Resource.from_contents(V091_COMMON_TYPES), + ), + ( + V091_SERVER_TO_CLIENT["$id"], + referencing.Resource.from_contents(V091_SERVER_TO_CLIENT), + ), + ( + V091_CLIENT_TO_SERVER["$id"], + referencing.Resource.from_contents(V091_CLIENT_TO_SERVER), + ), + ( + V091_CATALOG_STUB["$id"], + referencing.Resource.from_contents(V091_CATALOG_STUB), + ), +]) + +V10_AGENT_TO_RENDERER_VALIDATOR = jsonschema.Draft202012Validator( + V10_AGENT_TO_RENDERER, registry=SPEC_REGISTRY +) +V10_RENDERER_TO_AGENT_VALIDATOR = jsonschema.Draft202012Validator( + V10_RENDERER_TO_AGENT, registry=SPEC_REGISTRY +) +V091_SERVER_TO_CLIENT_VALIDATOR = jsonschema.Draft202012Validator( + V091_SERVER_TO_CLIENT, registry=SPEC_REGISTRY +) +V091_CLIENT_TO_SERVER_VALIDATOR = jsonschema.Draft202012Validator( + V091_CLIENT_TO_SERVER, registry=SPEC_REGISTRY +) + +SAMPLE_PLACEHOLDERS = { + "timestamp": "2026-01-01T00:00:00Z", + "functionCallId": "call-1", +} + + +def substitute_match_any(value: Any, field_name: str = "") -> Any: + if isinstance(value, dict): + if value == {"$match": "any"}: + return SAMPLE_PLACEHOLDERS.get(field_name, "sample") + return {k: substitute_match_any(v, k) for k, v in value.items()} + if isinstance(value, list): + return [substitute_match_any(item, field_name) for item in value] + return value + + +def part_to_dict(part: Any) -> dict[str, Any]: + if isinstance(part, TextPart): + return {"text": part.text} + if isinstance(part, A2uiPart): + return {"a2ui": to_message_dicts(part.a2ui)} + if isinstance(part, ErrorPart): + return {"error": {"category": getattr(part.error, "category", "unknown")}} + raise TypeError(f"Unexpected part type: {type(part)}") + + +def get_e2e_scenarios() -> list[tuple[str, dict[str, Any]]]: + scenarios: list[tuple[str, dict[str, Any]]] = [] + e2e_dir = CONFORMANCE_DIR / "e2e" + if not e2e_dir.exists(): + return scenarios + for yaml_path in sorted(e2e_dir.rglob("*.yaml")): + data = yaml.safe_load(yaml_path.read_text(encoding="utf-8")) or [] + for case in data: + if isinstance(case, dict) and case.get("action") == "e2e_scenario": + scenarios.append((case["name"], case)) + return scenarios + + +def test_unique_scenario_names(): + scenarios = get_e2e_scenarios() + assert scenarios, "No e2e scenarios found in conformance/e2e/**" + names = [name for name, _ in scenarios] + assert len(names) == len(set(names)), f"Duplicate scenario names found: {names}" + + +@pytest.mark.parametrize("name, scenario", get_e2e_scenarios()) +def test_e2e_scenario_validation(name: str, scenario: dict[str, Any]): + version = scenario["protocolVersion"] + catalogs = [] + for cat_path in scenario["catalogPaths"]: + full_path = REPO_ROOT / cat_path + cat_data = json.loads(full_path.read_text(encoding="utf-8")) + catalogs.append(Catalog.from_json(cat_data)) + + # Track component IDs seen per surface + surface_components: dict[str, set[str]] = {} + + for turn_idx, turn in enumerate(scenario["turns"]): + # 1. Validate expectWire messages against agent_to_renderer / server_to_client schema + for wire_part in turn.get("expectWire", []): + if "a2ui" in wire_part: + for msg in wire_part["a2ui"]: + if version == "1.0": + V10_AGENT_TO_RENDERER_VALIDATOR.validate(msg) + else: + norm = copy.deepcopy(msg) + if norm.get("version") == "v0.9.1": + norm["version"] = "v0.9" + V091_SERVER_TO_CLIENT_VALIDATOR.validate(norm) + + # Collect component IDs + for key in ("createSurface", "updateComponents"): + if key in msg: + payload = msg[key] + surface_id = payload.get("surfaceId", "") + comps = payload.get("components", []) + comp_ids = {c["id"] for c in comps if isinstance(c, dict) and "id" in c} + surface_components.setdefault(surface_id, set()).update(comp_ids) + + # 2. Validate expectOutbound envelopes + for env in turn.get("expectOutbound", []): + subst = substitute_match_any(env) + env_version = subst.get("version", version) + if env_version == "1.0": + V10_RENDERER_TO_AGENT_VALIDATOR.validate(subst) + else: + norm = copy.deepcopy(subst) + if norm.get("version") == "v0.9.1": + norm["version"] = "v0.9" + V091_CLIENT_TO_SERVER_VALIDATOR.validate(norm) + + # 3. Validate uiOps targets exist in that surface's components + for op in turn.get("uiOps", []): + surface_id = op["surfaceId"] + target = op["targetNode"] + known = surface_components.get(surface_id, set()) + assert target in known, ( + f"Turn {turn_idx}: uiOp targetNode '{target}' not found in surface '{surface_id}' components: {known}" + ) + + # 4. Validate chunk presence for every declared format and verify parsing + for fmt_name in scenario["formats"]: + if fmt_name == "direct_json": + parser = DirectJsonFormat(catalogs=catalogs).create_parser() + elif fmt_name == "express": + first_surface = next(iter(surface_components.keys()), "main") + parser = ExpressFormat( + catalogs=catalogs, + surface_id=first_surface, + version=version, + ).create_parser() + else: + raise ValueError(f"Unsupported format in test: {fmt_name}") + + state = parser._empty_state() + for turn_idx, turn in enumerate(scenario["turns"]): + assert fmt_name in turn["modelChunks"], ( + f"Turn {turn_idx} missing chunks for format '{fmt_name}'" + ) + chunks = turn["modelChunks"][fmt_name] + assert isinstance(chunks, list) and chunks, ( + f"Turn {turn_idx} format '{fmt_name}' chunks must be a non-empty list" + ) + + # Stream chunks through parser + stream = parser.open_stream(state=state) + parts = [] + for chunk in chunks: + parts.extend(stream.feed(chunk)) + parts.extend(stream.close()) + + # Convert parts and merge adjacent text parts + actual_parts: list[dict[str, Any]] = [] + for p in parts: + p_dict = part_to_dict(p) + if ( + "text" in p_dict + and actual_parts + and "text" in actual_parts[-1] + ): + actual_parts[-1]["text"] += p_dict["text"] + else: + actual_parts.append(p_dict) + + expected_parts = turn["expectWire"] + assert actual_parts == expected_parts, ( + f"Turn {turn_idx} format '{fmt_name}' produced:\n{actual_parts}\nExpected:\n{expected_parts}" + ) From ff3439c1e49109fd83d15d0f96eb9bfd7bc3d683 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 01:46:52 +0000 Subject: [PATCH 40/48] test(conformance): renderer-visible outcome of a transcript and compare tool --- conformance/scripts/compare_e2e_outcome.py | 134 +++++++++++++ conformance/scripts/e2e_outcome.py | 189 ++++++++++++++++++ conformance/tests/test_e2e_outcome.py | 215 +++++++++++++++++++++ 3 files changed, 538 insertions(+) create mode 100644 conformance/scripts/compare_e2e_outcome.py create mode 100644 conformance/scripts/e2e_outcome.py create mode 100644 conformance/tests/test_e2e_outcome.py diff --git a/conformance/scripts/compare_e2e_outcome.py b/conformance/scripts/compare_e2e_outcome.py new file mode 100644 index 0000000000..4397122154 --- /dev/null +++ b/conformance/scripts/compare_e2e_outcome.py @@ -0,0 +1,134 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import argparse +import difflib +import json +from pathlib import Path +import sys +from typing import Any + +import yaml + +try: + from conformance.scripts.e2e_outcome import outcome_of +except ImportError: + try: + from scripts.e2e_outcome import outcome_of + except ImportError: + from e2e_outcome import outcome_of + + +def _load_scenario_file(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.is_file(): + raise FileNotFoundError(f"Scenario file not found: {path_str}") + + content = path.read_text(encoding="utf-8") + try: + data = yaml.safe_load(content) + except Exception as exc: + raise ValueError(f"Failed to parse scenario file {path_str}: {exc}") from exc + + if isinstance(data, dict): + return [data] + if isinstance(data, list): + return [item for item in data if isinstance(item, dict)] + raise ValueError(f"Invalid root type in {path_str}: expected dict or list") + + +def _compare_scenarios( + golden_scenarios: list[dict[str, Any]], recorded_scenarios: list[dict[str, Any]] +) -> tuple[bool, list[str]]: + golden_by_name = {s["name"]: s for s in golden_scenarios if "name" in s} + recorded_by_name = {s["name"]: s for s in recorded_scenarios if "name" in s} + + diff_lines: list[str] = [] + all_matched = True + + for name, golden_s in golden_by_name.items(): + if name not in recorded_by_name: + all_matched = False + diff_lines.append(f"## Scenario `{name}`: Missing in recorded file\n") + continue + + recorded_s = recorded_by_name[name] + golden_outcome = outcome_of(golden_s) + recorded_outcome = outcome_of(recorded_s) + + if golden_outcome == recorded_outcome: + continue + + all_matched = False + diff_lines.append(f"## Scenario `{name}`: Outcome Mismatch\n") + + max_turns = max(len(golden_outcome), len(recorded_outcome)) + for turn_idx in range(max_turns): + g_turn = golden_outcome[turn_idx] if turn_idx < len(golden_outcome) else None + r_turn = recorded_outcome[turn_idx] if turn_idx < len(recorded_outcome) else None + + if g_turn == r_turn: + continue + + diff_lines.append(f"### Turn {turn_idx} Diff\n") + g_str = json.dumps(g_turn, indent=2, sort_keys=True).splitlines() + r_str = json.dumps(r_turn, indent=2, sort_keys=True).splitlines() + diff = difflib.unified_diff( + g_str, r_str, fromfile=f"golden_turn_{turn_idx}", tofile=f"recorded_turn_{turn_idx}", lineterm="" + ) + diff_lines.append("```diff") + diff_lines.extend(diff) + diff_lines.append("```\n") + + return all_matched, diff_lines + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Compare renderer-visible outcomes between golden and recorded e2e transcripts." + ) + parser.add_argument("--golden", required=True, help="Path to golden scenario YAML/JSON") + parser.add_argument("--recorded", required=True, help="Path to recorded scenario YAML/JSON") + parser.add_argument("--report", required=False, default=None, help="Optional output report path (.md)") + + try: + args = parser.parse_args(argv) + except SystemExit: + return 2 + + try: + golden_scenarios = _load_scenario_file(args.golden) + recorded_scenarios = _load_scenario_file(args.recorded) + except Exception as exc: + print(f"Input error: {exc}", file=sys.stderr) + return 2 + + matches, diff_report = _compare_scenarios(golden_scenarios, recorded_scenarios) + + if args.report is not None: + report_path = Path(args.report) + report_content = "# End-to-End Outcome Comparison Report\n\n" + if matches: + report_content += "All scenario outcomes match.\n" + else: + report_content += "\n".join(diff_report) + report_path.write_text(report_content, encoding="utf-8") + + return 0 if matches else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/scripts/e2e_outcome.py b/conformance/scripts/e2e_outcome.py new file mode 100644 index 0000000000..40c405d450 --- /dev/null +++ b/conformance/scripts/e2e_outcome.py @@ -0,0 +1,189 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from a2ui.core import Catalog, MessageProcessor, NodeResolver +from a2ui.core.resolution.component_node import ComponentNode, is_component_node + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +_ID_KEYS = frozenset({"id", "surfaceId", "functionCallId", "sourceComponentId", "targetNode"}) + + +def normalize_ids(value: Any, ordinals: dict[str, dict[str, str]] | None = None) -> Any: + """Replaces model-chosen IDs with first-seen canonical ordinals, dropping timestamp.""" + if ordinals is None: + ordinals = {} + + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, val in value.items(): + if key == "timestamp": + continue + if key in _ID_KEYS and isinstance(val, str): + namespace = "id" if key in ("id", "sourceComponentId", "targetNode") else key + id_map = ordinals.setdefault(namespace, {}) + if val not in id_map: + id_map[val] = f"{namespace}_{len(id_map)}" + result[key] = id_map[val] + else: + result[key] = normalize_ids(val, ordinals) + return result + + if isinstance(value, list): + return [normalize_ids(item, ordinals) for item in value] + + return value + + +def _extract_binding_value(val: Any) -> Any: + """Extracts raw value from a resolved binding or returns val directly.""" + if hasattr(val, "value"): + return getattr(val, "value") + return val + + +def _extract_node_outcome( + node: ComponentNode, surface: Any, ordinals: dict[str, dict[str, str]] +) -> dict[str, Any]: + """Builds the canonical outcome dict {type, text?, label?, value?, enabled?, action?} for a node.""" + outcome: dict[str, Any] = {"type": node.type} + + comp = surface.components_model.get(node.component_id) + props = comp.properties if comp is not None else {} + + # Extract text if present + if "text" in node.props: + outcome["text"] = _extract_binding_value(node.props["text"]) + elif "text" in props and isinstance(props["text"], str): + outcome["text"] = props["text"] + + # Extract label if present + if "label" in node.props: + outcome["label"] = _extract_binding_value(node.props["label"]) + elif "label" in props and isinstance(props["label"], str): + outcome["label"] = props["label"] + + # Extract value if present + if "value" in node.props: + outcome["value"] = _extract_binding_value(node.props["value"]) + elif "value" in props: + outcome["value"] = _extract_binding_value(props["value"]) + + # Extract enabled state if present + if "enabled" in node.props: + outcome["enabled"] = bool(_extract_binding_value(node.props["enabled"])) + elif "disabled" in node.props: + outcome["enabled"] = not bool(_extract_binding_value(node.props["disabled"])) + elif "disabled" in props: + outcome["enabled"] = not bool(props["disabled"]) + + # Extract action if present + action = props.get("action") or getattr(node, "action", None) + if action is not None: + outcome["action"] = normalize_ids(action, ordinals) + + return outcome + + +def _get_child_nodes(node: ComponentNode) -> list[ComponentNode]: + """Finds all child ComponentNode instances in declared hierarchy order.""" + children: list[ComponentNode] = [] + + if "child" in node.props and is_component_node(node.props["child"]): + children.append(node.props["child"]) + + if "children" in node.props and isinstance(node.props["children"], list): + for child in node.props["children"]: + if is_component_node(child): + children.append(child) + + for key, val in node.props.items(): + if key in ("child", "children"): + continue + if is_component_node(val): + children.append(val) + elif isinstance(val, list): + for item in val: + if is_component_node(item): + children.append(item) + + return children + + +def _walk_tree( + node: ComponentNode, surface: Any, ordinals: dict[str, dict[str, str]] +) -> list[dict[str, Any]]: + """Walks the resolved tree depth-first, producing ordered node outcomes.""" + nodes = [_extract_node_outcome(node, surface, ordinals)] + for child in _get_child_nodes(node): + nodes.extend(_walk_tree(child, surface, ordinals)) + return nodes + + +def _load_catalogs(catalog_paths: list[str]) -> list[Catalog]: + """Loads Catalog objects from repository-relative paths.""" + catalogs: list[Catalog] = [] + for cat_path in catalog_paths: + full_path = REPO_ROOT / cat_path if not Path(cat_path).is_absolute() else Path(cat_path) + data = json.loads(full_path.read_text(encoding="utf-8")) + catalogs.append(Catalog.from_json(data)) + return catalogs + + +def outcome_of(scenario: dict[str, Any]) -> list[dict[str, Any]]: + """Calculates the renderer-visible outcome across all turns of an e2e scenario.""" + catalogs = _load_catalogs(scenario.get("catalogPaths", [])) + processor = MessageProcessor(catalogs=catalogs) + default_catalog = catalogs[0] if catalogs else None + + ordinals: dict[str, dict[str, str]] = {} + turn_outcomes: list[dict[str, Any]] = [] + + for turn in scenario.get("turns", []): + # 1. Process all a2ui messages for this turn + for wire_part in turn.get("expectWire", []): + if isinstance(wire_part, dict) and "a2ui" in wire_part: + processor.process_messages(wire_part["a2ui"]) + + # 2. Extract resolved node trees for all current surfaces + surfaces_outcome: dict[str, list[dict[str, Any]]] = {} + for surface_id in sorted(processor.model.surfaces): + surface = processor.model.get_surface(surface_id) + if surface is None: + continue + cat = surface.default_catalog or default_catalog + resolver = NodeResolver(surface, catalog=cat) + root = resolver.root + norm_surface_id = normalize_ids({"surfaceId": surface_id}, ordinals)["surfaceId"] + if root is not None: + surfaces_outcome[norm_surface_id] = _walk_tree(root, surface, ordinals) + else: + surfaces_outcome[norm_surface_id] = [] + + # 3. Extract and normalize outbound envelopes + raw_outbound = turn.get("expectOutbound", []) + normalized_outbound = [normalize_ids(env, ordinals) for env in raw_outbound] + + turn_outcomes.append({ + "surfaces": surfaces_outcome, + "outbound": normalized_outbound, + }) + + return turn_outcomes diff --git a/conformance/tests/test_e2e_outcome.py b/conformance/tests/test_e2e_outcome.py new file mode 100644 index 0000000000..26c78a05c5 --- /dev/null +++ b/conformance/tests/test_e2e_outcome.py @@ -0,0 +1,215 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import copy +import json +from pathlib import Path +import sys +import pytest +import yaml + +CONFORMANCE_DIR = Path(__file__).resolve().parent.parent +if str(CONFORMANCE_DIR) not in sys.path: + sys.path.insert(0, str(CONFORMANCE_DIR)) + +from scripts.compare_e2e_outcome import main as compare_main +from scripts.e2e_outcome import normalize_ids, outcome_of + +CONFORMANCE_DIR = Path(__file__).resolve().parent.parent + + +@pytest.fixture +def base_scenario() -> dict: + scenario_path = CONFORMANCE_DIR / "e2e" / "newsletter_signup.yaml" + data = yaml.safe_load(scenario_path.read_text(encoding="utf-8")) + return copy.deepcopy(data[0]) + + +def test_normalize_ids_ordinals_and_timestamp(): + ordinals: dict[str, dict[str, str]] = {} + obj1 = { + "timestamp": "2026-11-03T00:00:00Z", + "surfaceId": "my-surface", + "id": "btn-1", + "nested": {"targetNode": "btn-1", "sourceComponentId": "btn-1"}, + } + normalized = normalize_ids(obj1, ordinals) + assert "timestamp" not in normalized + assert normalized["surfaceId"] == "surfaceId_0" + assert normalized["id"] == "id_0" + assert normalized["nested"]["targetNode"] == "id_0" + assert normalized["nested"]["sourceComponentId"] == "id_0" + + # Second surfaceId gets surfaceId_1 + obj2 = {"surfaceId": "another-surface"} + assert normalize_ids(obj2, ordinals)["surfaceId"] == "surfaceId_1" + + +def test_same_outcome_different_model_ids_and_text_wording(base_scenario): + golden_outcome = outcome_of(base_scenario) + + recorded_scenario = copy.deepcopy(base_scenario) + # 1. Change text wording in conversational messages + recorded_scenario["turns"][0]["userText"] = "Different user prompt entirely." + recorded_scenario["turns"][0]["expectWire"][0]["text"] = "Different assistant intro wording." + recorded_scenario["turns"][1]["expectWire"][0]["text"] = "Different thank you text." + + # 2. Change model-chosen IDs + def rename_ids(obj): + if isinstance(obj, dict): + res = {} + for k, v in obj.items(): + if k == "surfaceId" and v == "newsletter-signup": + res[k] = "custom-surface-999" + elif k == "id" and isinstance(v, str): + res[k] = "root" if v == "root" else f"custom-{v}" + elif k == "child" and isinstance(v, str): + res[k] = "root" if v == "root" else f"custom-{v}" + elif k == "children" and isinstance(v, list): + res[k] = [f"custom-{x}" if isinstance(x, str) else rename_ids(x) for x in v] + elif k == "sourceComponentId": + res[k] = f"custom-{v}" + elif k == "targetNode": + res[k] = f"custom-{v}" + else: + res[k] = rename_ids(v) + return res + if isinstance(obj, list): + return [rename_ids(x) for x in obj] + return obj + + recorded_scenario = rename_ids(recorded_scenario) + + recorded_outcome = outcome_of(recorded_scenario) + assert golden_outcome == recorded_outcome + + +def test_changed_outcome_missing_button(base_scenario): + golden_outcome = outcome_of(base_scenario) + + modified = copy.deepcopy(base_scenario) + # Remove the Button from components in turn 0 + t0_wire = modified["turns"][0]["expectWire"] + comps = t0_wire[1]["a2ui"][0]["createSurface"]["components"] + # Filter out subscribeBtn and subscribeText + t0_wire[1]["a2ui"][0]["createSurface"]["components"] = [ + c for c in comps if c["id"] not in ("subscribeBtn", "subscribeText") + ] + # Remove subscribeBtn from Column children + for c in t0_wire[1]["a2ui"][0]["createSurface"]["components"]: + if c["id"] == "container": + c["children"] = [x for x in c["children"] if x != "subscribeBtn"] + + modified_outcome = outcome_of(modified) + assert golden_outcome != modified_outcome + + +def test_changed_outcome_changed_label(base_scenario): + golden_outcome = outcome_of(base_scenario) + + modified = copy.deepcopy(base_scenario) + # Change TextField label in turn 0 + t0_wire = modified["turns"][0]["expectWire"] + for c in t0_wire[1]["a2ui"][0]["createSurface"]["components"]: + if c["id"] == "emailInput": + c["label"] = "Your Work Email Address" + + modified_outcome = outcome_of(modified) + assert golden_outcome != modified_outcome + + +def test_changed_outcome_different_action_name(base_scenario): + golden_outcome = outcome_of(base_scenario) + + modified = copy.deepcopy(base_scenario) + # Change button action event name + t0_wire = modified["turns"][0]["expectWire"] + for c in t0_wire[1]["a2ui"][0]["createSurface"]["components"]: + if c["id"] == "subscribeBtn": + c["action"]["event"]["name"] = "register_for_newsletter" + + modified_outcome = outcome_of(modified) + assert golden_outcome != modified_outcome + + +def test_changed_outcome_extra_outbound_envelope(base_scenario): + golden_outcome = outcome_of(base_scenario) + + modified = copy.deepcopy(base_scenario) + # Add an extra outbound envelope to turn 0 + modified["turns"][0]["expectOutbound"].append({ + "version": "1.0", + "action": { + "name": "secondary_action", + "surfaceId": "newsletter-signup", + "sourceComponentId": "subscribeBtn", + "timestamp": "2026-11-03T00:00:00Z", + }, + }) + + modified_outcome = outcome_of(modified) + assert golden_outcome != modified_outcome + + +def test_compare_cli_exit_codes_and_report(tmp_path, base_scenario): + golden_file = tmp_path / "golden.yaml" + golden_file.write_text(yaml.dump([base_scenario]), encoding="utf-8") + + # 1. Matching recorded file -> exit code 0 + recorded_file = tmp_path / "recorded_match.yaml" + recorded_file.write_text(yaml.dump([base_scenario]), encoding="utf-8") + report_file = tmp_path / "report_match.md" + + code_match = compare_main([ + "--golden", str(golden_file), + "--recorded", str(recorded_file), + "--report", str(report_file), + ]) + assert code_match == 0 + assert "All scenario outcomes match." in report_file.read_text(encoding="utf-8") + + # 2. Differing recorded file -> exit code 1 + diff_scenario = copy.deepcopy(base_scenario) + diff_scenario["turns"][0]["expectOutbound"].append({ + "version": "1.0", + "action": { + "name": "extra_outbound", + "surfaceId": "newsletter-signup", + "sourceComponentId": "subscribeBtn", + }, + }) + recorded_diff = tmp_path / "recorded_diff.yaml" + recorded_diff.write_text(yaml.dump([diff_scenario]), encoding="utf-8") + report_diff = tmp_path / "report_diff.md" + + code_diff = compare_main([ + "--golden", str(golden_file), + "--recorded", str(recorded_diff), + "--report", str(report_diff), + ]) + assert code_diff == 1 + assert "Outcome Mismatch" in report_diff.read_text(encoding="utf-8") + + # 3. Input error: non-existent file -> exit code 2 + code_err = compare_main([ + "--golden", str(tmp_path / "non_existent.yaml"), + "--recorded", str(recorded_file), + ]) + assert code_err == 2 + + # 4. Input error: missing required flag -> exit code 2 + code_bad_args = compare_main(["--golden", str(golden_file)]) + assert code_bad_args == 2 From 4cd51c7299ccefacacd5305d95c5482cce86deac Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 01:49:06 +0000 Subject: [PATCH 41/48] style(conformance): format E2E scenario files with prettier and pyink --- conformance/README.md | 2 + ...onformance_harness_implementation_guide.md | 3 ++ conformance/conformance_schema.json | 5 +- conformance/e2e/newsletter_signup.yaml | 2 +- conformance/scripts/compare_e2e_outcome.py | 34 +++++++++--- conformance/scripts/e2e_outcome.py | 16 ++++-- conformance/tests/test_e2e_outcome.py | 33 ++++++++---- conformance/tests/test_e2e_scenarios.py | 52 +++++++++---------- 8 files changed, 96 insertions(+), 51 deletions(-) diff --git a/conformance/README.md b/conformance/README.md index 9f5d2f3e1a..2c4a28b5da 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -81,11 +81,13 @@ End-to-end conformance scenarios in `e2e/` capture complete multi-turn conversat - **Renderer half**: The `a2ui` messages from each turn are dispatched into framework adapter harnesses (or `MessageProcessor`), which verify the resulting resolved view trees (`expectViewTree`), execute simulated user interactions (`uiOps`), and assert outbound event envelopes (`expectOutbound`) and data model states (`expectDataModel`). Harnesses running the agent half: + - Python Agent: `python/a2ui_agent` - TypeScript Agent: `typescript/agents/a2ui_agent` - Dart Agent: `dart/a2ui_agent` Harnesses running the renderer half: + - Web Renderers: `@a2ui/lit`, `@a2ui/react`, `@a2ui/angular` - Flutter Renderer: `dart/a2ui_flutter` - SwiftUI Renderer: `swift/swiftui` diff --git a/conformance/conformance_harness_implementation_guide.md b/conformance/conformance_harness_implementation_guide.md index d2a0f8e6df..a2648abfce 100644 --- a/conformance/conformance_harness_implementation_guide.md +++ b/conformance/conformance_harness_implementation_guide.md @@ -353,6 +353,7 @@ The end-to-end golden-transcript harness exercises complete conversational scena ### Agent half algorithm The agent half tests response chunk parsing and wire fidelity: + 1. Initialize the inference format parser for each declared format (`direct_json`, `express`), configured with the scenario's catalogs. 2. Initialize and maintain an empty parser `RendererState` across the turns of the scenario. 3. For each turn in `scenario["turns"]`: @@ -364,6 +365,7 @@ The agent half tests response chunk parsing and wire fidelity: ### Renderer half algorithm The renderer half tests UI tree construction, interaction, and outbound messaging: + 1. Initialize a `MessageProcessor` or UI framework adapter with the catalogs declared in `catalogPaths`. 2. For each turn in `scenario["turns"]`: - Feed the `a2ui` messages from `turn["expectWire"]` in order (chat text parts are ignored by renderers). @@ -383,6 +385,7 @@ The renderer half tests UI tree construction, interaction, and outbound messagin ### Renderer-visible outcome definition A transcript's renderer-visible outcome is defined by `outcome_of(scenario)` in `conformance/scripts/e2e_outcome.py`: + - Each turn's `a2ui` messages are applied to `MessageProcessor`. - Each active surface's resolved node tree is walked depth-first, mapping each node to `{type, text?, label?, value?, enabled?, action?}`. - Model-chosen IDs (`id`, `surfaceId`, `functionCallId`) are mapped to canonical first-seen ordinals. diff --git a/conformance/conformance_schema.json b/conformance/conformance_schema.json index 2c96e8dcda..11f5f40814 100644 --- a/conformance/conformance_schema.json +++ b/conformance/conformance_schema.json @@ -2163,7 +2163,10 @@ "type": "object", "required": ["name", "action", "protocolVersion", "catalogPaths", "formats", "turns"], "properties": { - "name": {"type": "string", "description": "Unique scenario name across conformance/e2e/**."}, + "name": { + "type": "string", + "description": "Unique scenario name across conformance/e2e/**." + }, "action": {"const": "e2e_scenario"}, "description": {"type": "string"}, "protocolVersion": {"$ref": "#/$defs/ProtocolVersion"}, diff --git a/conformance/e2e/newsletter_signup.yaml b/conformance/e2e/newsletter_signup.yaml index 333d845142..a4c309d75e 100644 --- a/conformance/e2e/newsletter_signup.yaml +++ b/conformance/e2e/newsletter_signup.yaml @@ -25,7 +25,7 @@ kind: hand_authored from: dart/a2ui_flutter/example/test/fixtures/newsletter_replies.json turns: - - userText: "Show a newsletter sign-up form with an email field and a \"Subscribe\" button." + - userText: 'Show a newsletter sign-up form with an email field and a "Subscribe" button.' modelChunks: direct_json: - "Here is a sign-up form for your newsletter.\n\n int: parser = argparse.ArgumentParser( - description="Compare renderer-visible outcomes between golden and recorded e2e transcripts." + description=( + "Compare renderer-visible outcomes between golden and recorded e2e" + " transcripts." + ) + ) + parser.add_argument( + "--golden", required=True, help="Path to golden scenario YAML/JSON" + ) + parser.add_argument( + "--recorded", required=True, help="Path to recorded scenario YAML/JSON" + ) + parser.add_argument( + "--report", + required=False, + default=None, + help="Optional output report path (.md)", ) - parser.add_argument("--golden", required=True, help="Path to golden scenario YAML/JSON") - parser.add_argument("--recorded", required=True, help="Path to recorded scenario YAML/JSON") - parser.add_argument("--report", required=False, default=None, help="Optional output report path (.md)") try: args = parser.parse_args(argv) diff --git a/conformance/scripts/e2e_outcome.py b/conformance/scripts/e2e_outcome.py index 40c405d450..c64cbb211c 100644 --- a/conformance/scripts/e2e_outcome.py +++ b/conformance/scripts/e2e_outcome.py @@ -23,7 +23,9 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent -_ID_KEYS = frozenset({"id", "surfaceId", "functionCallId", "sourceComponentId", "targetNode"}) +_ID_KEYS = frozenset( + {"id", "surfaceId", "functionCallId", "sourceComponentId", "targetNode"} +) def normalize_ids(value: Any, ordinals: dict[str, dict[str, str]] | None = None) -> Any: @@ -37,7 +39,9 @@ def normalize_ids(value: Any, ordinals: dict[str, dict[str, str]] | None = None) if key == "timestamp": continue if key in _ID_KEYS and isinstance(val, str): - namespace = "id" if key in ("id", "sourceComponentId", "targetNode") else key + namespace = ( + "id" if key in ("id", "sourceComponentId", "targetNode") else key + ) id_map = ordinals.setdefault(namespace, {}) if val not in id_map: id_map[val] = f"{namespace}_{len(id_map)}" @@ -141,7 +145,9 @@ def _load_catalogs(catalog_paths: list[str]) -> list[Catalog]: """Loads Catalog objects from repository-relative paths.""" catalogs: list[Catalog] = [] for cat_path in catalog_paths: - full_path = REPO_ROOT / cat_path if not Path(cat_path).is_absolute() else Path(cat_path) + full_path = ( + REPO_ROOT / cat_path if not Path(cat_path).is_absolute() else Path(cat_path) + ) data = json.loads(full_path.read_text(encoding="utf-8")) catalogs.append(Catalog.from_json(data)) return catalogs @@ -171,7 +177,9 @@ def outcome_of(scenario: dict[str, Any]) -> list[dict[str, Any]]: cat = surface.default_catalog or default_catalog resolver = NodeResolver(surface, catalog=cat) root = resolver.root - norm_surface_id = normalize_ids({"surfaceId": surface_id}, ordinals)["surfaceId"] + norm_surface_id = normalize_ids({"surfaceId": surface_id}, ordinals)[ + "surfaceId" + ] if root is not None: surfaces_outcome[norm_surface_id] = _walk_tree(root, surface, ordinals) else: diff --git a/conformance/tests/test_e2e_outcome.py b/conformance/tests/test_e2e_outcome.py index 26c78a05c5..7ec375ca4c 100644 --- a/conformance/tests/test_e2e_outcome.py +++ b/conformance/tests/test_e2e_outcome.py @@ -64,7 +64,9 @@ def test_same_outcome_different_model_ids_and_text_wording(base_scenario): recorded_scenario = copy.deepcopy(base_scenario) # 1. Change text wording in conversational messages recorded_scenario["turns"][0]["userText"] = "Different user prompt entirely." - recorded_scenario["turns"][0]["expectWire"][0]["text"] = "Different assistant intro wording." + recorded_scenario["turns"][0]["expectWire"][0][ + "text" + ] = "Different assistant intro wording." recorded_scenario["turns"][1]["expectWire"][0]["text"] = "Different thank you text." # 2. Change model-chosen IDs @@ -79,7 +81,10 @@ def rename_ids(obj): elif k == "child" and isinstance(v, str): res[k] = "root" if v == "root" else f"custom-{v}" elif k == "children" and isinstance(v, list): - res[k] = [f"custom-{x}" if isinstance(x, str) else rename_ids(x) for x in v] + res[k] = [ + f"custom-{x}" if isinstance(x, str) else rename_ids(x) + for x in v + ] elif k == "sourceComponentId": res[k] = f"custom-{v}" elif k == "targetNode": @@ -174,9 +179,12 @@ def test_compare_cli_exit_codes_and_report(tmp_path, base_scenario): report_file = tmp_path / "report_match.md" code_match = compare_main([ - "--golden", str(golden_file), - "--recorded", str(recorded_file), - "--report", str(report_file), + "--golden", + str(golden_file), + "--recorded", + str(recorded_file), + "--report", + str(report_file), ]) assert code_match == 0 assert "All scenario outcomes match." in report_file.read_text(encoding="utf-8") @@ -196,17 +204,22 @@ def test_compare_cli_exit_codes_and_report(tmp_path, base_scenario): report_diff = tmp_path / "report_diff.md" code_diff = compare_main([ - "--golden", str(golden_file), - "--recorded", str(recorded_diff), - "--report", str(report_diff), + "--golden", + str(golden_file), + "--recorded", + str(recorded_diff), + "--report", + str(report_diff), ]) assert code_diff == 1 assert "Outcome Mismatch" in report_diff.read_text(encoding="utf-8") # 3. Input error: non-existent file -> exit code 2 code_err = compare_main([ - "--golden", str(tmp_path / "non_existent.yaml"), - "--recorded", str(recorded_file), + "--golden", + str(tmp_path / "non_existent.yaml"), + "--recorded", + str(recorded_file), ]) assert code_err == 2 diff --git a/conformance/tests/test_e2e_scenarios.py b/conformance/tests/test_e2e_scenarios.py index f1fe39d315..fa1709a0e6 100644 --- a/conformance/tests/test_e2e_scenarios.py +++ b/conformance/tests/test_e2e_scenarios.py @@ -57,9 +57,9 @@ } V091_COMMON_TYPES = json.loads( - ( - REPO_ROOT / "specification" / "v0_9_1" / "json" / "common_types.json" - ).read_text(encoding="utf-8") + (REPO_ROOT / "specification" / "v0_9_1" / "json" / "common_types.json").read_text( + encoding="utf-8" + ) ) V091_CATALOG_STUB = { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -74,11 +74,7 @@ "$id": "https://a2ui.org/specification/v0_9/server_to_client.json", **json.loads( ( - REPO_ROOT - / "specification" - / "v0_9_1" - / "json" - / "server_to_client.json" + REPO_ROOT / "specification" / "v0_9_1" / "json" / "server_to_client.json" ).read_text(encoding="utf-8") ), } @@ -87,11 +83,7 @@ "$id": "https://a2ui.org/specification/v0_9/client_to_server.json", **json.loads( ( - REPO_ROOT - / "specification" - / "v0_9_1" - / "json" - / "client_to_server.json" + REPO_ROOT / "specification" / "v0_9_1" / "json" / "client_to_server.json" ).read_text(encoding="utf-8") ), } @@ -221,8 +213,14 @@ def test_e2e_scenario_validation(name: str, scenario: dict[str, Any]): payload = msg[key] surface_id = payload.get("surfaceId", "") comps = payload.get("components", []) - comp_ids = {c["id"] for c in comps if isinstance(c, dict) and "id" in c} - surface_components.setdefault(surface_id, set()).update(comp_ids) + comp_ids = { + c["id"] + for c in comps + if isinstance(c, dict) and "id" in c + } + surface_components.setdefault(surface_id, set()).update( + comp_ids + ) # 2. Validate expectOutbound envelopes for env in turn.get("expectOutbound", []): @@ -242,7 +240,8 @@ def test_e2e_scenario_validation(name: str, scenario: dict[str, Any]): target = op["targetNode"] known = surface_components.get(surface_id, set()) assert target in known, ( - f"Turn {turn_idx}: uiOp targetNode '{target}' not found in surface '{surface_id}' components: {known}" + f"Turn {turn_idx}: uiOp targetNode '{target}' not found in surface" + f" '{surface_id}' components: {known}" ) # 4. Validate chunk presence for every declared format and verify parsing @@ -261,13 +260,13 @@ def test_e2e_scenario_validation(name: str, scenario: dict[str, Any]): state = parser._empty_state() for turn_idx, turn in enumerate(scenario["turns"]): - assert fmt_name in turn["modelChunks"], ( - f"Turn {turn_idx} missing chunks for format '{fmt_name}'" - ) + assert ( + fmt_name in turn["modelChunks"] + ), f"Turn {turn_idx} missing chunks for format '{fmt_name}'" chunks = turn["modelChunks"][fmt_name] - assert isinstance(chunks, list) and chunks, ( - f"Turn {turn_idx} format '{fmt_name}' chunks must be a non-empty list" - ) + assert ( + isinstance(chunks, list) and chunks + ), f"Turn {turn_idx} format '{fmt_name}' chunks must be a non-empty list" # Stream chunks through parser stream = parser.open_stream(state=state) @@ -280,16 +279,13 @@ def test_e2e_scenario_validation(name: str, scenario: dict[str, Any]): actual_parts: list[dict[str, Any]] = [] for p in parts: p_dict = part_to_dict(p) - if ( - "text" in p_dict - and actual_parts - and "text" in actual_parts[-1] - ): + if "text" in p_dict and actual_parts and "text" in actual_parts[-1]: actual_parts[-1]["text"] += p_dict["text"] else: actual_parts.append(p_dict) expected_parts = turn["expectWire"] assert actual_parts == expected_parts, ( - f"Turn {turn_idx} format '{fmt_name}' produced:\n{actual_parts}\nExpected:\n{expected_parts}" + f"Turn {turn_idx} format '{fmt_name}'" + f" produced:\n{actual_parts}\nExpected:\n{expected_parts}" ) From 060852b5367560247fac8ccf0b319cb63b4e1d7e Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 02:07:11 +0000 Subject: [PATCH 42/48] test(a2ui_agent): run the e2e scenarios' agent half in Python --- .../tests/conformance/test_e2e_conformance.py | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 python/a2ui_agent/tests/conformance/test_e2e_conformance.py diff --git a/python/a2ui_agent/tests/conformance/test_e2e_conformance.py b/python/a2ui_agent/tests/conformance/test_e2e_conformance.py new file mode 100644 index 0000000000..9b9b5776a7 --- /dev/null +++ b/python/a2ui_agent/tests/conformance/test_e2e_conformance.py @@ -0,0 +1,275 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end golden-transcript conformance runner for the Python Agent SDK.""" + +from __future__ import annotations + +from collections.abc import Mapping +import json +import os +from pathlib import Path +import traceback +from typing import Any + +import pytest + +from a2ui.core import A2uiCatalogError +from a2ui.core import A2uiError +from a2ui.core import A2uiIntegrityError +from a2ui.core import A2uiParseError +from a2ui.core import A2uiRecursionError +from a2ui.core import A2uiValidationError +from a2ui.core.common import to_protocol_version +from a2ui.inference_formats import DirectJsonFormat +from a2ui.inference_formats import to_message_dicts +from a2ui.inference_formats.express import ExpressFormat +from a2ui.parser import A2uiCompilationError +from a2ui.parser import A2uiPart +from a2ui.parser import ErrorPart +from a2ui.parser import Parser +from a2ui.parser import TextPart +from a2ui.processor import CatalogConfig +from a2ui.utils import RendererState + +from .conformance_helpers import assert_known_keys +from .conformance_helpers import case_params +from .conformance_helpers import get_conformance_path +from .conformance_helpers import load_conformance_yaml + +_CATEGORY_TO_EXCEPTION = { + "ParseError": A2uiParseError, + "ValidationError": A2uiValidationError, + "CatalogError": A2uiCatalogError, + "IntegrityError": A2uiIntegrityError, + "RecursionError": A2uiRecursionError, + "CompilationError": A2uiCompilationError, +} + +_SCENARIO_KEYS = frozenset({ + "name", + "action", + "description", + "features", + "protocolVersion", + "catalogPaths", + "formats", + "source", + "turns", +}) + +_SOURCE_KEYS = frozenset({ + "kind", + "from", + "model", + "recordedAt", +}) + +_TURN_KEYS = frozenset({ + "userText", + "modelChunks", + "expectWire", + "expectViewTree", + "expectErrors", + "uiOps", + "expectDataModel", + "expectOutbound", +}) + +_UI_OP_KEYS = frozenset({ + "op", + "surfaceId", + "targetNode", + "inputValue", + "userActivation", +}) + +_DEFAULT_SURFACE_ID = "default_surface" + +SKIP_SCENARIOS: dict[str, str] = {} +KNOWN_GAPS: dict[str, str] = {} + + +def load_e2e_scenarios() -> list[tuple[str, dict[str, Any]]]: + """Globs `conformance/e2e/**/*.yaml` and returns `(suite, scenario)` pairs.""" + e2e_dir = Path(get_conformance_path("e2e")) + yaml_files = sorted(e2e_dir.rglob("*.yaml")) + scenarios: list[tuple[str, dict[str, Any]]] = [] + for yaml_file in yaml_files: + rel_suite = f"e2e/{yaml_file.relative_to(e2e_dir).as_posix()}" + loaded = load_conformance_yaml(rel_suite) + if not isinstance(loaded, list): + raise ValueError(f"Expected list of scenarios in {rel_suite}") + for item in loaded: + scenarios.append((rel_suite, item)) + assert scenarios, "No E2E conformance scenarios found under conformance/e2e/" + return scenarios + + +def _build_parametrized_cases() -> list[Any]: + grouped: dict[str, list[dict[str, Any]]] = {} + for suite, scenario in load_e2e_scenarios(): + grouped.setdefault(suite, []).append(scenario) + params: list[Any] = [] + for suite, cases in grouped.items(): + params.extend(case_params(suite, cases, skip=SKIP_SCENARIOS, xfail=KNOWN_GAPS)) + return params + + +def _resolve_catalog_path(cat_path: str) -> str: + candidate = get_conformance_path(cat_path) + if os.path.exists(candidate): + return candidate + return os.path.normpath(get_conformance_path(os.path.join("..", cat_path))) + + +def _load_scenario_catalogs(scenario: Mapping[str, Any]) -> list[Any]: + protocol_version = str(scenario.get("protocolVersion", "1.0")) + if protocol_version.startswith("0."): + protocol_version = f"v{protocol_version}" + elif protocol_version == "v1.0": + protocol_version = "1.0" + + catalogs = [] + for rel_path in scenario["catalogPaths"]: + full_path = _resolve_catalog_path(str(rel_path)) + with open(full_path, "r", encoding="utf-8") as handle: + doc = json.load(handle) + config = CatalogConfig.from_path( + catalog_path=full_path, + protocol_version=(None if "protocolVersion" in doc else protocol_version), + catalog_id=( + None + if "catalogId" in doc + else os.path.basename(str(rel_path)).removesuffix(".json") + ), + ) + catalogs.append(config.transformed_catalog) + return catalogs + + +def _make_parser(format_name: str, catalogs: list[Any]) -> Parser: + if format_name == "direct_json": + return DirectJsonFormat(catalogs=catalogs).create_parser() + if format_name == "express": + version = to_protocol_version(catalogs[0].protocol_version).value + return ExpressFormat( + catalogs=catalogs, + surface_id=_DEFAULT_SURFACE_ID, + version=version, + ).create_parser() + raise ValueError(f"Unsupported inference format: {format_name!r}") + + +def _category_of(error: BaseException) -> str: + for category, cls in _CATEGORY_TO_EXCEPTION.items(): + if isinstance(error, cls): + return category + if isinstance(error, A2uiError): + return type(error).__name__ + return type(error).__name__ + + +def _part_to_dict(part: Any) -> dict[str, Any]: + if isinstance(part, TextPart): + return {"text": part.text} + if isinstance(part, A2uiPart): + return {"a2ui": to_message_dicts(part.a2ui)} + if isinstance(part, ErrorPart): + return {"error": {"category": _category_of(part.error)}} + raise TypeError(f"Unexpected response part type: {type(part)}") + + +def _merge_text_parts(parts: list[dict[str, Any]]) -> list[dict[str, Any]]: + merged: list[dict[str, Any]] = [] + for part in parts: + if merged and "text" in merged[-1] and "text" in part: + merged[-1] = {"text": merged[-1]["text"] + part["text"]} + else: + merged.append(part) + return merged + + +def _assert_scenario_keys(scenario: Mapping[str, Any]) -> None: + assert scenario.get("action") == "e2e_scenario" + assert_known_keys(scenario, _SCENARIO_KEYS, "e2e_scenario case") + source = scenario.get("source") + if isinstance(source, Mapping): + assert_known_keys(source, _SOURCE_KEYS, "e2e_scenario.source") + turns = scenario.get("turns") + assert isinstance(turns, list) and turns, "Scenario must contain turns" + for turn_idx, turn in enumerate(turns): + assert isinstance(turn, Mapping) + assert_known_keys(turn, _TURN_KEYS, f"turns[{turn_idx}]") + for op_idx, ui_op in enumerate(turn.get("uiOps", [])): + if isinstance(ui_op, Mapping): + assert_known_keys( + ui_op, _UI_OP_KEYS, f"turns[{turn_idx}].uiOps[{op_idx}]" + ) + + +def _run_format( + scenario: Mapping[str, Any], format_name: str, catalogs: list[Any] +) -> None: + parser = _make_parser(format_name, catalogs) + stream_state = RendererState(catalogs) + single_shot_state = RendererState(catalogs) + + for turn_idx, turn in enumerate(scenario["turns"]): + model_chunks = turn["modelChunks"] + assert ( + format_name in model_chunks + ), f"Turn {turn_idx} missing chunks for format {format_name!r}" + chunks: list[str] = model_chunks[format_name] + expected_wire: list[dict[str, Any]] = turn["expectWire"] + + stream = parser.open_stream(state=stream_state) + streamed_parts: list[dict[str, Any]] = [] + for chunk in chunks: + for part in stream.feed(chunk): + streamed_parts.append(_part_to_dict(part)) + for part in stream.close(): + streamed_parts.append(_part_to_dict(part)) + + merged_streamed = _merge_text_parts(streamed_parts) + assert merged_streamed == expected_wire, ( + f"Format {format_name!r} turn {turn_idx} streaming mismatch:\n" + f"actual={merged_streamed}\nexpected={expected_wire}" + ) + + single_shot_raw = parser.parse_response( + "".join(chunks), state=single_shot_state + ) + merged_single_shot = _merge_text_parts( + [_part_to_dict(part) for part in single_shot_raw] + ) + assert merged_single_shot == expected_wire, ( + f"Format {format_name!r} turn {turn_idx} single-shot mismatch:\n" + f"actual={merged_single_shot}\nexpected={expected_wire}" + ) + + +@pytest.mark.parametrize("suite, case", _build_parametrized_cases()) +def test_e2e_conformance(suite: str, case: dict[str, Any]) -> None: + del suite + _assert_scenario_keys(case) + catalogs = _load_scenario_catalogs(case) + failures: list[str] = [] + for format_name in case["formats"]: + try: + _run_format(case, str(format_name), catalogs) + except Exception as exc: # pylint: disable=broad-except + failures.append(f"[{format_name}] {exc}\n{traceback.format_exc()}") + if failures: + pytest.fail("\n".join(failures)) From a656c71b3297109300d6f0ca4a10e241920edb9a Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 02:07:13 +0000 Subject: [PATCH 43/48] test(a2ui_agent-ts): run the e2e scenarios' agent half in TypeScript --- .../tests/conformance/e2e-scenarios.test.ts | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 typescript/agents/a2ui_agent/tests/conformance/e2e-scenarios.test.ts diff --git a/typescript/agents/a2ui_agent/tests/conformance/e2e-scenarios.test.ts b/typescript/agents/a2ui_agent/tests/conformance/e2e-scenarios.test.ts new file mode 100644 index 0000000000..4b5ce201cd --- /dev/null +++ b/typescript/agents/a2ui_agent/tests/conformance/e2e-scenarios.test.ts @@ -0,0 +1,249 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {existsSync, readdirSync} from 'fs'; +import {join, resolve} from 'path'; + +import {expect} from 'vitest'; + +import { + DirectJsonFormatFactory, + ExpressFormatFactory, + FileSystemCatalogProvider, + Parser, + RendererState, + StreamPart, +} from '../../src/index.js'; +import {CatalogApi} from '../../src/internal/web-core.js'; +import {CONFORMANCE_ROOT, LoadedCase, loadCases} from './loader.js'; +import {categoryOf, conformancePath, describeSuite} from './suite-helpers.js'; + +const KNOWN_FAILURES = new Map(); + +const SCENARIO_KEYS = new Set([ + 'name', + 'action', + 'description', + 'features', + 'protocolVersion', + 'catalogPaths', + 'formats', + 'source', + 'turns', + 'sourceFile', +]); + +const SOURCE_KEYS = new Set(['kind', 'from', 'model', 'recordedAt']); + +const TURN_KEYS = new Set([ + 'userText', + 'modelChunks', + 'expectWire', + 'expectViewTree', + 'expectErrors', + 'uiOps', + 'expectDataModel', + 'expectOutbound', +]); + +const UI_OP_KEYS = new Set(['op', 'surfaceId', 'targetNode', 'inputValue', 'userActivation']); + +interface E2eTurn { + userText?: string | null; + modelChunks: Record; + expectWire: Array>; + expectViewTree?: Record | null>; + expectErrors?: Array>; + uiOps?: Array>; + expectDataModel?: Record; + expectOutbound?: Array>; +} + +function findE2eYamlFiles(dir: string): string[] { + const results: string[] = []; + for (const entry of readdirSync(dir, {withFileTypes: true})) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...findE2eYamlFiles(fullPath)); + } else if (entry.isFile() && entry.name.endsWith('.yaml')) { + results.push(fullPath); + } + } + return results.sort(); +} + +function assertKnownKeys( + obj: Record, + allowed: ReadonlySet, + where: string, +): void { + const unknown = Object.keys(obj) + .filter(k => !allowed.has(k)) + .sort(); + if (unknown.length > 0) { + throw new Error(`Unknown ${where} keys: ${unknown.join(', ')}`); + } +} + +function assertScenarioKeys(testCase: LoadedCase): void { + expect(testCase.action).toBe('e2e_scenario'); + assertKnownKeys(testCase, SCENARIO_KEYS, `scenario(${testCase.name})`); + if (testCase.source && typeof testCase.source === 'object') { + assertKnownKeys( + testCase.source as Record, + SOURCE_KEYS, + `scenario(${testCase.name}).source`, + ); + } + const turns = testCase.turns as Array> | undefined; + expect(Array.isArray(turns) && turns.length > 0, 'scenario must define turns').toBe(true); + for (const [turnIdx, turn] of (turns ?? []).entries()) { + assertKnownKeys(turn, TURN_KEYS, `scenario(${testCase.name}).turns[${turnIdx}]`); + const uiOps = turn.uiOps as Array> | undefined; + for (const [opIdx, uiOp] of (uiOps ?? []).entries()) { + assertKnownKeys( + uiOp, + UI_OP_KEYS, + `scenario(${testCase.name}).turns[${turnIdx}].uiOps[${opIdx}]`, + ); + } + } +} + +function resolveCatalogPath(catPath: string): string { + const inConformance = conformancePath(catPath); + if (existsSync(inConformance)) { + return inConformance; + } + return resolve(CONFORMANCE_ROOT, '..', catPath); +} + +function loadScenarioCatalogs(testCase: LoadedCase): CatalogApi[] { + const paths = testCase.catalogPaths as string[] | undefined; + if (!paths || paths.length === 0) { + throw new Error(`Scenario ${testCase.name} declares no catalogPaths`); + } + return paths.map(catPath => new FileSystemCatalogProvider(resolveCatalogPath(catPath)).load()); +} + +function makeParser(formatName: string, catalogs: CatalogApi[], protocolVersion: string): Parser { + if (formatName === 'direct_json') { + return new DirectJsonFormatFactory().createFormat(catalogs).createParser(); + } + if (formatName === 'express') { + return new ExpressFormatFactory({ + surfaceId: 'default_surface', + version: protocolVersion, + }) + .createFormat(catalogs) + .createParser(); + } + throw new Error(`Unsupported inference format '${formatName}'`); +} + +function partToDict(part: StreamPart): Record { + if (part.type === 'text') return {text: part.text}; + if (part.type === 'error') return {error: {category: categoryOf(part.error)}}; + return {a2ui: part.a2ui}; +} + +function mergeAdjacentText(parts: Array>): Array> { + const merged: Array> = []; + for (const part of parts) { + if (merged.length > 0 && 'text' in merged[merged.length - 1] && 'text' in part) { + merged[merged.length - 1] = { + text: `${merged[merged.length - 1].text}${part.text}`, + }; + } else { + merged.push(part); + } + } + return merged; +} + +function runFormat( + testCase: LoadedCase, + formatName: string, + catalogs: CatalogApi[], + protocolVersion: string, +): void { + const parser = makeParser(formatName, catalogs, protocolVersion); + const streamState = new RendererState(catalogs); + const singleShotState = new RendererState(catalogs); + const turns = testCase.turns as E2eTurn[]; + + for (const [turnIdx, turn] of turns.entries()) { + const chunks = turn.modelChunks[formatName]; + expect( + chunks, + `scenario ${testCase.name} turn ${turnIdx} missing chunks for ${formatName}`, + ).toBeDefined(); + + const stream = parser.openStream({state: streamState}); + const streamedParts: Array> = []; + for (const chunk of chunks) { + streamedParts.push(...stream.feed(chunk).map(partToDict)); + } + streamedParts.push(...stream.close().map(partToDict)); + expect( + mergeAdjacentText(streamedParts), + `[${formatName}] turn ${turnIdx} streaming wire`, + ).toEqual(turn.expectWire); + + const singleShotParts = parser + .parseResponse(chunks.join(''), {state: singleShotState}) + .map(partToDict); + expect( + mergeAdjacentText(singleShotParts), + `[${formatName}] turn ${turnIdx} single-shot wire`, + ).toEqual(turn.expectWire); + } +} + +function runScenario(testCase: LoadedCase): void { + assertScenarioKeys(testCase); + const catalogs = loadScenarioCatalogs(testCase); + const protocolVersion = String(testCase.protocolVersion ?? '1.0'); + const formats = testCase.formats as string[]; + const failures: Error[] = []; + + for (const formatName of formats) { + try { + runFormat(testCase, formatName, catalogs, protocolVersion); + } catch (error) { + const msg = error instanceof Error ? (error.stack ?? error.message) : String(error); + failures.push(new Error(`[${formatName}] ${msg}`)); + } + } + + if (failures.length > 0) { + throw new Error(failures.map(f => f.message).join('\n\n')); + } +} + +const e2eFiles = findE2eYamlFiles(resolve(CONFORMANCE_ROOT, 'e2e')); +if (e2eFiles.length === 0) { + throw new Error('No E2E scenario YAML files found under conformance/e2e'); +} + +describeSuite( + 'Conformance: e2e scenarios', + loadCases(e2eFiles), + testCase => runScenario(testCase), + { + knownFailures: KNOWN_FAILURES, + }, +); From 7de5c170411ff08dd37e53d761683ba627c7b676 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 02:07:16 +0000 Subject: [PATCH 44/48] test(dart/a2ui_agent): run the e2e scenarios' agent half in Dart --- .../e2e_scenarios_conformance_test.dart | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 dart/a2ui_agent/test/conformance/e2e_scenarios_conformance_test.dart diff --git a/dart/a2ui_agent/test/conformance/e2e_scenarios_conformance_test.dart b/dart/a2ui_agent/test/conformance/e2e_scenarios_conformance_test.dart new file mode 100644 index 0000000000..3b191d6809 --- /dev/null +++ b/dart/a2ui_agent/test/conformance/e2e_scenarios_conformance_test.dart @@ -0,0 +1,239 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:a2ui_agent/a2ui_agent.dart'; +import 'package:a2ui_core/a2ui_core.dart'; +import 'package:test/test.dart'; + +import 'suites.dart'; + +const Map> _knownKeys = { + 'scenario': { + ...caseMetadataKeys, + 'protocolVersion', + 'catalogPaths', + 'formats', + 'source', + 'turns', + }, + 'source': {'kind', 'from', 'model', 'recordedAt'}, + 'turn': { + 'userText', + 'modelChunks', + 'expectWire', + 'expectViewTree', + 'expectErrors', + 'uiOps', + 'expectDataModel', + 'expectOutbound', + }, + 'uiOp': {'op', 'surfaceId', 'targetNode', 'inputValue', 'userActivation'}, +}; + +void main() { + final List suitePaths = _findE2eSuites(); + if (suitePaths.isEmpty) { + throw StateError('Expected E2E suites in conformance/e2e'); + } + + for (final String suitePath in suitePaths) { + group('conformance $suitePath', () { + final List> scenarios = loadSuite(suitePath); + test('suite is not empty', () => expect(scenarios, isNotEmpty)); + for (final Map scenario in scenarios) { + final name = scenario['name']! as String; + test(name, () => _runScenario(scenario)); + } + }); + } +} + +List _findE2eSuites() { + final e2eDir = Directory('$conformanceRoot/e2e'); + if (!e2eDir.existsSync()) return const []; + final List paths = [ + for (final FileSystemEntity entity in e2eDir.listSync(recursive: true)) + if (entity is File && entity.path.endsWith('.yaml')) + entity.path.substring(conformanceRoot.length + 1).replaceAll(r'\', '/'), + ]..sort(); + return paths; +} + +void _assertScenarioKeys(Map scenario) { + final name = scenario['name'] as String? ?? ''; + expect(scenario['action'], 'e2e_scenario'); + assertKnownKeys(scenario, _knownKeys['scenario']!, 'scenario($name)'); + if (scenario['source'] case final Map source) { + assertKnownKeys(source, _knownKeys['source']!, 'scenario($name).source'); + } + final turns = scenario['turns']! as List; + expect(turns, isNotEmpty); + for (final (int turnIdx, Object? rawTurn) in turns.indexed) { + final turn = rawTurn! as Map; + assertKnownKeys( + turn, + _knownKeys['turn']!, + 'scenario($name).turns[$turnIdx]', + ); + if (turn['uiOps'] case final List uiOps) { + for (final (int opIdx, Object? rawOp) in uiOps.indexed) { + assertKnownKeys( + rawOp! as Map, + _knownKeys['uiOp']!, + 'scenario($name).turns[$turnIdx].uiOps[$opIdx]', + ); + } + } + } +} + +A2uiProtocolVersion _parseProtocolVersion(String rawVersion) => + rawVersion.startsWith('v0.9') || rawVersion.startsWith('0.9') + ? A2uiProtocolVersion.v0_9 + : A2uiProtocolVersion.v1_0; + +List _loadScenarioCatalogs( + Map scenario, + A2uiProtocolVersion version, +) { + final String repoRoot = Directory(conformanceRoot).parent.path; + final catalogPaths = scenario['catalogPaths']! as List; + return [ + for (final Object? rawPath in catalogPaths) + () { + final path = rawPath! as String; + final inConformance = File('$conformanceRoot/$path'); + final File file = inConformance.existsSync() + ? inConformance + : File('$repoRoot/$path'); + final document = + jsonDecode(file.readAsStringSync()) as Map; + return InMemoryCatalogProvider( + document, + catalogId: document.containsKey('catalogId') ? null : path, + protocolVersion: document.containsKey('protocolVersion') + ? null + : version, + ).load(); + }(), + ]; +} + +Parser _buildParser( + String formatName, + List catalogs, + A2uiProtocolVersion version, +) => switch (formatName) { + 'direct_json' => DirectJsonFormatFactory( + bufferIncompleteComponents: version == A2uiProtocolVersion.v1_0, + ).createFormat(catalogs).createParser(), + 'express' => ExpressFormatFactory().createFormat(catalogs).createParser(), + _ => throw UnsupportedError('Unknown inference format: $formatName'), +}; + +List _mergeAdjacentText(List parts) { + final merged = []; + for (final Object? part in parts) { + if (merged.isNotEmpty && + merged.last is Map && + part is Map && + (merged.last as Map).containsKey('text') && + part.containsKey('text')) { + final prev = Map.from( + merged.last as Map, + ); + prev['text'] = '${prev['text']}${part['text']}'; + merged[merged.length - 1] = prev; + } else { + merged.add(part); + } + } + return merged; +} + +void _runFormat( + Map scenario, + String formatName, + List catalogs, + A2uiProtocolVersion version, +) { + final Parser parser = _buildParser(formatName, catalogs, version); + final streamState = RendererState(catalogs); + final singleShotState = RendererState(catalogs); + try { + final turns = scenario['turns']! as List; + for (final (int turnIdx, Object? rawTurn) in turns.indexed) { + final turn = rawTurn! as Map; + final modelChunks = turn['modelChunks']! as Map; + final rawChunks = modelChunks[formatName] as List?; + expect( + rawChunks, + isNotNull, + reason: 'turn $turnIdx missing chunks for $formatName', + ); + final List chunks = rawChunks!.cast(); + final expectWire = turn['expectWire']! as List; + + final ResponseStream stream = parser.openStream(state: streamState); + final streamed = []; + for (final String chunk in chunks) { + streamed.addAll(stream.feed(chunk)); + } + streamed.addAll(stream.close()); + expect( + _mergeAdjacentText(partsAsWritten(streamed)), + expectWire, + reason: '[$formatName] turn $turnIdx streaming wire', + ); + + final List singleShot = parser.parseResponse( + chunks.join(), + state: singleShotState, + ); + expect( + _mergeAdjacentText(partsAsWritten(singleShot)), + expectWire, + reason: '[$formatName] turn $turnIdx single-shot wire', + ); + } + } finally { + streamState.dispose(); + singleShotState.dispose(); + } +} + +void _runScenario(Map scenario) { + _assertScenarioKeys(scenario); + final A2uiProtocolVersion version = _parseProtocolVersion( + (scenario['protocolVersion'] as String?) ?? '1.0', + ); + final List catalogs = _loadScenarioCatalogs(scenario, version); + final formats = (scenario['formats']! as List).cast(); + final failures = []; + + for (final String formatName in formats) { + try { + _runFormat(scenario, formatName, catalogs, version); + } on Object catch (error, stackTrace) { + failures.add('[$formatName] $error\n$stackTrace'); + } + } + + if (failures.isNotEmpty) { + fail(failures.join('\n\n')); + } +} From 7fed215e630fe04b84efa416d8a0de36e3659325 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 02:07:18 +0000 Subject: [PATCH 45/48] test(a2ui_flutter): run the e2e scenarios' renderer half --- .../a2ui_flutter/test/e2e_scenarios_test.dart | 398 ++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 dart/a2ui_flutter/test/e2e_scenarios_test.dart diff --git a/dart/a2ui_flutter/test/e2e_scenarios_test.dart b/dart/a2ui_flutter/test/e2e_scenarios_test.dart new file mode 100644 index 0000000000..414216d043 --- /dev/null +++ b/dart/a2ui_flutter/test/e2e_scenarios_test.dart @@ -0,0 +1,398 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:io'; + +import 'package:a2ui_core/a2ui_core.dart'; +import 'package:a2ui_flutter/a2ui_flutter.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'adapter_harness.dart'; + +const Set _e2eScenarioKnownKeys = { + 'name', + 'description', + 'features', + 'action', + 'protocolVersion', + 'catalogPaths', + 'formats', + 'source', + 'turns', +}; + +const Set _e2eSourceKnownKeys = {'kind', 'from', 'model', 'recordedAt'}; + +const Set _e2eTurnKnownKeys = { + 'userText', + 'modelChunks', + 'expectWire', + 'expectViewTree', + 'expectErrors', + 'uiOps', + 'expectDataModel', + 'expectOutbound', +}; + +const Set _e2eUiOpKnownKeys = { + 'op', + 'surfaceId', + 'targetNode', + 'inputValue', + 'userActivation', +}; + +const Set _e2eViewNodeKnownKeys = { + ...viewNodeKnownKeys, + 'label', + 'disabled', +}; + +void main() { + final List<(String, String)> suiteFiles = _findE2eSuiteFiles(); + if (suiteFiles.isEmpty) { + throw StateError( + 'Expected E2E scenario YAML files under ../../conformance/e2e', + ); + } + + tearDownAll(assertAllUnsupportedMatched); + + for (final (String suiteRelPath, String filePath) in suiteFiles) { + final List scenarios = loadYamlTests(filePath); + group('E2E Scenarios ($suiteRelPath)', () { + for (final Object? entry in scenarios) { + final scenario = entry! as Map; + final name = scenario['name']! as String; + testWidgets(name, (tester) async { + var status = 'failed'; + String? reason; + addTearDown(() { + writeManifestRecord({ + 'harness': 'flutter', + 'suite': suiteRelPath, + 'case': name, + 'status': status, + 'reason': ?reason, + }); + }); + try { + _assertKnownE2eScenario(scenario); + final String? skipReason = findUnsupportedReason(name); + if (skipReason != null) { + status = 'skipped'; + reason = skipReason; + markTestSkipped(skipReason); + return; + } + await _runScenario(tester, scenario); + status = 'passed'; + } catch (error) { + status = 'failed'; + reason = error.toString(); + rethrow; + } + }); + } + }); + } +} + +List<(String, String)> _findE2eSuiteFiles() { + final e2eDir = Directory('../../conformance/e2e'); + if (!e2eDir.existsSync()) return const []; + final List<(String, String)> entries = []; + for (final FileSystemEntity entity in e2eDir.listSync(recursive: true)) { + if (entity is File && entity.path.endsWith('.yaml')) { + final String normalized = entity.path.replaceAll(r'\', '/'); + final int idx = normalized.indexOf('conformance/'); + final String suiteRel = normalized.substring(idx + 'conformance/'.length); + entries.add((suiteRel, entity.path)); + } + } + entries.sort((a, b) => a.$1.compareTo(b.$1)); + return entries; +} + +void _assertKnownE2eViewNode(Map node, String where) { + assertKnownKeys(node, _e2eViewNodeKnownKeys, where); + if (node['validationResults'] case final List results) { + for (var i = 0; i < results.length; i++) { + assertKnownKeys( + results[i]! as Map, + validationResultKnownKeys, + '$where.validationResults[$i]', + ); + } + } + if (node['children'] case final List children) { + for (var i = 0; i < children.length; i++) { + _assertKnownE2eViewNode( + children[i]! as Map, + '$where.children[$i]', + ); + } + } +} + +void _assertKnownE2eScenario(Map scenario) { + final name = scenario['name']! as String; + assertKnownKeys(scenario, _e2eScenarioKnownKeys, 'scenario($name)'); + if (scenario['action'] != 'e2e_scenario') { + fail('Unsupported action in scenario($name): ${scenario['action']}'); + } + if (scenario['source'] case final Map source) { + assertKnownKeys(source, _e2eSourceKnownKeys, 'scenario($name).source'); + } + final turns = scenario['turns']! as List; + expect(turns, isNotEmpty); + for (var i = 0; i < turns.length; i++) { + final turn = turns[i]! as Map; + assertKnownKeys(turn, _e2eTurnKnownKeys, 'scenario($name).turns[$i]'); + if (turn['expectViewTree'] case final Map viewTrees) { + for (final MapEntry entry in viewTrees.entries) { + if (entry.value case final Map tree) { + _assertKnownE2eViewNode( + tree, + 'scenario($name).turns[$i].expectViewTree.${entry.key}', + ); + } + } + } + if (turn['uiOps'] case final List uiOps) { + for (var j = 0; j < uiOps.length; j++) { + final uiOp = uiOps[j]! as Map; + assertKnownKeys( + uiOp, + _e2eUiOpKnownKeys, + 'scenario($name).turns[$i].uiOps[$j]', + ); + } + } + } +} + +Map _stripExtendedViewNodeKeys( + Map expected, +) => { + for (final MapEntry entry in expected.entries) + if (entry.key == 'children' && entry.value is List) + 'children': [ + for (final Object? child in entry.value! as List) + _stripExtendedViewNodeKeys(child! as Map), + ] + else if (viewNodeKnownKeys.contains(entry.key)) + entry.key: entry.value, +}; + +List> _directChildrenOf( + ComponentNode node, +) { + final props = ComponentProps(node.props.peek()); + if (props.child('child') + case final ComponentNode child) { + return [child]; + } + return props.children('children'); +} + +void _assertExtendedViewNodeProps( + ComponentNode node, + Map expected, +) { + final props = ComponentProps(node.props.peek()); + for (final key in const ['label', 'disabled']) { + if (expected[key] case final Object expectedVal) { + expect(props.value(key), expectedVal); + } + } + if (expected['children'] case final List expectedChildren) { + final List> children = + _directChildrenOf(node); + for (var i = 0; i < expectedChildren.length; i++) { + _assertExtendedViewNodeProps( + children[i], + expectedChildren[i]! as Map, + ); + } + } +} + +List> captureOutbound(AdapterHarnessSession session) => + List>.from(session.outbound); + +bool _isMatchAny(Object? value) => + value is Map && value.length == 1 && value[r'$match'] == 'any'; + +void _assertMatchesOutbound(Object? actual, Object? expected, String where) { + if (_isMatchAny(expected)) { + expect( + actual, + isNotNull, + reason: '$where: expected non-null value for {r"\$match": "any"}', + ); + return; + } + if (expected is List) { + expect(actual, isA>(), reason: '$where: expected list'); + final actualList = actual! as List; + expect(actualList, hasLength(expected.length)); + for (var i = 0; i < expected.length; i++) { + _assertMatchesOutbound(actualList[i], expected[i], '$where[$i]'); + } + return; + } + if (expected is Map) { + expect( + actual, + isA>(), + reason: '$where: expected object', + ); + final actualMap = actual! as Map; + final List actualKeys = actualMap.keys.toList()..sort(); + final List expectedKeys = + expected.keys.map((k) => k.toString()).toList()..sort(); + expect(actualKeys, equals(expectedKeys), reason: '$where: keys mismatch'); + for (final key in expectedKeys) { + _assertMatchesOutbound(actualMap[key], expected[key], '$where.$key'); + } + return; + } + expect(actual, equals(expected), reason: '$where: value mismatch'); +} + +void assertOutbound( + List> actual, + List expected, + String protocolVersion, +) { + expect(protocolVersion, isNotEmpty); + expect( + actual, + hasLength(expected.length), + reason: 'expectOutbound length mismatch', + ); + for (var i = 0; i < expected.length; i++) { + _assertMatchesOutbound(actual[i], expected[i], 'expectOutbound[$i]'); + } +} + +Future _selectSurface( + WidgetTester tester, + AdapterHarnessSession session, + String surfaceId, +) async { + final SurfaceModel? target = session.processor.model + .getSurface(surfaceId); + expect(target, isNotNull, reason: 'Expected surface "$surfaceId" to exist'); + if (!identical(session.surface, target) || !session.mounted) { + session.surface = target; + session.mounted = true; + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SingleChildScrollView(child: A2uiSurface(surface: target!)), + ), + ), + ); + } +} + +Future _runScenario( + WidgetTester tester, + Map scenario, +) async { + final SemanticsHandle semantics = tester.ensureSemantics(); + final AdapterHarnessSession session = mountSurface(scenario, tester); + try { + final turns = scenario['turns']! as List; + for (final Object? rawTurn in turns) { + final turn = rawTurn! as Map; + final turnOutbound = >[]; + final turnErrors = []; + + final expectWire = turn['expectWire']! as List; + for (final Object? rawPart in expectWire) { + final part = rawPart! as Map; + if (part['a2ui'] case final List messages) { + await runStep(session, { + 'op': 'process_messages', + 'messages': messages, + }, tester); + turnOutbound.addAll(captureOutbound(session)); + turnErrors.addAll(session.errors); + } + } + + if (turn['expectViewTree'] case final Map viewTrees) { + for (final MapEntry entry in viewTrees.entries) { + final String surfaceId = entry.key; + final Object? expectedTree = entry.value; + if (expectedTree == null) { + expect(session.processor.model.getSurface(surfaceId), isNull); + } else { + await _selectSurface(tester, session, surfaceId); + final expectedMap = expectedTree as Map; + final ComponentNode? root = + extractViewTree(session); + assertViewTree( + root, + _stripExtendedViewNodeKeys(expectedMap), + tester, + ); + _assertExtendedViewNodeProps(root!, expectedMap); + } + } + } + + if (turn['uiOps'] case final List uiOps) { + for (final Object? rawOp in uiOps) { + final uiOp = rawOp! as Map; + final surfaceId = uiOp['surfaceId']! as String; + await _selectSurface(tester, session, surfaceId); + final step = { + for (final MapEntry e in uiOp.entries) + if (e.key != 'surfaceId') e.key: e.value, + }; + await runStep(session, step, tester); + turnOutbound.addAll(captureOutbound(session)); + turnErrors.addAll(session.errors); + } + } + + if (turn['expectDataModel'] case final Map dataModels) { + for (final MapEntry entry in dataModels.entries) { + final SurfaceModel? target = session + .processor + .model + .getSurface(entry.key); + expect(target, isNotNull); + expect(target!.dataModel.get('/'), equals(entry.value)); + } + } + + if (turn['expectErrors'] case final List expectedErrors) { + expect(turnErrors, hasLength(expectedErrors.length)); + } + + if (turn['expectOutbound'] case final List expectedOutbound) { + assertOutbound(turnOutbound, expectedOutbound, session.protocolVersion); + } + } + } finally { + session.dispose(); + semantics.dispose(); + } +} From 9f3eb618116ff23f7ab8f881590785f80e0bafa5 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 02:07:20 +0000 Subject: [PATCH 46/48] test(swiftui): replay e2e scenarios in hosted views --- .../A2UISwiftUITests/E2EScenarioTests.swift | 582 ++++++++++++++++++ 1 file changed, 582 insertions(+) create mode 100644 swift/swiftui/Tests/A2UISwiftUITests/E2EScenarioTests.swift diff --git a/swift/swiftui/Tests/A2UISwiftUITests/E2EScenarioTests.swift b/swift/swiftui/Tests/A2UISwiftUITests/E2EScenarioTests.swift new file mode 100644 index 0000000000..390c188419 --- /dev/null +++ b/swift/swiftui/Tests/A2UISwiftUITests/E2EScenarioTests.swift @@ -0,0 +1,582 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import A2UICore +import A2UISwiftUI +import BasicCatalog +import BasicCatalogSwiftUI +import Foundation +import OrderedJSON +import Testing +import Yams + +/// Parameterized E2E scenario test case loaded from `conformance/e2e/**/*.yaml`. +struct E2EScenarioCase: CustomTestStringConvertible, Sendable { + let suite: String + let name: String + let specJSON: JSONValue + + var testDescription: String { name } + + func makeSpec() -> [String: Any] { + (AdapterHarness.fromJSONValue(specJSON) as? [String: Any]) ?? [:] + } +} + +/// Replays end-to-end golden-transcript scenarios under `conformance/e2e/**/*.yaml` +/// through `MessageProcessor`, `A2UISwiftUI.Surface`, `HostedViewDriver`, and `AdapterHarness`. +@Suite(.serialized) +@MainActor +struct E2EScenarioTests { + static let scenarios: [E2EScenarioCase] = loadScenarios() + + /// Reasoned expected failures keyed by scenario `name`. + private static let expectedScenarioFailures: [String: String] = [:] + + private static let e2eScenarioKnownKeys: Set = [ + "name", + "description", + "features", + "action", + "protocolVersion", + "catalogPaths", + "formats", + "source", + "turns", + ] + + private static let e2eSourceKnownKeys: Set = [ + "kind", + "from", + "model", + "recordedAt", + ] + + private static let e2eTurnKnownKeys: Set = [ + "userText", + "modelChunks", + "expectWire", + "expectViewTree", + "expectErrors", + "uiOps", + "expectDataModel", + "expectOutbound", + ] + + private static let e2eUiOpKnownKeys: Set = [ + "op", + "surfaceId", + "targetNode", + "inputValue", + "userActivation", + ] + + private static let e2eViewNodeKnownKeys: Set = AdapterHarness.viewNodeKnownKeys.union([ + "label", + "disabled", + ]) + + nonisolated private static var conformanceDirectory: URL { + if let envDir = ProcessInfo.processInfo.environment["A2UI_CONFORMANCE_DIR"], !envDir.isEmpty { + let url = URL(fileURLWithPath: envDir) + return url.lastPathComponent == "conformance" + ? url : url.appendingPathComponent("conformance") + } + let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + return repoRoot.appendingPathComponent("conformance") + } + + nonisolated private static func loadScenarios() -> [E2EScenarioCase] { + let e2eURL = conformanceDirectory.appendingPathComponent("e2e") + guard + let enumerator = FileManager.default.enumerator( + at: e2eURL, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles] + ) + else { + return [] + } + var yamlURLs: [URL] = [] + for case let fileURL as URL in enumerator where fileURL.pathExtension == "yaml" { + yamlURLs.append(fileURL) + } + yamlURLs.sort { $0.path < $1.path } + + let basePath = conformanceDirectory.standardizedFileURL.path + var loaded: [E2EScenarioCase] = [] + for fileURL in yamlURLs { + let fullPath = fileURL.standardizedFileURL.path + let relSuite = String(fullPath.dropFirst(basePath.count + 1)) + .replacingOccurrences(of: "\\", with: "/") + guard + let text = try? String(contentsOf: fileURL, encoding: .utf8), + let rawCases = try? Yams.load(yaml: text) as? [[String: Any]] + else { + continue + } + for rawCase in rawCases { + let name = rawCase["name"] as? String ?? "" + loaded.append( + E2EScenarioCase( + suite: relSuite, + name: name, + specJSON: AdapterHarness.toJSONValue(rawCase) + ) + ) + } + } + return loaded + } + + @Test(arguments: E2EScenarioTests.scenarios) + func e2eScenario(_ c: E2EScenarioCase) throws { + let suite = c.suite + let name = c.name + let scenario = c.makeSpec() + let knownScenarioNames = Set(Self.scenarios.map(\.name)) + let unusedExpectedFailures = Self.expectedScenarioFailures.keys + .filter { !knownScenarioNames.contains($0) } + .sorted() + #expect( + unusedExpectedFailures.isEmpty, + "Unused expectedScenarioFailures entries: \(unusedExpectedFailures)" + ) + + do { + try Self.assertKnownScenario(scenario) + } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) + writeManifestRecord( + AdapterManifestRecord( + harness: "swiftui", + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error + } + + if let skipReason = AdapterHarness.findUnsupportedReason(name) { + writeManifestRecord( + AdapterManifestRecord( + harness: "swiftui", + suite: suite, + caseName: name, + status: "skipped", + reason: skipReason + ) + ) + return + } + + if let reason = Self.expectedScenarioFailures[name] { + withKnownIssue("\(name): \(reason)") { + try runScenario(scenario, name: name) + } + writeManifestRecord( + AdapterManifestRecord( + harness: "swiftui", + suite: suite, + caseName: name, + status: "xfailed", + reason: reason + ) + ) + return + } + + do { + try runScenario(scenario, name: name) + writeManifestRecord( + AdapterManifestRecord( + harness: "swiftui", + suite: suite, + caseName: name, + status: "passed" + ) + ) + } catch { + Issue.record(Comment(rawValue: "\(name): \(error)")) + writeManifestRecord( + AdapterManifestRecord( + harness: "swiftui", + suite: suite, + caseName: name, + status: "failed", + reason: String(describing: error) + ) + ) + throw error + } + } + + // MARK: - Scenario Validation & Execution + + private static func assertKnownE2EViewNode( + _ node: [String: Any], + where location: String + ) throws { + try AdapterHarness.assertKnownKeys( + node, + allowed: e2eViewNodeKnownKeys, + where: location + ) + if let results = node["validationResults"] as? [[String: Any]] { + for (idx, item) in results.enumerated() { + try AdapterHarness.assertKnownKeys( + item, + allowed: AdapterHarness.validationResultKnownKeys, + where: "\(location).validationResults[\(idx)]" + ) + } + } + if let children = node["children"] as? [[String: Any]] { + for (idx, child) in children.enumerated() { + try assertKnownE2EViewNode(child, where: "\(location).children[\(idx)]") + } + } + } + + private static func assertKnownScenario(_ scenario: [String: Any]) throws { + let name = scenario["name"] as? String ?? "" + try AdapterHarness.assertKnownKeys( + scenario, + allowed: e2eScenarioKnownKeys, + where: "scenario(\(name))" + ) + guard (scenario["action"] as? String) == "e2e_scenario" else { + let msg = "scenario(\(name)): unsupported action \(String(describing: scenario["action"]))" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + if let source = scenario["source"] as? [String: Any] { + try AdapterHarness.assertKnownKeys( + source, + allowed: e2eSourceKnownKeys, + where: "scenario(\(name)).source" + ) + } + guard let turns = scenario["turns"] as? [[String: Any]], !turns.isEmpty else { + let msg = "scenario(\(name)): turns must be a non-empty array" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + for (turnIdx, turn) in turns.enumerated() { + try AdapterHarness.assertKnownKeys( + turn, + allowed: e2eTurnKnownKeys, + where: "scenario(\(name)).turns[\(turnIdx)]" + ) + if let viewTrees = turn["expectViewTree"] as? [String: Any] { + for (surfaceID, rawTree) in viewTrees { + if let tree = rawTree as? [String: Any] { + try assertKnownE2EViewNode( + tree, + where: "scenario(\(name)).turns[\(turnIdx)].expectViewTree.\(surfaceID)" + ) + } + } + } + if let uiOps = turn["uiOps"] as? [[String: Any]] { + for (opIdx, uiOp) in uiOps.enumerated() { + try AdapterHarness.assertKnownKeys( + uiOp, + allowed: e2eUiOpKnownKeys, + where: "scenario(\(name)).turns[\(turnIdx)].uiOps[\(opIdx)]" + ) + } + } + } + } + + private static func stripExtendedViewNodeKeys(_ expected: [String: Any]) -> [String: Any] { + var result: [String: Any] = [:] + for (key, value) in expected { + if key == "children", let children = value as? [[String: Any]] { + result["children"] = children.map { stripExtendedViewNodeKeys($0) } + } else if AdapterHarness.viewNodeKnownKeys.contains(key) { + result[key] = value + } + } + return result + } + + private static func assertExtendedViewNodeProps( + _ actual: AdapterViewNode?, + expected: [String: Any], + context: String + ) throws { + guard let viewNode = actual, let node = viewNode.node else { return } + if let expLabel = expected["label"] as? String { + let actualLabel = node.string(for: "label") + #expect(actualLabel == expLabel, "\(context): label mismatch") + if actualLabel != expLabel { + throw AdapterConformanceError("\(context): label mismatch") + } + } + if let expDisabled = expected["disabled"] as? Bool { + let actualDisabled = node.bool(for: "disabled") ?? false + #expect(actualDisabled == expDisabled, "\(context): disabled mismatch") + if actualDisabled != expDisabled { + throw AdapterConformanceError("\(context): disabled mismatch") + } + } + if let expChildren = expected["children"] as? [[String: Any]] { + for (idx, expChild) in expChildren.enumerated() where idx < viewNode.children.count { + try assertExtendedViewNodeProps( + viewNode.children[idx], + expected: expChild, + context: "\(context)/children/\(idx)" + ) + } + } + } + + private static func captureOutbound(_ session: AdapterSession) -> [[String: Any]] { + session.handler.outbound + } + + private static func isMatchAny(_ value: Any) -> Bool { + guard let dict = value as? [String: Any], dict.count == 1 else { return false } + return (dict["$match"] as? String) == "any" + } + + private static func assertMatchesOutbound( + _ actual: Any, + expected: Any, + where location: String + ) throws { + if isMatchAny(expected) { + let isNonNull = !(actual is NSNull) + #expect(isNonNull, "\(location): expected non-null value for {$match: \"any\"}") + if !isNonNull { + throw AdapterConformanceError("\(location): expected non-null value") + } + return + } + if let expArray = expected as? [Any] { + guard let actArray = actual as? [Any], actArray.count == expArray.count else { + let msg = "\(location): array length mismatch" + Issue.record(Comment(rawValue: msg)) + throw AdapterConformanceError(msg) + } + for idx in 0.. = [] + if let active = session.mountedHarness { + tornDown.insert(ObjectIdentifier(active)) + active.driver.tearDown() + } + for harness in mountedBySurfaceID.values { + if tornDown.insert(ObjectIdentifier(harness)).inserted { + harness.driver.tearDown() + } + } + } + + let turns = (scenario["turns"] as? [[String: Any]]) ?? [] + for (turnIdx, turn) in turns.enumerated() { + let turnCtx = "\(name) [turn \(turnIdx)]" + var turnOutbound: [[String: Any]] = [] + + let expectWire = (turn["expectWire"] as? [[String: Any]]) ?? [] + for (partIdx, part) in expectWire.enumerated() { + guard let messages = part["a2ui"] as? [[String: Any]] else { continue } + try AdapterHarness.runStep( + session, + ["op": "process_messages", "messages": messages], + context: "\(turnCtx) [wire \(partIdx)]" + ) + if let mounted = session.mountedHarness { + for (sid, vm) in session.processor.surfaceGroupModel.surfacesMap where + mounted.surface.viewModel === vm + { + mountedBySurfaceID[sid] = mounted + } + } + turnOutbound.append(contentsOf: Self.captureOutbound(session)) + } + + if let viewTrees = turn["expectViewTree"] as? [String: Any] { + for (surfaceID, rawTree) in viewTrees { + if rawTree is NSNull { + let exists = session.processor.surfaceGroupModel.surfacesMap[surfaceID] != nil + #expect(!exists, "\(turnCtx): expected surface '\(surfaceID)' to be absent") + if exists { + throw AdapterConformanceError( + "\(turnCtx): expected surface '\(surfaceID)' to be absent" + ) + } + } else if let expectedTree = rawTree as? [String: Any] { + try selectSurface( + surfaceID, + session: session, + mountedBySurfaceID: &mountedBySurfaceID, + context: turnCtx + ) + let actualTree = extractViewTree(session) + try AdapterHarness.assertViewTree( + actualTree, + Self.stripExtendedViewNodeKeys(expectedTree), + context: "\(turnCtx) [\(surfaceID)]" + ) + try Self.assertExtendedViewNodeProps( + actualTree, + expected: expectedTree, + context: "\(turnCtx) [\(surfaceID)]" + ) + } + } + } + + if let uiOps = turn["uiOps"] as? [[String: Any]] { + for (opIdx, uiOp) in uiOps.enumerated() { + guard let surfaceID = uiOp["surfaceId"] as? String else { + throw AdapterConformanceError("\(turnCtx) [uiOp \(opIdx)]: missing surfaceId") + } + try selectSurface( + surfaceID, + session: session, + mountedBySurfaceID: &mountedBySurfaceID, + context: "\(turnCtx) [uiOp \(opIdx)]" + ) + var step = uiOp + step.removeValue(forKey: "surfaceId") + try AdapterHarness.runStep( + session, + step, + context: "\(turnCtx) [uiOp \(opIdx)]" + ) + turnOutbound.append(contentsOf: Self.captureOutbound(session)) + } + } + + if let expDataModels = turn["expectDataModel"] as? [String: Any] { + for (surfaceID, expData) in expDataModels { + guard let vm = session.processor.surfaceGroupModel.surfacesMap[surfaceID] else { + throw AdapterConformanceError("\(turnCtx): missing surface '\(surfaceID)'") + } + let expectedJSON = AdapterHarness.toJSONValue(expData) + #expect(vm.dataModel.data == expectedJSON, "\(turnCtx) [\(surfaceID)]: dataModel") + if vm.dataModel.data != expectedJSON { + throw AdapterConformanceError("\(turnCtx) [\(surfaceID)]: dataModel mismatch") + } + } + } + + if let expOutbound = turn["expectOutbound"] as? [[String: Any]] { + try Self.assertOutbound(turnOutbound, expected: expOutbound, context: turnCtx) + } + } + } +} From 2e41ee5f142a6386b8a2c4a404ccec910abecf9d Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 02:10:20 +0000 Subject: [PATCH 47/48] style(conformance): apply license headers to E2E outcome scripts --- conformance/scripts/compare_e2e_outcome.py | 2 +- conformance/scripts/e2e_outcome.py | 2 +- conformance/tests/test_e2e_outcome.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conformance/scripts/compare_e2e_outcome.py b/conformance/scripts/compare_e2e_outcome.py index b1701e7688..2e8d62c88b 100644 --- a/conformance/scripts/compare_e2e_outcome.py +++ b/conformance/scripts/compare_e2e_outcome.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/conformance/scripts/e2e_outcome.py b/conformance/scripts/e2e_outcome.py index c64cbb211c..0503e7e260 100644 --- a/conformance/scripts/e2e_outcome.py +++ b/conformance/scripts/e2e_outcome.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/conformance/tests/test_e2e_outcome.py b/conformance/tests/test_e2e_outcome.py index 7ec375ca4c..b0b52da76b 100644 --- a/conformance/tests/test_e2e_outcome.py +++ b/conformance/tests/test_e2e_outcome.py @@ -1,4 +1,4 @@ -# Copyright 2026 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 49484138f61e436136c9e9f437ec2e7d386dbd49 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 3 Nov 2026 03:56:58 +0000 Subject: [PATCH 48/48] fix(conformance): resolve mypy type annotations in compare_e2e_outcome --- conformance/scripts/compare_e2e_outcome.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conformance/scripts/compare_e2e_outcome.py b/conformance/scripts/compare_e2e_outcome.py index 2e8d62c88b..8372afb38b 100644 --- a/conformance/scripts/compare_e2e_outcome.py +++ b/conformance/scripts/compare_e2e_outcome.py @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,9 +27,9 @@ from conformance.scripts.e2e_outcome import outcome_of except ImportError: try: - from scripts.e2e_outcome import outcome_of + from scripts.e2e_outcome import outcome_of # type: ignore[import-not-found,no-redef] except ImportError: - from e2e_outcome import outcome_of + from e2e_outcome import outcome_of # type: ignore[import-not-found,no-redef] def _load_scenario_file(path_str: str) -> list[dict[str, Any]]: