From f871fa95b9b0a10000e8616cf3b13633a1c79a63 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 22 Sep 2026 10:59:32 -0400 Subject: [PATCH 1/3] feat(v2): validate identify and alias delivery with typed JSON assertions --- docs/harness-v2.md | 21 ++ src/posthog_test_harness/v2/ai_steps.py | 10 + .../v2/analytics_wire_steps.py | 9 +- tests/test_v2_analytics_batching.py | 2 +- tests/test_v2_bundle.py | 6 +- tests/test_v2_identify_alias.py | 211 ++++++++++++++++++ tests/test_v2_legacy_capture.py | 8 +- 7 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 tests/test_v2_identify_alias.py diff --git a/docs/harness-v2.md b/docs/harness-v2.md index 79aa256..f76ee92 100644 --- a/docs/harness-v2.md +++ b/docs/harness-v2.md @@ -41,6 +41,27 @@ Completion is exactly one of: The migrated bindings use `/setup`, `/capture`, `/capture_ai`, `/flush`, `/get_feature_flag` and `/reload_feature_flags`. Their argument objects appear directly in feature doc strings or named step bindings. Additional shared public operations can be added with concrete scenarios; object references, callback continuations and private fixture-control endpoints are not part of this draft. +## Black-box server identify and alias + +Explicitly select `--feature black-box/public/identify.feature` and +`--feature black-box/public/alias.feature` from a companion specs checkout to run +these four server-only cases. They are separate from the 157-case migration suite. +The steps `identify is called with JSON arguments:` and +`alias is called with JSON arguments:` forward JSON doc strings unchanged to the +negotiated `/identify` and `/alias` routes: + +- `/identify`: `{ "distinct_id": "user-123", "set": { "active": false, "score": 0, "note": null } }` +- `/alias`: `{ "distinct_id": "anon-123", "alias": "user-123" }` + +Adapters translate these arguments to their public SDK methods. Unexpected operation +throws fail the scenario. After public setup, operation and explicit flush, received +events must have the exact event name (`$identify` or `$create_alias`), root +`distinct_id`, and `$set` or `alias` property. For alias, root `distinct_id` is the +previous identity and `properties.alias` is the target. JSON property assertions +preserve boolean, numeric and null types and require the property to be present. +These delivery cases require no private queue observations or identity persistence +controls and do not establish broader client-side identity behavior. + ## Black-box local evaluation A fresh receiver loads controlled definitions through the mock definitions service using public SDK configuration. Scenarios call the local-only getter directly after initialization and compare conclusive results with exact expectations, recording authenticated HTTP 200 definitions fetched during setup or initial evaluation. When definitions change, scenarios explicitly reload through the public SDK method, require fresh authenticated HTTP 200 definitions within five seconds, and evaluate again. Different properties and changed definitions detect constants, defaults and stale reloads. Definitions downloads are allowed; `/flags` and `/decide` requests are forbidden throughout setup, loading, evaluation and public cleanup. No private evaluator observation is required. diff --git a/src/posthog_test_harness/v2/ai_steps.py b/src/posthog_test_harness/v2/ai_steps.py index 67fb76a..3b822bb 100644 --- a/src/posthog_test_harness/v2/ai_steps.py +++ b/src/posthog_test_harness/v2/ai_steps.py @@ -49,6 +49,16 @@ async def capture(ctx, step): await ctx.call("/capture", json_arguments(step)) +@STEPS.step("identify is called with JSON arguments:", "docString", routes=("/identify",)) +async def identify(ctx, step): + await ctx.call("/identify", json_arguments(step)) + + +@STEPS.step("alias is called with JSON arguments:", "docString", routes=("/alias",)) +async def alias(ctx, step): + await ctx.call("/alias", json_arguments(step)) + + @STEPS.step("pending captures are flushed", routes=("/flush",)) async def flush(ctx, step): # YAML flush actions do not assert delivery success or a native return value. diff --git a/src/posthog_test_harness/v2/analytics_wire_steps.py b/src/posthog_test_harness/v2/analytics_wire_steps.py index 7f1f9b9..76f3e05 100644 --- a/src/posthog_test_harness/v2/analytics_wire_steps.py +++ b/src/posthog_test_harness/v2/analytics_wire_steps.py @@ -12,7 +12,7 @@ from .ai_steps import STEPS as PREVIOUS_STEPS from .ai_steps import first_events, json_arguments, requests, utc_instant -from .contracts import decode_json +from .contracts import decode_json, json_equal from .steps import Registry, expect, table STEPS = Registry() @@ -150,8 +150,11 @@ async def capture_sequence(ctx, step, count): async def property_json(ctx, step, name, encoded): expected = decode_json(encoded) properties = first_events(ctx)[0].get("properties", {}) - # The pinned property helper uses value equality, including Python's bool/number equality. - expect(properties.get(name) == expected, "event_property", f"Received property differs: {name}") + expect( + isinstance(properties, dict) and name in properties and json_equal(properties[name], expected), + "event_property", + f"Received property differs: {name}", + ) @STEPS.step(r'the first received event property "([^"]*)" should be an object') diff --git a/tests/test_v2_analytics_batching.py b/tests/test_v2_analytics_batching.py index fffd5be..f37df6a 100644 --- a/tests/test_v2_analytics_batching.py +++ b/tests/test_v2_analytics_batching.py @@ -28,6 +28,7 @@ def contracts(): ('property_value:0:custom_string:"wrong"', 0, "event_property"), ('property_value:0:custom_number:"42"', 0, "event_property"), ("property_value:0:custom_bool:false", 0, "event_property"), + ("property_value:0:custom_bool:1", 0, "event_property"), ("missing_property:custom_bool", 0, "event_property"), ("property_value:0:$set:[]", 1, "event_property_object"), ("property_value:0:$set_once:null", 2, "event_property_object"), @@ -77,7 +78,6 @@ async def test_assertion_families_reject_attributed_http_defects(contracts, defe @pytest.mark.parametrize( "variation,index", [ - ("property_value:0:custom_bool:1", 0), ("property_value:0:custom_number:42.0", 0), ("property_value:0:$set:{}", 1), ('property_value:0:$set_once:{"other":false}', 2), diff --git a/tests/test_v2_bundle.py b/tests/test_v2_bundle.py index 5eb830e..04b1106 100644 --- a/tests/test_v2_bundle.py +++ b/tests/test_v2_bundle.py @@ -26,11 +26,13 @@ def test_bundle_preserves_both_case_inventories_and_source_bytes(snapshot, specs manifest = validate_bundle(snapshot) assert type(manifest["source"]["dirty"]) is bool assert len(manifest["source"]["commit"]) == 40 - for paths, count in [(None, 885), (migration_paths(snapshot), 157)]: + for paths in [None, migration_paths(snapshot)]: packaged = discover(snapshot, paths) source = discover(specs, paths) assert packaged == source - assert len(packaged["cases"]) == count + assert packaged["cases"] + if paths is not None: + assert len(packaged["cases"]) == 157 assert all(name.endswith(".feature") for name in manifest["files"]) diff --git a/tests/test_v2_identify_alias.py b/tests/test_v2_identify_alias.py new file mode 100644 index 0000000..e8d7183 --- /dev/null +++ b/tests/test_v2_identify_alias.py @@ -0,0 +1,211 @@ +"""Public identify/alias bindings over the existing controlled HTTP host.""" + +import json +from types import SimpleNamespace + +import pytest + +from posthog_test_harness.v2.contracts import BoundaryError, Contracts, json_equal +from posthog_test_harness.v2.gherkin import load_cases +from posthog_test_harness.v2.report import strict_exit_code +from posthog_test_harness.v2.runner import STEPS, run +from tests.v2_analytics_wire_host import AnalyticsWireHost +from tests.v2_flush_host import serve + +FEATURES = ["black-box/public/identify.feature", "black-box/public/alias.feature"] +ARGUMENTS = { + "identify": {"distinct_id": "user-123", "set": {"active": False, "score": 0, "note": None}}, + "alias": {"distinct_id": "anon-123", "alias": "user-123"}, +} +EVENT_NAMES = {"identify": "$identify", "alias": "$create_alias"} + + +class IdentifyAliasHost(AnalyticsWireHost): + """Translate the two public operations; reuse admission, flush and transport.""" + + def __init__(self, contracts, **options): + super().__init__(contracts, **options) + self.routes.extend(r for r in ("/identify", "/alias") if r != options.get("missing_route")) + + async def invoke(self, fixture, call): + route, args = call["route"], call["args"] + if route not in ("/identify", "/alias"): + return await super().invoke(fixture, call) + if fixture.defect == "operation_throw": + raise RuntimeError("Controlled operation failure") + identity = args["distinct_id"] + if route == "/identify": + properties = {"$set": args["set"]} + else: + properties = {"alias": args["alias"]} + if fixture.defect == "reversed_alias": + identity, properties["alias"] = properties["alias"], identity + await fixture.engine.capture( + {"event": EVENT_NAMES[route[1:]], "distinct_id": identity, "properties": properties} + ) + + +def public_feature(operation): + args = ARGUMENTS[operation] + property_assertion = ( + '"$set" should equal JSON ' + json.dumps(args["set"]) + if operation == "identify" + else '"alias" should equal "user-123"' + ) + return f'''@public @black_box @sdk:server +Feature: Public {operation} regression + @case:regression:{operation} + Scenario: Deliver the public operation + Given an isolated SDK with empty persistent storage + And the SDK is initialized with token "test-token" and flush threshold 20 + When {operation} is called with JSON arguments: + """application/json + {json.dumps(args)} + """ + And pending captures are flushed + Then exactly 1 capture request should have been received + And the first request should contain exactly 1 parsed events + And the first received event field "event" should equal "{EVENT_NAMES[operation]}" + And the first received event field "distinct_id" should equal "{args['distinct_id']}" + And the first received event property {property_assertion} +''' + + +@pytest.fixture +def regression_source(tmp_path): + for operation in ARGUMENTS: + (tmp_path / f"{operation}.feature").write_text(public_feature(operation)) + return tmp_path + + +@pytest.mark.parametrize("operation", ARGUMENTS) +async def test_public_operation_delivers_exact_event(regression_source, operation): + contracts = Contracts() + async with serve(contracts, host_type=IdentifyAliasHost) as (host, url): + report, diagnostics = await run(contracts, regression_source, [f"{operation}.feature"], url, host.profile["id"]) + assert strict_exit_code(contracts, report) == 0, report + assert report["results"][0]["result"]["status"] == "passed" + assert [call["route"] for call in host.inputs] == ["/setup", f"/{operation}", "/flush"] + assert json_equal(host.inputs[1]["args"], ARGUMENTS[operation]) + assert len(host.closed) == 1 + assert len(diagnostics["cases"][0]["ingestion"]) == 1 + + +COMMON_DEFECTS = [ + ("wrong_event", "event_field"), + ("omit_event:event", "event_field"), + ("numeric_identity", "event_field"), + ("omit_event:distinct_id", "event_field"), + ("no_delivery", "request_count"), + ("empty_batch", "batch_count"), + ("append_duplicate_uuid", "batch_count"), + ("duplicate_request", "request_count"), + ("operation_throw", "unexpected_throw"), +] + + +@pytest.mark.parametrize( + "operation,defect,code", + [(op, defect, code) for op in ARGUMENTS for defect, code in COMMON_DEFECTS] + + [ + ("identify", "missing_property:$set", "event_property"), + ("identify", 'property_value:0:$set:{"active":0,"score":0,"note":null}', "event_property"), + ("identify", 'property_value:0:$set:{"active":false,"score":false,"note":null}', "event_property"), + ("identify", 'property_value:0:$set:{"active":false,"score":0}', "event_property"), + ("identify", 'property_value:0:$set:{"active":false,"score":0,"note":false}', "event_property"), + ("alias", "missing_property:alias", "event_property"), + ("alias", 'property_value:0:alias:"wrong"', "event_property"), + ("alias", "reversed_alias", "event_field"), + ], +) +async def test_public_operation_rejects_wire_defects(regression_source, operation, defect, code): + contracts = Contracts() + async with serve(contracts, host_type=IdentifyAliasHost, defect=defect) as (host, url): + report, _ = await run(contracts, regression_source, [f"{operation}.feature"], url, host.profile["id"]) + result = report["results"][0]["result"] + assert result["status"] == "failed_assertion", report + assert result["failure"]["code"] == code + assert result["failure"]["failed_step"]["source"]["path"] == f"{operation}.feature" + assert len(result["failure"]["call_ids"]) == (2 if defect == "operation_throw" else 3) + if defect == "operation_throw": + assert [call["route"] for call in host.inputs] == ["/setup", f"/{operation}"] + outcome = report["calls"][-1]["completion"]["outcome"] + assert outcome["kind"] == "thrown" + assert outcome["error"]["message"] == "Controlled operation failure" + assert len(host.closed) == 1 + assert strict_exit_code(contracts, report) == 1 + + +@pytest.mark.parametrize("operation", ARGUMENTS) +async def test_missing_public_operation_is_not_silently_excluded(regression_source, operation): + contracts = Contracts() + async with serve(contracts, host_type=IdentifyAliasHost, missing_route=f"/{operation}") as (host, url): + report, _ = await run(contracts, regression_source, [f"{operation}.feature"], url, host.profile["id"]) + result = report["results"][0]["result"] + assert result["status"] == "unsupported_binding", report + assert result["failure"]["code"] == "missing_operation" + assert not result["executed"] and not host.fixtures and not host.inputs + assert strict_exit_code(contracts, report) == 1 + + +@pytest.mark.parametrize("operation", ARGUMENTS) +async def test_server_cases_do_not_run_on_client_profile(regression_source, operation): + contracts = Contracts() + async with serve(contracts, host_type=IdentifyAliasHost, runtime="browser") as (host, url): + report, _ = await run(contracts, regression_source, [f"{operation}.feature"], url, host.profile["id"]) + assert report["results"][0]["result"]["status"] == "not_applicable", report + assert not host.inputs and not host.fixtures + assert strict_exit_code(contracts, report) == 1 # An entirely inapplicable scope cannot pass. + + +@pytest.mark.parametrize( + "properties,expected,passes", + [ + ({"value": False}, False, True), + ({"value": 0}, 0, True), + ({"value": None}, None, True), + ({"value": True}, 1, False), + ({"value": 1}, True, False), + ({"value": 0}, False, False), + ({"value": False}, 0, False), + ({}, None, False), + (None, None, False), + ([], None, False), + ({"value": {"nested": [0, None]}}, {"nested": [False, None]}, False), + ({"value": {"nested": [False]}}, {"nested": [False, None]}, False), + ], +) +async def test_received_property_json_preserves_types_and_presence(properties, expected, passes): + observed = [SimpleNamespace(parsed_events=[{"properties": properties}])] + ctx = SimpleNamespace(server=SimpleNamespace(state=SimpleNamespace(get_requests=lambda: observed))) + step = SimpleNamespace( + text='the first received event property "value" should equal JSON ' + json.dumps(expected), + argument={}, + source={"path": "property.feature", "line": 1}, + ) + handler, args = STEPS.bind(step) + if passes: + await handler(ctx, step, *args) + else: + with pytest.raises(BoundaryError) as error: + await handler(ctx, step, *args) + assert error.value.code == "event_property" + + +async def test_companion_identify_alias_features_through_public_http(specs): + cases, _ = load_cases(specs, FEATURES) + assert {case.id for case in cases} == { + "black-box:server:identify:scalar-values", + "black-box:server:identify:nested-values", + "black-box:server:alias:signup", + "black-box:server:alias:second-person", + } + contracts = Contracts() + async with serve(contracts, host_type=IdentifyAliasHost) as (host, url): + report, _ = await run(contracts, specs, FEATURES, url, host.profile["id"]) + assert strict_exit_code(contracts, report) == 0, report + assert [row["result"]["status"] for row in report["results"]] == ["passed"] * 4 + assert len(host.closed) == 4 + assert [call["route"] for call in host.inputs] == [ + route for case in cases for route in ("/setup", "/alias" if "alias" in case.id else "/identify", "/flush") + ] diff --git a/tests/test_v2_legacy_capture.py b/tests/test_v2_legacy_capture.py index 557dc6e..4696133 100644 --- a/tests/test_v2_legacy_capture.py +++ b/tests/test_v2_legacy_capture.py @@ -124,8 +124,12 @@ async def test_presence_identity_batch_and_uuid_assertions_retain_source_weaknes await check(text, [first, observation([{}]), observation([{"uuid": "later"}])]) first.parsed_events = [{"uuid": "0198c0de-0000-4000-8000-000000000abc"}, {"uuid": "invalid"}] await check("the first received event UUID should be valid", [first]) - first.parsed_events = [{"properties": {"custom_number": True}}] - await check('the first received event property "custom_number" should equal JSON 1', [first]) + + +async def test_legacy_property_assertion_rejects_boolean_for_number(): + observed = observation([{"properties": {"custom_number": True}}]) + with pytest.raises(BoundaryError): + await check('the first received event property "custom_number" should equal JSON 1', [observed]) async def test_legacy_counts_include_flags_and_first_delay_is_not_exponential_proof(): From dfc532966b3b42c1e7c4284a790f8590de05d044 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 22 Sep 2026 14:46:43 -0400 Subject: [PATCH 2/3] chore: record v2 identify and alias coverage release --- .sampo/changesets/v2-identify-alias-blackbox.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .sampo/changesets/v2-identify-alias-blackbox.md diff --git a/.sampo/changesets/v2-identify-alias-blackbox.md b/.sampo/changesets/v2-identify-alias-blackbox.md new file mode 100644 index 0000000..b45233d --- /dev/null +++ b/.sampo/changesets/v2-identify-alias-blackbox.md @@ -0,0 +1,5 @@ +--- +pypi/posthog-sdk-test-harness: minor +--- + +Add opt-in v2 server identify and alias delivery assertions and distinguish JSON booleans from numbers in event-property comparisons. From 5f04c69bde3825aacd0a40965c6f4719b6fa7789 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Tue, 22 Sep 2026 15:20:15 -0400 Subject: [PATCH 3/3] test(v2): select in-place server identity acceptance cases --- docs/harness-v2.md | 15 ++++++---- src/posthog_test_harness/v2/ai_steps.py | 9 ++++-- tests/test_v2_discovery.py | 5 +++- tests/test_v2_identify_alias.py | 39 ++++++++++++++++++------- 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/docs/harness-v2.md b/docs/harness-v2.md index f76ee92..8476390 100644 --- a/docs/harness-v2.md +++ b/docs/harness-v2.md @@ -41,11 +41,16 @@ Completion is exactly one of: The migrated bindings use `/setup`, `/capture`, `/capture_ai`, `/flush`, `/get_feature_flag` and `/reload_feature_flags`. Their argument objects appear directly in feature doc strings or named step bindings. Additional shared public operations can be added with concrete scenarios; object references, callback continuations and private fixture-control endpoints are not part of this draft. -## Black-box server identify and alias - -Explicitly select `--feature black-box/public/identify.feature` and -`--feature black-box/public/alias.feature` from a companion specs checkout to run -these four server-only cases. They are separate from the 157-case migration suite. +## Server identify and alias delivery + +Select `--feature acceptance/public/identify.feature` and +`--feature acceptance/public/alias.feature` from a companion specs checkout, with +`--case-id acceptance:server:identify:scalar-values`, +`--case-id acceptance:server:identify:nested-values`, +`--case-id acceptance:server:alias:signup`, and +`--case-id acceptance:server:alias:second-person` to run the four server delivery +cases without selecting the remaining client and validation cases. These are +separate from the 157-case migration suite. The steps `identify is called with JSON arguments:` and `alias is called with JSON arguments:` forward JSON doc strings unchanged to the negotiated `/identify` and `/alias` routes: diff --git a/src/posthog_test_harness/v2/ai_steps.py b/src/posthog_test_harness/v2/ai_steps.py index 3b822bb..59052b1 100644 --- a/src/posthog_test_harness/v2/ai_steps.py +++ b/src/posthog_test_harness/v2/ai_steps.py @@ -26,14 +26,19 @@ def json_arguments(step): return value -@STEPS.step("an isolated SDK with empty persistent storage", fixtures=("storage.empty.v1",)) -async def isolated(ctx, step): +@STEPS.step("an isolated SDK instance") +async def isolated_instance(ctx, step): require(ctx.fixture is None, "invalid_state", "A case can allocate only one fixture") ctx.fixture = await ctx.client.allocate( ctx.diagnostics["fixture_id"], ctx.case.id, ctx.profile["id"], ctx.timeout_ms ) +@STEPS.step("an isolated SDK with empty persistent storage", fixtures=("storage.empty.v1",)) +async def isolated(ctx, step): + await isolated_instance(ctx, step) + + @STEPS.step(r'the SDK is initialized with token "([^"]*)" and flush threshold ([0-9]+)', routes=("/setup",)) async def setup(ctx, step, token, threshold): await ctx.call("/setup", {"project_token": token, "config": {"host": ctx.server.url, "flush_at": int(threshold)}}) diff --git a/tests/test_v2_discovery.py b/tests/test_v2_discovery.py index c354087..20fe0c5 100644 --- a/tests/test_v2_discovery.py +++ b/tests/test_v2_discovery.py @@ -20,7 +20,10 @@ def test_discovery_from_features_without_catalog_or_ledger(specs): assert all(c["status"] == "harness_ready" for c in cases) assert all(c["required_fixture_capabilities"] == ["storage.empty.v1"] for c in cases) canonical = [p for p in feature_paths(specs) if p.startswith("acceptance/")] - assert len(discover(specs, canonical)["cases"]) == 728 + discovered = discover(specs, canonical)["cases"] + parsed, _ = load_cases(specs, canonical) + assert len(discovered) == len(parsed) > 0 + assert len({c["case_id"] for c in discovered}) == len(discovered) def test_unlisted_local_feature_and_changed_content_are_executable_inputs(tmp_path): diff --git a/tests/test_v2_identify_alias.py b/tests/test_v2_identify_alias.py index e8d7183..78746d0 100644 --- a/tests/test_v2_identify_alias.py +++ b/tests/test_v2_identify_alias.py @@ -12,7 +12,13 @@ from tests.v2_analytics_wire_host import AnalyticsWireHost from tests.v2_flush_host import serve -FEATURES = ["black-box/public/identify.feature", "black-box/public/alias.feature"] +FEATURES = ["acceptance/public/identify.feature", "acceptance/public/alias.feature"] +SERVER_CASE_IDS = { + "acceptance:server:identify:scalar-values", + "acceptance:server:identify:nested-values", + "acceptance:server:alias:signup", + "acceptance:server:alias:second-person", +} ARGUMENTS = { "identify": {"distinct_id": "user-123", "set": {"active": False, "score": 0, "note": None}}, "alias": {"distinct_id": "anon-123", "alias": "user-123"}, @@ -192,20 +198,33 @@ async def test_received_property_json_preserves_types_and_presence(properties, e assert error.value.code == "event_property" +async def test_server_delivery_needs_no_storage_control(regression_source): + feature = regression_source / "identify.feature" + feature.write_text( + feature.read_text().replace("an isolated SDK with empty persistent storage", "an isolated SDK instance") + ) + contracts = Contracts() + async with serve(contracts, host_type=IdentifyAliasHost, missing_capability="storage.empty.v1") as (host, url): + report, _ = await run(contracts, regression_source, ["identify.feature"], url, host.profile["id"]) + assert strict_exit_code(contracts, report) == 0, report + assert [call["route"] for call in host.inputs] == ["/setup", "/identify", "/flush"] + + async def test_companion_identify_alias_features_through_public_http(specs): cases, _ = load_cases(specs, FEATURES) - assert {case.id for case in cases} == { - "black-box:server:identify:scalar-values", - "black-box:server:identify:nested-values", - "black-box:server:alias:signup", - "black-box:server:alias:second-person", - } + selected = [case for case in cases if case.id in SERVER_CASE_IDS] + assert {case.id for case in selected} == SERVER_CASE_IDS + assert len(cases) == 8 # Existing client and validation cases remain in their acceptance files. + assert all(case.migration is None and "@sdk:server" in case.tags for case in selected) contracts = Contracts() async with serve(contracts, host_type=IdentifyAliasHost) as (host, url): - report, _ = await run(contracts, specs, FEATURES, url, host.profile["id"]) + report, _ = await run(contracts, specs, FEATURES, url, host.profile["id"], case_ids=sorted(SERVER_CASE_IDS)) assert strict_exit_code(contracts, report) == 0, report - assert [row["result"]["status"] for row in report["results"]] == ["passed"] * 4 + assert [row["result"]["status"] for row in report["results"] if row["case_id"] in SERVER_CASE_IDS] == ["passed"] * 4 + assert [row["result"]["status"] for row in report["results"] if row["case_id"] not in SERVER_CASE_IDS] == [ + "not_selected" + ] * 4 assert len(host.closed) == 4 assert [call["route"] for call in host.inputs] == [ - route for case in cases for route in ("/setup", "/alias" if "alias" in case.id else "/identify", "/flush") + route for case in selected for route in ("/setup", "/alias" if "alias" in case.id else "/identify", "/flush") ]