From 4fadb1d77052f26a53f60ca9fb7f65cb3061a747 Mon Sep 17 00:00:00 2001 From: Chris Uthe Date: Wed, 7 Oct 2026 14:52:48 -0500 Subject: [PATCH 1/2] fix(sendspin-dotnet adapter): advertise 24-bit PCM in server-initiated-pcm-24bit The adapter listed 16-bit PCM in every scenario, so the server streamed 16-bit and the SDK's 24-bit decode path never ran in the scenario that exists to test it. It now lists 24-bit as the only depth there, as the scenario describes. --- adapters/sendspin-dotnet/client/Program.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/adapters/sendspin-dotnet/client/Program.cs b/adapters/sendspin-dotnet/client/Program.cs index 8303cf4..bad0426 100644 --- a/adapters/sendspin-dotnet/client/Program.cs +++ b/adapters/sendspin-dotnet/client/Program.cs @@ -402,10 +402,13 @@ async Task WaitForRegistryAsync(string path, string serverName, Cancella static ClientCapabilities BuildCapabilities(CliOptions options) { + // The 24-bit scenario offers 24-bit as its only depth, so the server has no + // 16-bit entry to stream and the SDK's 24-bit decode path is the one that runs. + var pcmBitDepth = options.ScenarioId == "server-initiated-pcm-24bit" ? 24 : 16; var audioFormats = options.PreferredCodec.Equals("pcm", StringComparison.OrdinalIgnoreCase) ? new List { - new() { Codec = "pcm", SampleRate = 8000, Channels = 1, BitDepth = 16 }, + new() { Codec = "pcm", SampleRate = 8000, Channels = 1, BitDepth = pcmBitDepth }, } : new List { From a6b2991010b4341a0b13cad3456848c3fb69fcef Mon Sep 17 00:00:00 2001 From: Chris Uthe Date: Wed, 7 Oct 2026 14:52:48 -0500 Subject: [PATCH 2/2] feat(conformance): fail server-initiated-pcm-24bit when the stream was not 24-bit The canonical PCM hash is taken over float samples, so a 16-bit stream of the clip matched it and the cell passed without the 24-bit path being exercised. The scenario now also reads the negotiated bit depth from the server's recorded stream, and from the client's where it reports one. RC1 requires no bit depth of a player, so the reason names a harness gap or the server's selection, never a client defect. scenario_revision goes from 6 to 7. --- README.md | 2 +- src/conformance/declared_formats.py | 75 +++++++++- src/conformance/models.py | 4 + src/conformance/runner.py | 17 ++- src/conformance/scenarios.py | 7 +- tests/test_stream_bit_depth.py | 205 ++++++++++++++++++++++++++++ 6 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 tests/test_stream_bit_depth.py diff --git a/README.md b/README.md index cd0e771..35e3b4e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Current scenarios: - `server-initiated-controller` (Server initiates connection and client wants Controller): start the server first, let the client advertise a listener, let the server connect in, observe controller state, send a control command, and verify the server recorded it - `server-initiated-flac` (Server initiates connection and client wants FLAC): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and FLAC as its only supported audio format, let the server connect in, encode the PCM to FLAC using the SDK, stream it to the client, and compare the transported FLAC bytes; where the client reports the audio it decoded and the stream kept the source's sample rate and channel count, also compare its canonical PCM hash against the source clip, so an intact but incomplete stream fails - `server-initiated-opus` (Server initiates connection and client prefers OPUS): start the server first with PCM audio decoded from `almost_silent.flac`, let the client advertise a listener and list OPUS first in its supported audio formats, followed by the PCM or FLAC entry every player must list, let the server connect in, and verify that it honoured that priority by encoding the PCM to OPUS using the SDK and streaming it to the client; compare the declared list, the negotiated format and the transported OPUS bytes. OPUS is lossy, so decoded audio is not compared against the source clip -- `server-initiated-pcm-24bit` (Server initiates connection and client wants 24-bit PCM): start the server first, let the client advertise a listener and 24-bit PCM as its only supported audio format, let the server connect in, negotiate the 24-bit packed wire format, stream it to the client, and compare canonical PCM hashes — a client SDK with no 24-bit decode path misreads the bytes and produces a hash mismatch +- `server-initiated-pcm-24bit` (Server initiates connection and client wants 24-bit PCM): start the server first, let the client advertise a listener and 24-bit PCM as its only supported audio format, let the server connect in, negotiate the 24-bit packed wire format, stream it to the client, compare canonical PCM hashes, and verify that the negotiated stream was 24-bit — a client SDK with no 24-bit decode path misreads the bytes and produces a hash mismatch, and a pair that streamed another depth has not tested the 24-bit path - `client-initiated-state-format-pcm` (Client changes its preferred PCM bit depth via `client/state`): start the server first, let the client connect advertising two PCM formats (24-bit then 16-bit), let the server stream the higher-priority 24-bit format, have the client send a `client/state` whose player `format` prefers the 16-bit entry, and verify the server received that preference in `client/state` and the client observed a new `stream/start` in the 16-bit format - `client-initiated-state-format-flac` (Client changes its preferred format from PCM to FLAC via `client/state`): start the server first, let the client connect advertising PCM then FLAC, let the server stream the higher-priority PCM format, have the client send a `client/state` whose player `format` prefers the FLAC entry, and verify the server received that preference in `client/state` and the client observed a new `stream/start` in the FLAC format - `client-initiated-metadata` (Client initiates connection and client wants Metadata): start the server first, let the client discover/connect to it, receive a metadata snapshot, and compare normalized metadata fields diff --git a/src/conformance/declared_formats.py b/src/conformance/declared_formats.py index 914ac09..52718c7 100644 --- a/src/conformance/declared_formats.py +++ b/src/conformance/declared_formats.py @@ -6,12 +6,14 @@ claim is visible as such instead of hiding behind a green case. The join is mostly observational: it reports what a client claimed and what the -wire carried. Two verdicts are drawn from it. `undeclared_format_violation` +wire carried. Three verdicts are drawn from it. `undeclared_format_violation` asserts the spec MUST that a negotiated format be one the client listed, and applies to every case. `format_priority_violation` asserts that the server picked the first entry of a list offering it a choice, and applies only to a -scenario built around that choice. Whether any other scenario got the format it -set out to test is not recorded in machine-readable form anywhere yet. +scenario built around that choice. `stream_bit_depth_violation` asserts that the +negotiated stream carried the bit depth a scenario exists to test, and applies +only to a scenario naming one. Whether any other scenario got the format it set +out to test is not recorded in machine-readable form anywhere yet. """ from __future__ import annotations @@ -209,6 +211,73 @@ def format_priority_violation( return None +def stream_bit_depth_violation( + server_summary: dict[str, Any], + client_summary: dict[str, Any], + *, + bit_depth: int, +) -> str | None: + """Return why the case did not exercise a stream of `bit_depth`, or None. + + For a scenario whose subject is one bit depth. The canonical PCM hash is + taken over float samples, so a stream of the same clip at another depth + still matches it; only the negotiated format shows which path the pair ran. + + The spec requires no particular bit depth of a player, so a reason returned + here never describes a conformance defect in the client. It says the case + did not test what the scenario is for, and names the side whose evidence + shows why: + + - The server's recorded stream is what was put on the wire, so it decides. + A server summary without a readable one is a harness gap rather than a + pass, because the server adapters belong to this harness. + - Where that stream has another depth and the client's `supported_formats` + offered none at `bit_depth`, the client adapter did not set up the + stimulus. That is a harness gap too: the adapter writes the declaration. + A declaration that is missing or holds an unreadable entry cannot show + that, so the reason then names neither side. + - Where the client did offer `bit_depth`, the server chose otherwise. + - A client that reports the stream it observed has to have observed the + same depth. A client summary without a readable stream is not judged. + """ + stream = normalized_format(server_summary.get("stream")) + if stream is None: + return ( + "Harness gap: the server adapter recorded no stream format, so the case " + f"cannot show that it tested {bit_depth}-bit audio" + ) + + declared, unreadable = _declared_formats(server_summary) + offered = ", ".join(format_label(audio_format) for audio_format in declared) or "none recorded" + if stream["bit_depth"] != bit_depth: + if not declared or unreadable: + return ( + f"Harness gap: the stream was {format_label(stream)}, so the case cannot " + f"test {bit_depth}-bit audio, and the server summary records no readable " + "supported_formats from client/hello to show which side chose it; " + f"declared: {offered}" + ) + if any(audio_format["bit_depth"] == bit_depth for audio_format in declared): + return ( + f"Server streamed {format_label(stream)} although the client listed a " + f"{bit_depth}-bit format, so the case cannot test {bit_depth}-bit audio; " + f"declared: {offered}" + ) + return ( + f"Harness gap: the client adapter advertised no {bit_depth}-bit format and " + f"the stream was {format_label(stream)}, so the case cannot test " + f"{bit_depth}-bit audio; declared: {offered}" + ) + + observed = normalized_format(client_summary.get("stream")) + if observed is not None and observed["bit_depth"] != bit_depth: + return ( + f"Client observed a {format_label(observed)} stream although the server " + f"sent {format_label(stream)}; declared: {offered}" + ) + return None + + def _declared_formats( server_summary: dict[str, Any], ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: diff --git a/src/conformance/models.py b/src/conformance/models.py index 873e783..7aceef8 100644 --- a/src/conformance/models.py +++ b/src/conformance/models.py @@ -147,6 +147,10 @@ class ScenarioSpec: # The case also checks that the client listed `preferred_codec` first with a # pcm or flac entry after it, and that the server picked the first entry. verifies_format_priority: bool = False + # The case also checks that the stream the pair negotiated carried this bit + # depth. It is a verdict on a case that ran, never a reason to keep an + # implementation out of the scenario. + verifies_stream_bit_depth: int | None = None extra_cli_args: tuple[tuple[str, str], ...] = field(default_factory=tuple) protocol_assertions: tuple[str, ...] = field(default_factory=tuple) diff --git a/src/conformance/runner.py b/src/conformance/runner.py index 90227cf..9dc6df3 100644 --- a/src/conformance/runner.py +++ b/src/conformance/runner.py @@ -15,7 +15,11 @@ from typing import Any from .chunk_framing import chunk_framing_verdict -from .declared_formats import format_priority_violation, undeclared_format_violation +from .declared_formats import ( + format_priority_violation, + stream_bit_depth_violation, + undeclared_format_violation, +) from .environment import resolve_environment from .fixtures import fixture_path from .implementations import ( @@ -1235,7 +1239,16 @@ def _dispatch_comparison( return False, failure return True, f"Passed {len(scenario.protocol_assertions)} protocol assertions" if scenario.verification_mode == "audio-pcm": - return _compare_audio_summaries(server_summary, client_summary) + matches, reason = _compare_audio_summaries(server_summary, client_summary) + if matches and scenario.verifies_stream_bit_depth is not None: + violation = stream_bit_depth_violation( + server_summary, + client_summary, + bit_depth=scenario.verifies_stream_bit_depth, + ) + if violation is not None: + return False, violation + return matches, reason if scenario.verification_mode == "audio-encoded-bytes": # A server that failed keeps its own reason; the status checks below name it. if scenario.verifies_format_priority and server_summary.get("status") == "ok": diff --git a/src/conformance/scenarios.py b/src/conformance/scenarios.py index c51c519..6c9cfd1 100644 --- a/src/conformance/scenarios.py +++ b/src/conformance/scenarios.py @@ -190,13 +190,16 @@ "client advertises a listener and 24-bit PCM as its only supported audio " "format. The matrix compares canonical PCM hashes after the client unpacks " "the 24-bit stream. A client SDK with no 24-bit decode path misreads the " - "bytes and produces a hash mismatch." + "bytes and produces a hash mismatch. The matrix also verifies that the " + "negotiated stream was 24-bit, because a 16-bit stream of the same clip " + "yields the same canonical hash without exercising the 24-bit path." ), initiator_role="server", preferred_codec="pcm", required_role_families=("player",), verification_mode="audio-pcm", - scenario_revision=6, + scenario_revision=7, + verifies_stream_bit_depth=24, ) diff --git a/tests/test_stream_bit_depth.py b/tests/test_stream_bit_depth.py new file mode 100644 index 0000000..c754e89 --- /dev/null +++ b/tests/test_stream_bit_depth.py @@ -0,0 +1,205 @@ +"""Coverage for the negotiated-depth rule behind `server-initiated-pcm-24bit`. + +The canonical PCM hash is taken over float samples, so a 16-bit stream of the +clip matches it as well as a 24-bit one does. The scenario therefore also reads +the negotiated bit depth, and a pair that streamed anything else has not tested +the 24-bit path. `roles/player/v1.md` requires no bit depth of a player, so the +reason must name a harness gap or the server's selection, never a client defect. +Reproducing each over the wire needs an adapter built to misbehave, so the rule +is exercised here against synthetic summaries. +""" + +from __future__ import annotations + +import unittest +from typing import Any + +from conformance.declared_formats import stream_bit_depth_violation +from conformance.runner import _compare_summaries +from conformance.scenarios import SCENARIOS, require_scenario + +PCM_16 = {"codec": "pcm", "sample_rate": 8000, "bit_depth": 16, "channels": 1} +PCM_24 = {"codec": "pcm", "sample_rate": 8000, "bit_depth": 24, "channels": 1} + +SOURCE_HASH = "a" * 64 + +# Every server summary carries one, and a case whose summary lacks it never passes. +GROUP_UPDATE = { + "type": "group/update", + "payload": {"playback_state": "stopped", "group_id": "group-1", "group_name": "Kitchen"}, +} + +# What a server that held stream/start until the client reported available records. +AVAILABILITY_TRACE = [ + {"type": "client/state", "available": True}, + {"type": "stream/start", "phase": "sending", "roles": ["player"]}, + {"type": "stream/start", "phase": "sent", "roles": ["player"]}, +] + + +def _server_summary(*, declared: Any, stream: dict[str, Any] | None) -> dict[str, Any]: + return { + "status": "ok", + "implementation": "synthetic-server", + "role": "server", + "group_updates": [GROUP_UPDATE], + "time_exchange": [], + "availability_trace": AVAILABILITY_TRACE, + "peer_hello": { + "type": "client/hello", + "payload": {"player@v1_support": {"supported_formats": declared}}, + }, + "stream": stream, + "audio": {"source_pcm_sha256": SOURCE_HASH, "channels": 1, "frame_count": 1000}, + } + + +def _client_summary(*, stream: dict[str, Any] | None = None) -> dict[str, Any]: + summary: dict[str, Any] = { + "status": "ok", + "implementation": "synthetic-client", + "role": "client", + "audio": { + "received_pcm_sha256": SOURCE_HASH, + "received_sample_count": 1000, + "audio_chunk_count": 10, + }, + } + if stream is not None: + summary["stream"] = stream + return summary + + +class StreamBitDepthViolationTests(unittest.TestCase): + """A case passes the rule only when the stream carried the depth under test.""" + + def test_a_24_bit_stream_seen_by_both_sides_passes(self) -> None: + self.assertIsNone( + self._violation( + _server_summary(declared=[PCM_24], stream=PCM_24), + _client_summary(stream=PCM_24), + ) + ) + + def test_a_client_reporting_no_stream_is_not_judged(self) -> None: + self.assertIsNone( + self._violation(_server_summary(declared=[PCM_24], stream=PCM_24), _client_summary()) + ) + + def test_an_undeclared_depth_is_a_harness_gap_not_a_client_defect(self) -> None: + reason = self._violation( + _server_summary(declared=[PCM_16], stream=PCM_16), + _client_summary(stream=PCM_16), + ) + assert reason is not None + self.assertTrue(reason.startswith("Harness gap: the client adapter advertised no 24-bit")) + self.assertIn("the stream was PCM · mono · 8 kHz · 16-bit", reason) + self.assertIn("declared: PCM · mono · 8 kHz · 16-bit", reason) + self.assertNotIn("support", reason) + + def test_a_server_that_streamed_another_depth_than_offered_is_named(self) -> None: + reason = self._violation( + _server_summary(declared=[PCM_24, PCM_16], stream=PCM_16), + _client_summary(stream=PCM_16), + ) + assert reason is not None + self.assertTrue(reason.startswith("Server streamed PCM · mono · 8 kHz · 16-bit")) + self.assertNotIn("Harness gap", reason) + + def test_a_client_that_observed_another_depth_is_named(self) -> None: + reason = self._violation( + _server_summary(declared=[PCM_24], stream=PCM_24), + _client_summary(stream=PCM_16), + ) + assert reason is not None + self.assertTrue(reason.startswith("Client observed a PCM · mono · 8 kHz · 16-bit stream")) + + def test_a_server_summary_without_a_stream_fails_closed(self) -> None: + reason = self._violation( + _server_summary(declared=[PCM_24], stream=None), + _client_summary(stream=PCM_24), + ) + assert reason is not None + self.assertTrue(reason.startswith("Harness gap: the server adapter recorded no stream")) + + def test_an_unrecorded_declaration_still_fails_a_16_bit_stream(self) -> None: + server = _server_summary(declared=[PCM_16], stream=PCM_16) + server["peer_hello"] = None + reason = self._violation(server, _client_summary()) + assert reason is not None + self.assertTrue(reason.startswith("Harness gap: the stream was PCM")) + self.assertNotIn("client adapter", reason) + self.assertIn("declared: none recorded", reason) + + def test_an_unreadable_declaration_names_neither_side(self) -> None: + reason = self._violation( + _server_summary(declared=[PCM_16, {"codec": "pcm"}], stream=PCM_16), + _client_summary(), + ) + assert reason is not None + self.assertTrue(reason.startswith("Harness gap: the stream was PCM")) + self.assertNotIn("client adapter", reason) + + def test_a_client_stream_that_cannot_be_read_is_not_judged(self) -> None: + self.assertIsNone( + self._violation( + _server_summary(declared=[PCM_24], stream=PCM_24), + _client_summary(stream={"codec": "pcm"}), + ) + ) + + def _violation(self, server: dict[str, Any], client: dict[str, Any]) -> str | None: + return stream_bit_depth_violation(server, client, bit_depth=24) + + +class Pcm24BitScenarioTests(unittest.TestCase): + """The scenario applies the rule, and no other scenario does.""" + + def test_matching_hashes_on_a_16_bit_stream_fail_the_case(self) -> None: + matches, reason = _compare_summaries( + require_scenario("server-initiated-pcm-24bit"), + _server_summary(declared=[PCM_16], stream=PCM_16), + _client_summary(), + ) + self.assertFalse(matches) + self.assertIn("cannot test 24-bit audio", reason) + + def test_matching_hashes_on_a_24_bit_stream_pass_the_case(self) -> None: + matches, reason = _compare_summaries( + require_scenario("server-initiated-pcm-24bit"), + _server_summary(declared=[PCM_24], stream=PCM_24), + _client_summary(stream=PCM_24), + ) + self.assertTrue(matches, reason) + self.assertEqual(reason, "PCM hashes match exactly") + + def test_a_hash_mismatch_keeps_its_own_reason(self) -> None: + client = _client_summary() + client["audio"]["received_pcm_sha256"] = "b" * 64 + matches, reason = _compare_summaries( + require_scenario("server-initiated-pcm-24bit"), + _server_summary(declared=[PCM_16], stream=PCM_16), + client, + ) + self.assertFalse(matches) + self.assertTrue(reason.startswith("PCM hash mismatch")) + + def test_only_the_24_bit_scenario_names_a_bit_depth(self) -> None: + naming = { + scenario.id: scenario.verifies_stream_bit_depth + for scenario in SCENARIOS.values() + if scenario.verifies_stream_bit_depth is not None + } + self.assertEqual(naming, {"server-initiated-pcm-24bit": 24}) + + def test_a_16_bit_stream_still_passes_the_plain_pcm_scenario(self) -> None: + matches, reason = _compare_summaries( + require_scenario("server-initiated-pcm"), + _server_summary(declared=[PCM_16], stream=PCM_16), + _client_summary(stream=PCM_16), + ) + self.assertTrue(matches, reason) + + +if __name__ == "__main__": + unittest.main()