diff --git a/.bun-version b/.bun-version index 428b770..1c99cf0 100644 --- a/.bun-version +++ b/.bun-version @@ -1 +1 @@ -1.4.3 +1.4.4 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..92dc01a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.rs text eol=lf +*.toml text eol=lf +Cargo.lock text eol=lf +conformance/** text eol=lf +protocol/** text eol=lf +*.der binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bccc2d7..8942651 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,7 @@ jobs: with: bun-version-file: .bun-version - run: npm ci + - run: node conformance/verify.mjs - run: npm run typecheck - run: npm test - run: npm run test:cjs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..001c5d7 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,34 @@ +name: Rust native stream acceptance +on: + push: + branches: [master, dev] + pull_request: +permissions: + contents: read +concurrency: + group: rust-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + rust: + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + rust: ["1.88.0", stable] + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.rust == '1.88.0' && '24' || '26' }} + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec + with: + toolchain: ${{ matrix.rust }} + components: rustfmt, clippy + - run: node conformance/verify.mjs --core + - run: cargo fmt --all -- --check + - run: cargo clippy --workspace --all-targets --locked -- -D warnings + - run: cargo test --workspace --all-targets --locked + - run: cargo test --workspace --doc --locked + - run: cargo doc --workspace --no-deps --locked + - run: cargo package -p hyperion-stream-core --locked diff --git a/.gitignore b/.gitignore index 44a5953..0586890 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ logs *.log npm-debug.log* node_modules/ +target/ *.tsbuildinfo .npm .eslintcache diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..86a7d1b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,651 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyperion-stream-core" +version = "0.1.0" + +[[package]] +name = "hyperion-stream-ws" +version = "0.1.0" +dependencies = [ + "futures-util", + "hyperion-stream-core", + "rustls", + "serde", + "serde_json", + "tokio", + "tokio-rustls", + "tokio-tungstenite", + "webpki-roots 1.0.11", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots 0.26.11", +] + +[[package]] +name = "tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.11", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6746725 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,21 @@ +[workspace] +members = ["crates/hyperion-stream-core", "crates/hyperion-stream-ws"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +rust-version = "1.88" +license = "MIT" +repository = "https://github.com/eosrio/hyperion-stream-client" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.lints.clippy] +all = "warn" + +[profile.release] +lto = "thin" +codegen-units = 1 + diff --git a/README.md b/README.md index 9ff154d..bce525f 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,13 @@ independently of the server. Merging changes does not publish an npm release or every server deployment. See [migration notes](docs/MIGRATION.md) and the [launch checklist](docs/LAUNCH.md). +An experimental [Rust native client](crates/hyperion-stream-ws/README.md) is also +available in this workspace: a [zero-dependency core](crates/hyperion-stream-core/README.md) +and a separate Tokio/Tungstenite/Rustls adapter. It targets the additive native +WebSocket + JSON protocol coordinated in [HAPI #294](https://github.com/eosrio/hyperion-history-api/issues/294). +See the [wire contract](protocol/v1/README.md) and [shared conformance](conformance/README.md). +The native endpoint and retained-chain acceptance are still being integrated. + ## Usage ```sh diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..aae1ee9 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,33 @@ +# Shared native streaming conformance + +`v1/wire-vectors.json` contains literal valid/invalid wire messages. Both the Rust +codec and independent native JSON/BigInt driver consume the same cases. Unknown +fields are reserved extensions; unknown packet types fail. + +`v1/scenarios.json` describes state-machine actions and literal public effects. +Operations are server (a decoded wire packet), subscribe, poll, cancel, disconnect, +and advance (injected monotonic milliseconds). Every operation asserts commands +and events, or an explicit fatal code. Expected objects may select fields; arrays +match exactly. Rust runs these through the public core + codec. Future language +clients can add a driver for this same format without translating expectations. + +`v1/network-sessions.json` runs the same ordered action/delta history/live exchange +against both the Rust adapter and native Node WebSocket peer. The fixture checks +subscription JSON, exact ACK scopes/order, independent heartbeat controls, record +identity and payload preservation. The Node peer is a test peer, not a production +client with recovery/finality parity. The HAPI lane consumes the pinned corpus in +eosrio/hyperion-history-api#294; fixture success alone does not qualify its real +history, routing, cancellation, filter, fanout or retained-chain completeness. + +```sh +node conformance/verify.mjs --core +cargo test --workspace --all-targets --locked +``` + +Node 24+ must be on PATH for the native network peer. Set `HYPERION_NODE` to an +explicit executable to select another runtime. Core-only tests require only Rust: +`cargo test -p hyperion-stream-core --locked`. + +The SHA256 manifest pins the contract and literal corpora. Update it deliberately +when changing scenarios; send the exact Git revision/hash to the server owner. +All corpus text is checked out with LF so frozen hashes match on Windows/Linux. diff --git a/conformance/v1/manifest.json b/conformance/v1/manifest.json new file mode 100644 index 0000000..6932e04 --- /dev/null +++ b/conformance/v1/manifest.json @@ -0,0 +1,22 @@ +{ + "version": 1, + "revision": 2, + "resources": [ + { + "path": "protocol/v1/README.md", + "sha256": "e7635fead5c909277f04086b0c857b0a20954bf6bd442eac15a71f862489f907" + }, + { + "path": "conformance/v1/wire-vectors.json", + "sha256": "dfa5e6fc58ac3a9ef510c32e9cb2f9db6a05e2ef7a7dbe7d8be1678e79d58f49" + }, + { + "path": "conformance/v1/scenarios.json", + "sha256": "856fd39dd1c50651c9cd2e73494c9874e94a5100a71341b559edea609181f3bf" + }, + { + "path": "conformance/v1/network-sessions.json", + "sha256": "065506e140c7b54cebf15ace41382447782f6b7797fea277a33c60a48a248d09" + } + ] +} diff --git a/conformance/v1/network-sessions.json b/conformance/v1/network-sessions.json new file mode 100644 index 0000000..2572293 --- /dev/null +++ b/conformance/v1/network-sessions.json @@ -0,0 +1,268 @@ +{ + "version": 1, + "cases": [ + { + "name": "action native interoperability", + "request": { + "kind": "action", + "contract": "eosio.token", + "action": "transfer", + "start_from": 100, + "filter_op": "or", + "filters": [ + { + "field": "data.flag", + "value": false + }, + { + "field": "data.count", + "value": 0 + }, + { + "field": "data.memo", + "value": "" + } + ] + }, + "server_frames": [ + { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615", + "act": { + "data": { + "memo": "日本語 🌐", + "sequence": 42 + } + } + } + } + ] + }, + { + "type": "ping", + "id": "18446744073709551615" + }, + { + "type": "history_end", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100" + }, + { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "2", + "mode": "live", + "records": [ + { + "block_num": 101, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615", + "act": { + "data": { + "memo": "日本語 🌐", + "sequence": 42 + } + } + } + } + ] + }, + { + "type": "end", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100" + } + ], + "expected_records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615", + "act": { + "data": { + "memo": "日本語 🌐", + "sequence": 42 + } + } + } + }, + { + "block_num": 101, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615", + "act": { + "data": { + "memo": "日本語 🌐", + "sequence": 42 + } + } + } + } + ], + "expected_acks": [ + "1", + "2" + ] + }, + { + "name": "delta native interoperability", + "request": { + "kind": "delta", + "code": "eosio.token", + "table": "accounts", + "scope": "*", + "start_from": 100, + "filter_op": "or", + "filters": [ + { + "field": "data.flag", + "value": false + }, + { + "field": "data.count", + "value": 0 + }, + { + "field": "data.memo", + "value": "" + } + ] + }, + "server_frames": [ + { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "primary_key": "account:340282366920938463463374607431768211455", + "data": { + "balance": "1.0000 EOS" + } + } + } + ] + }, + { + "type": "ping", + "id": "18446744073709551615" + }, + { + "type": "history_end", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100" + }, + { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "2", + "mode": "live", + "records": [ + { + "block_num": 101, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "primary_key": "account:340282366920938463463374607431768211455", + "data": { + "balance": "1.0000 EOS" + } + } + } + ] + }, + { + "type": "end", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100" + } + ], + "expected_records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "primary_key": "account:340282366920938463463374607431768211455", + "data": { + "balance": "1.0000 EOS" + } + } + }, + { + "block_num": 101, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "primary_key": "account:340282366920938463463374607431768211455", + "data": { + "balance": "1.0000 EOS" + } + } + } + ], + "expected_acks": [ + "1", + "2" + ] + } + ] +} diff --git a/conformance/v1/node-peer.mjs b/conformance/v1/node-peer.mjs new file mode 100644 index 0000000..4499870 --- /dev/null +++ b/conformance/v1/node-peer.mjs @@ -0,0 +1,34 @@ +// Minimal native WebSocket protocol peer for cross-language network conformance. +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { validateServerFrame } from './node-wire.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('./network-sessions.json', import.meta.url), 'utf8')).cases[Number(process.argv[3] ?? 0)]; +const ws = new WebSocket(process.argv[2], 'hyperion-stream.v1.json'); +const records = []; +let historyEnd = false; +const deadline = setTimeout(() => { console.error('Native peer deadline elapsed'); ws.close(); process.exitCode = 1; }, 5000); +const send = value => ws.send(JSON.stringify(value)); +ws.addEventListener('error', event => { console.error(event.message ?? 'WebSocket error'); clearTimeout(deadline); process.exitCode = 1; }); +ws.addEventListener('message', event => { + try { + const packet = validateServerFrame(event.data); + switch (packet.type) { + case 'hello': + assert.equal(ws.protocol, 'hyperion-stream.v1.json'); + send({ type: 'ready', connection_id: packet.connection_id, window: { records: 4096, bytes: 8388608 } }); + send({ type: 'subscribe', connection_id: packet.connection_id, request_id: '1', stream_id: '1', generation: '1', request: fixture.request, resume_from: null, window: { records: 1024, bytes: 2097152 } }); break; + case 'accepted': assert.equal(packet.request_id, '1'); break; + case 'data': + assert.equal(packet.mode === 'live', historyEnd); + records.push(...packet.records); + send({ type: 'ack', connection_id: packet.connection_id, stream_id: packet.stream_id, generation: packet.generation, subscription_id: packet.subscription_id, sequence: packet.sequence }); break; + case 'ping': send({ type: 'pong', id: packet.id }); break; + case 'history_end': assert(!historyEnd); historyEnd = true; break; + case 'end': + assert(historyEnd); assert.deepEqual(records, fixture.expected_records); + clearTimeout(deadline); console.log(JSON.stringify({ records })); ws.close(); break; + default: assert.fail(`unexpected ${packet.type}`); + } + } catch (error) { clearTimeout(deadline); console.error(error); process.exitCode = 1; ws.close(); } +}); diff --git a/conformance/v1/node-wire.mjs b/conformance/v1/node-wire.mjs new file mode 100644 index 0000000..a233834 --- /dev/null +++ b/conformance/v1/node-wire.mjs @@ -0,0 +1,93 @@ +// Independent platform JSON/BigInt validation for the shared wire vectors. +// This is a conformance peer, not the published JavaScript streaming client. +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const decimal = value => typeof value === 'string' && value.length <= 20 && /^(0|[1-9][0-9]*)$/.test(value) && BigInt(value) <= 18446744073709551615n; +const positive = value => decimal(value) && value !== '0'; +const block = value => Number.isInteger(value) && value > 0 && value <= 4294967295; +const hash = value => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value); +const object = value => value !== null && typeof value === 'object' && !Array.isArray(value); +const envelopeKeys = new Set(['type', 'connection_id', 'request_id', 'stream_id', 'generation', 'subscription_id', 'sequence', 'mode', 'records', 'id', 'version', 'chain_id', 'heartbeat_interval_ms', 'start_from', 'code', 'message', 'retryable', 'block_num', 'block_id', 'starting_block', 'ending_block', 'new_block_id']); + +function uniqueEnvelope(text) { + // JSON.parse validates grammar first. Tokenize strings/punctuation to detect + // duplicate routing keys before native JSON's last-value behavior can hide one. + const stack = []; + for (const token of text.matchAll(/"(?:\\.|[^"\\])*"|[{}\[\]:,]/g)) { + const value = token[0]; + if (value === '{' || value === '[') stack.push({ object: value === '{', key: true, seen: new Set() }); + else if (value === '}' || value === ']') stack.pop(); + else if (value === ':') { if (stack.at(-1)) stack.at(-1).key = false; } + else if (value === ',') { if (stack.at(-1)?.object) stack.at(-1).key = true; } + else if (stack.length === 1 && stack[0].object && stack[0].key) { + const key = JSON.parse(value); + if (envelopeKeys.has(key)) { assert(!stack[0].seen.has(key), `duplicate ${key}`); stack[0].seen.add(key); } + } + } +} + +function metadata(payload) { + assert(object(payload)); + if (Object.hasOwn(payload, 'global_sequence')) assert(decimal(payload.global_sequence)); + const receipt = value => { + assert(object(value)); + for (const key of ['global_sequence', 'recv_sequence']) if (Object.hasOwn(value, key)) assert(decimal(value[key])); + if (Object.hasOwn(value, 'auth_sequence')) { + assert(Array.isArray(value.auth_sequence)); + for (const auth of value.auth_sequence) { assert(object(auth)); if (Object.hasOwn(auth, 'sequence')) assert(decimal(auth.sequence)); } + } + }; + if (payload.receipt != null) { + if (Array.isArray(payload.receipt)) { + assert.equal(payload.receipt.length, 2); assert.equal(typeof payload.receipt[0], 'string'); receipt(payload.receipt[1]); + } else receipt(payload.receipt); + } + if (Object.hasOwn(payload, 'receipts')) { + assert(Array.isArray(payload.receipts)); + for (const value of payload.receipts) receipt(value); + } +} + +export function validateServerFrame(text) { + const packet = JSON.parse(text, (key, value) => { + assert(key.isWellFormed()); + if (typeof value === 'string') assert(value.isWellFormed()); + return value; + }); + assert(object(packet)); uniqueEnvelope(text); + const scope = () => { for (const key of ['connection_id', 'stream_id', 'generation', 'subscription_id']) assert(positive(packet[key]), key); }; + switch (packet.type) { + case 'hello': assert(Number.isInteger(packet.version) && packet.version >= 0 && packet.version <= 65535); assert(hash(packet.chain_id)); assert(positive(packet.connection_id)); assert(Number.isSafeInteger(packet.heartbeat_interval_ms)); break; + case 'accepted': scope(); assert(positive(packet.request_id)); assert(block(packet.start_from)); break; + case 'data': + scope(); assert(positive(packet.sequence)); assert(['history', 'live'].includes(packet.mode)); assert(Array.isArray(packet.records) && packet.records.length); + for (const record of packet.records) { assert(block(record.block_num)); assert(hash(record.block_id)); if (record.cursor != null) assert(decimal(record.cursor)); metadata(record.payload); } + break; + case 'history_end': case 'end': case 'cancelled': scope(); break; + case 'ping': case 'pong': assert(decimal(packet.id)); break; + case 'error': + assert(positive(packet.connection_id)); assert((packet.stream_id != null) === (packet.generation != null)); + if (packet.stream_id != null) { assert(positive(packet.stream_id)); assert(positive(packet.generation)); } + if (packet.request_id != null) assert(decimal(packet.request_id)); + assert(typeof packet.code === 'string' && packet.code.length > 0 && Buffer.byteLength(packet.code) <= 128); + assert(typeof packet.message === 'string' && Buffer.byteLength(packet.message) <= 4096); assert(typeof packet.retryable === 'boolean'); break; + case 'lib': assert(positive(packet.connection_id)); assert(hash(packet.chain_id)); assert(block(packet.block_num)); assert(hash(packet.block_id)); break; + case 'fork': assert(positive(packet.connection_id)); assert(hash(packet.chain_id)); assert(block(packet.starting_block)); assert(block(packet.ending_block)); assert(hash(packet.new_block_id)); break; + default: assert.fail(`unknown type ${packet.type}`); + } + return packet; +} + +export function runWireVectors() { + const corpus = JSON.parse(readFileSync(new URL('./wire-vectors.json', import.meta.url), 'utf8')); + for (const test of corpus.cases) { + let valid = true; + try { validateServerFrame(test.text); } catch { valid = false; } + assert.equal(valid, test.valid, test.name); + } + return corpus.cases.length; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) console.log(`${runWireVectors()} native JSON/BigInt wire vectors passed`); diff --git a/conformance/v1/scenarios.json b/conformance/v1/scenarios.json new file mode 100644 index 0000000..0f0ec90 --- /dev/null +++ b/conformance/v1/scenarios.json @@ -0,0 +1,3846 @@ +{ + "version": 1, + "cases": [ + { + "name": "head history to live and draining completion", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "1", + "generation": "1", + "sequence": "1" + } + ], + "events": [ + { + "type": "data", + "stream_id": "1", + "generation": "1", + "sequence": "1", + "blocks": [ + 100 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "history_end", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "history_end", + "stream_id": "1" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "2", + "mode": "live", + "records": [ + { + "block_num": 101, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "end", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100" + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "1", + "generation": "1", + "sequence": "2" + } + ], + "events": [ + { + "type": "data", + "stream_id": "1", + "generation": "1", + "sequence": "2", + "blocks": [ + 101 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [ + { + "type": "end", + "stream_id": "1" + } + ] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [] + } + } + ] + }, + { + "name": "LIB cannot release old connection data", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "2" + }, + { + "type": "subscribe", + "stream_id": "1", + "generation": "2", + "resume_from": 100 + } + ], + "events": [ + { + "type": "connected", + "connection_id": "2" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "2", + "stream_id": "1", + "generation": "2", + "subscription_id": "100", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "2", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 200, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "lib", + "block_num": 200 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [] + } + } + ] + }, + { + "name": "repeated outages preserve inclusive recovery", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "2" + }, + { + "type": "subscribe", + "stream_id": "1", + "generation": "2", + "resume_from": 100 + } + ], + "events": [ + { + "type": "connected", + "connection_id": "2" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "2", + "stream_id": "1", + "generation": "2", + "subscription_id": "100", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "3", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "3" + }, + { + "type": "subscribe", + "stream_id": "1", + "generation": "3", + "resume_from": 100 + } + ], + "events": [ + { + "type": "connected", + "connection_id": "3" + } + ] + } + } + ] + }, + { + "name": "replay disabled fails visibly", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": false, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + }, + { + "type": "error", + "stream_id": "1", + "code": "STREAM_REPLAY_REQUIRED", + "retryable": true + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "2" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "2" + } + ] + } + } + ] + }, + { + "name": "sequence gap only fails its own subscription", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "2", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "2", + "generation": "2", + "request_id": "2", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "2", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "2", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "PROTOCOL_ERROR", + "retryable": false + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "2", + "generation": "2", + "sequence": "1" + } + ], + "events": [ + { + "type": "data", + "stream_id": "2", + "generation": "2", + "sequence": "1", + "blocks": [ + 100 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + } + ] + }, + { + "name": "stale server subscription cannot mutate sequence", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "999", + "sequence": "77", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "1", + "generation": "1", + "sequence": "1" + } + ], + "events": [ + { + "type": "data", + "stream_id": "1", + "generation": "1", + "sequence": "1", + "blocks": [ + 100 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + } + ] + }, + { + "name": "record window fails the overflowing stream", + "config": { + "stream_records": 1 + }, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "2", + "mode": "history", + "records": [ + { + "block_num": 101, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "SLOW_CONSUMER", + "retryable": true + } + ] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [] + } + } + ] + }, + { + "name": "complete UTF8 frame bytes count against stream window", + "config": { + "stream_bytes": 128 + }, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "memo": "日本語 🌐日本語 🌐日本語 🌐日本語 🌐" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "SLOW_CONSUMER", + "retryable": true + } + ] + } + } + ] + }, + { + "name": "connection record window also bounds independent streams", + "config": { + "connection_records": 1 + }, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "2", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "2", + "generation": "2", + "request_id": "2", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "2", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "2", + "generation": "2" + } + ], + "events": [ + { + "type": "error", + "stream_id": "2", + "code": "SLOW_CONSUMER", + "retryable": true + } + ] + } + } + ] + }, + { + "name": "cancel pending attempt and ignore late acceptance", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "cancel", + "stream_id": "1", + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1", + "subscription_id": null + } + ], + "events": [ + { + "type": "cancelled", + "stream_id": "1" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [] + } + } + ] + }, + { + "name": "stream ID reuse fences previous generation", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "cancel", + "stream_id": "1", + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "cancelled", + "stream_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "2", + "request_id": "2", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "2", + "subscription_id": "101", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "2", + "subscription_id": "101", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "1", + "generation": "2", + "sequence": "1" + } + ], + "events": [ + { + "type": "data", + "stream_id": "1", + "generation": "2", + "sequence": "1", + "blocks": [ + 100 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + } + ] + }, + { + "name": "live before history_end is rejected", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "live", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "PROTOCOL_ERROR", + "retryable": false + } + ] + } + } + ] + }, + { + "name": "request deadline is finite", + "config": { + "request_timeout_ms": 20 + }, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "advance", + "now_ms": 19, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "advance", + "now_ms": 20, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "REQUEST_TIMEOUT", + "retryable": true + } + ] + } + } + ] + }, + { + "name": "equal LIB does not hide finality timeout", + "config": { + "lib_timeout_ms": 30 + }, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 99, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "lib", + "block_num": 99 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "advance", + "now_ms": 29, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 99, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "ping", + "id": "1" + }, + "expect": { + "commands": [ + { + "type": "pong", + "id": "1" + } + ], + "events": [] + } + }, + { + "op": "advance", + "now_ms": 30, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "LIB_TIMEOUT", + "retryable": true + } + ] + } + } + ] + }, + { + "name": "fork synchronously invalidates buffers before later LIB", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "fork", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "starting_block": 100, + "ending_block": 101, + "new_block_id": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "expect": { + "commands": [ + { + "type": "cancel", + "generation": "1" + }, + { + "type": "subscribe", + "generation": "2", + "resume_from": 100 + } + ], + "events": [ + { + "type": "fork", + "starting_block": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "2", + "subscription_id": "101", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 101, + "block_id": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "lib", + "block_num": 101 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "2", + "subscription_id": "101", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "1", + "generation": "2", + "sequence": "1" + } + ], + "events": [ + { + "type": "data", + "stream_id": "1", + "generation": "2", + "sequence": "1", + "blocks": [ + 100 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + } + ] + }, + { + "name": "scoped source error preserves unrelated stream", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "2", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "2", + "generation": "2", + "request_id": "2", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "2", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "error", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "code": "HISTORY_UNAVAILABLE", + "message": "source unavailable", + "retryable": true + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "HISTORY_UNAVAILABLE", + "retryable": true + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "2", + "generation": "2", + "sequence": "1" + } + ], + "events": [ + { + "type": "data", + "stream_id": "2", + "generation": "2", + "sequence": "1", + "blocks": [ + 100 + ], + "cursors": [ + "18446744073709551615" + ], + "payloads": [ + { + "global_sequence": "18446744073709551615" + } + ] + } + ] + } + } + ] + }, + { + "name": "data before accepted fails instead of silently dropping", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "PROTOCOL_ERROR", + "retryable": false + } + ] + } + } + ] + }, + { + "name": "data cannot precede accepted replay lower bound", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 99, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1", + "generation": "1" + } + ], + "events": [ + { + "type": "error", + "stream_id": "1", + "code": "PROTOCOL_ERROR", + "retryable": false + } + ] + } + } + ] + }, + { + "name": "LIB block identity cannot confirm contradictory data", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 100, + "block_id": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "expect": { + "fatal": "PROTOCOL_ERROR" + } + } + ] + }, + { + "name": "LIB cannot regress across reconnect", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "lib", + "block_num": 100 + } + ] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "2" + }, + { + "type": "subscribe", + "stream_id": "1", + "generation": "2", + "resume_from": 100 + } + ], + "events": [ + { + "type": "connected", + "connection_id": "2" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "2", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 99, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "fatal": "PROTOCOL_ERROR" + } + } + ] + }, + { + "name": "equal LIB identity survives reconnect", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "lib", + "block_num": 100 + } + ] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "2" + }, + { + "type": "subscribe", + "stream_id": "1", + "generation": "2", + "resume_from": 100 + } + ], + "events": [ + { + "type": "connected", + "connection_id": "2" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "2", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 100, + "block_id": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "expect": { + "fatal": "PROTOCOL_ERROR" + } + } + ] + }, + { + "name": "cancel releases both connection budgets", + "config": { + "connection_records": 1 + }, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "2", + "delivery": "irreversible", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "2", + "generation": "2", + "request_id": "2", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "request_id": "2", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "2", + "generation": "2", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "cancel", + "stream_id": "1", + "expect": { + "commands": [ + { + "type": "cancel", + "stream_id": "1" + } + ], + "events": [ + { + "type": "cancelled", + "stream_id": "1" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "data", + "connection_id": "1", + "stream_id": "2", + "generation": "2", + "subscription_id": "101", + "sequence": "1", + "mode": "history", + "records": [ + { + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "cursor": "18446744073709551615", + "payload": { + "global_sequence": "18446744073709551615" + } + } + ] + }, + "expect": { + "commands": [], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "lib", + "connection_id": "1", + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "block_num": 100, + "block_id": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "expect": { + "commands": [], + "events": [ + { + "type": "lib", + "block_num": 100 + } + ] + } + }, + { + "op": "poll", + "expect": { + "commands": [ + { + "type": "ack", + "stream_id": "2" + } + ], + "events": [ + { + "type": "data", + "stream_id": "2" + } + ] + } + } + ] + }, + { + "name": "relay replacement needs a new transport rather than in-band hello", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "fatal": "PROTOCOL_ERROR" + } + } + ] + }, + { + "name": "chain identity remains pinned across reconnect", + "config": {}, + "steps": [ + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "connection_id": "1", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "commands": [ + { + "type": "ready", + "connection_id": "1" + } + ], + "events": [ + { + "type": "connected", + "connection_id": "1" + } + ] + } + }, + { + "op": "subscribe", + "stream_id": "1", + "delivery": "head", + "replay": true, + "request": { + "kind": "action", + "start_from": 100 + }, + "expect": { + "commands": [ + { + "type": "subscribe", + "stream_id": "1", + "generation": "1", + "request_id": "1", + "resume_from": null + } + ], + "events": [] + } + }, + { + "op": "server", + "packet": { + "type": "accepted", + "connection_id": "1", + "stream_id": "1", + "generation": "1", + "subscription_id": "100", + "request_id": "1", + "start_from": 100 + }, + "expect": { + "commands": [], + "events": [ + { + "type": "accepted", + "stream_id": "1", + "generation": "1", + "start_from": 100 + } + ] + } + }, + { + "op": "disconnect", + "expect": { + "commands": [], + "events": [ + { + "type": "disconnected" + } + ] + } + }, + { + "op": "server", + "packet": { + "type": "hello", + "version": 1, + "chain_id": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "connection_id": "2", + "heartbeat_interval_ms": 15000 + }, + "expect": { + "fatal": "CHAIN_MISMATCH" + } + } + ] + } + ] +} diff --git a/conformance/v1/wire-vectors.json b/conformance/v1/wire-vectors.json new file mode 100644 index 0000000..0a6824a --- /dev/null +++ b/conformance/v1/wire-vectors.json @@ -0,0 +1,190 @@ +{ + "version": 1, + "cases": [ + { + "name": "exact uint64 and arbitrary contract JSON", + "valid": true, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "hello", + "valid": true, + "text": "{\"type\":\"hello\",\"version\":1,\"chain_id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"connection_id\":\"1\",\"heartbeat_interval_ms\":15000}" + }, + { + "name": "numeric identifier", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":1,\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "overflowing uint64", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"18446744073709551616\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "leading zero", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"01\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "negative uint64", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"-1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "numeric uint64 metadata", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":9007199254740992,\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "empty data frame", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[]}" + }, + { + "name": "fractional block", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":1.5,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "zero block", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":0,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "invalid block hash", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"fake\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "unknown data mode", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"other\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "zero identifier", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"0\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "compound delta identity", + "valid": true, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"primary_key\":\"account:340282366920938463463374607431768211455\",\"data\":{\"present\":false}}}]}" + }, + { + "name": "receipt precision", + "valid": true, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":[\"receipt\",{\"global_sequence\":\"18446744073709551615\",\"recv_sequence\":\"0\",\"auth_sequence\":[{\"account\":\"eosio\",\"sequence\":\"18446744073709551615\"}]}]}}]}" + }, + { + "name": "numeric receipt precision", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipts\":[{\"recv_sequence\":9007199254740992}]}}]}" + }, + { + "name": "malformed JSON", + "valid": false, + "text": "{\"type\":" + }, + { + "name": "unknown message", + "valid": false, + "text": "{\"type\":\"mystery\"}" + }, + { + "name": "unpaired error scope", + "valid": false, + "text": "{\"type\":\"error\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"code\":\"X\",\"message\":\"x\",\"retryable\":true}" + }, + { + "name": "duplicate top level identity", + "valid": false, + "text": "{\"type\":\"pong\",\"id\":\"1\",\"id\":\"2\"}" + }, + { + "name": "duplicate scope identity", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"stream_id\":\"2\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "binary hint does not make JSON numeric IDs safe", + "valid": false, + "text": "{\"type\":\"pong\",\"id\":18446744073709551615}" + }, + { + "name": "uint64 zero heartbeat nonce", + "valid": true, + "text": "{\"type\":\"pong\",\"id\":\"0\"}" + }, + { + "name": "trailing JSON document", + "valid": false, + "text": "{\"type\":\"pong\",\"id\":\"1\"} {}" + }, + { + "name": "pending-cancelled can allocate a server subscription before accepted", + "valid": true, + "text": "{\"type\":\"cancelled\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"request_id\":\"1\"}" + }, + { + "name": "unpaired Unicode surrogate is not a valid UTF8 string", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"\\u65e5\\u672c\\u8a9e \\ud83c\\udf10\"}},\"memo\":\"\\ud800\"}}]}" + }, + { + "name": "escaped duplicate scope key", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"\\u0067eneration\":\"2\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"global_sequence\":\"18446744073709551615\",\"act\":{\"data\":{\"global_sequence\":42,\"memo\":\"日本語 🌐\"}}}}]}" + }, + { + "name": "receipt null is optional", + "valid": true, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":null}}]}" + }, + { + "name": "grouped receipts must be an array", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipts\":\"invalid\"}}]}" + }, + { + "name": "grouped receipts cannot contain null", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipts\":[null]}}]}" + }, + { + "name": "auth_sequence must be an array", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":{\"auth_sequence\":\"18446744073709551615\"}}}]}" + }, + { + "name": "authorization entries must be objects", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":{\"auth_sequence\":[null]}}}]}" + }, + { + "name": "receipt variant needs two entries", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":[\"receipt\"]}}]}" + }, + { + "name": "receipt variant tag must be a string", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":[1,{}]}}]}" + }, + { + "name": "receipt variant body must be an object", + "valid": false, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipt\":[\"receipt\",null]}}]}" + }, + { + "name": "empty grouped receipts are valid", + "valid": true, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"receipts\":[]}}]}" + }, + { + "name": "arbitrary contract sequence fields stay opaque", + "valid": true, + "text": "{\"type\":\"data\",\"connection_id\":\"1\",\"stream_id\":\"1\",\"generation\":\"1\",\"subscription_id\":\"100\",\"sequence\":\"1\",\"mode\":\"history\",\"records\":[{\"block_num\":100,\"block_id\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"cursor\":\"18446744073709551615\",\"payload\":{\"sequence\":42,\"act\":{\"data\":{\"receipt\":\"contract data\",\"auth_sequence\":\"opaque\",\"global_sequence\":42}}}}]}" + } + ] +} diff --git a/conformance/verify.mjs b/conformance/verify.mjs new file mode 100644 index 0000000..72b686a --- /dev/null +++ b/conformance/verify.mjs @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { runWireVectors } from './v1/node-wire.mjs'; + +const root = new URL('../', import.meta.url); +const manifest = JSON.parse(readFileSync(new URL('./v1/manifest.json', import.meta.url), 'utf8')); +for (const resource of manifest.resources) { + assert.match(resource.path, /^(protocol|conformance)\//); + assert(!resource.path.includes('..')); + const bytes = readFileSync(new URL(resource.path, root)); + assert.equal(createHash('sha256').update(bytes).digest('hex'), resource.sha256, resource.path); +} +if (process.argv.includes('--core')) { + const command = spawnSync('cargo', ['metadata', '--no-deps', '--format-version=1', '--locked'], { cwd: root, encoding: 'utf8' }); + assert.equal(command.status, 0, command.stderr); + const core = JSON.parse(command.stdout).packages.find(pkg => pkg.name === 'hyperion-stream-core'); + assert(core); assert.deepEqual(core.dependencies, [], 'Core must have no third-party, development or build dependencies'); +} +console.log(`${manifest.resources.length} corpus hashes and ${runWireVectors()} native JSON/BigInt vectors passed`); diff --git a/crates/hyperion-stream-core/Cargo.toml b/crates/hyperion-stream-core/Cargo.toml new file mode 100644 index 0000000..e4041ad --- /dev/null +++ b/crates/hyperion-stream-core/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hyperion-stream-core" +description = "Dependency-free Hyperion stream subscription and recovery state machine" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[lints] +workspace = true + diff --git a/crates/hyperion-stream-core/LICENSE b/crates/hyperion-stream-core/LICENSE new file mode 100644 index 0000000..82c3c98 --- /dev/null +++ b/crates/hyperion-stream-core/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 EOS Rio + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/hyperion-stream-core/README.md b/crates/hyperion-stream-core/README.md new file mode 100644 index 0000000..9bed819 --- /dev/null +++ b/crates/hyperion-stream-core/README.md @@ -0,0 +1,21 @@ +# hyperion-stream-core + +Experimental Hyperion native-stream v1 subscription and recovery state machine. +Zero third-party dependencies, including development/build dependencies. Uses +Rust `std`; no networking, serialization, runtime, TLS or wall clock is required. + +Feed typed `Input` values to `Client::receive`, inject monotonic milliseconds +with `advance`, execute returned `Command` values, and call `poll` to accept a +ready batch. `R` and `T` are application-owned request/payload types. Batch ACK +means acceptance, never application durability. Save inclusive checkpoints and +deduplicate if durable at-least-once processing is needed. + +The state machine owns bounded stream/connection buffers, request/connection/ +generation fencing, history/live ordering, cancellation, finite request/finality +deadlines, chain identity, irreversible delivery and inclusive replay. A later +connection cannot release an old reversible buffer. The verified LIB consistency +watermark survives reconnect independently from current delivery eligibility. + +For a complete connection use the separate `hyperion-stream-ws` adapter. Core-only +is not a standalone `wss://` client. Full contract and shared conformance live in +the workspace's `protocol/v1` and `conformance/v1` directories. diff --git a/crates/hyperion-stream-core/src/lib.rs b/crates/hyperion-stream-core/src/lib.rs new file mode 100644 index 0000000..38f4417 --- /dev/null +++ b/crates/hyperion-stream-core/src/lib.rs @@ -0,0 +1,710 @@ +//! Pure subscription state machine. No networking, serialization, clock or runtime dependencies. +#![doc = include_str!("../README.md")] + +mod types; +use std::collections::{BTreeMap, VecDeque}; + +#[derive(Default, Clone, Copy)] +struct Usage { + records: usize, + bytes: usize, +} + +impl Usage { + fn release(&mut self, other: Self) { + self.records -= other.records; + self.bytes -= other.bytes; + } +} +pub use types::*; + +struct Subscription { + request: R, + delivery: Delivery, + replay: bool, + generation: u64, + scope: Option, + started_ms: Option, + resume_from: Option, + anchor: Option, + queue: VecDeque>, + used: Usage, + sequence: u64, + last_block: Option, + history_end: bool, + ended: bool, + lib_progress_ms: u64, +} + +/// Single-owner deterministic state machine. Call `advance` with monotonic time, +/// feed decoded inputs, execute returned commands, and call `poll` for delivery. +pub struct Client { + config: Config, + hello: Option, + chain_id: Option, + next_generation: u64, + streams: BTreeMap>, + used: Usage, + now_ms: u64, + lib: Option, + // Consistency survives reconnect; eligibility to release data does not. + highest_lib: Option, +} + +fn protocol(message: &str) -> Failure { + Failure::new("PROTOCOL_ERROR", message, false) +} + +pub fn is_hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +impl Client { + pub fn new(config: Config) -> Result { + for w in [&config.stream_window, &config.connection_window] { + if w.records == 0 + || w.bytes == 0 + || w.records > 1_000_000 + || w.bytes > 1024 * 1024 * 1024 + { + return Err(Failure::new( + "INVALID_CONFIG", + "Window outside supported bounds", + false, + )); + } + } + if config.max_streams == 0 + || config.max_streams > 1024 + || config.request_timeout_ms == 0 + || config.lib_timeout_ms == 0 + || config + .expected_chain_id + .as_ref() + .is_some_and(|s| !is_hash(s)) + { + return Err(Failure::new( + "INVALID_CONFIG", + "Invalid stream count, deadline or chain ID", + false, + )); + } + Ok(Self { + chain_id: config.expected_chain_id.clone(), + config, + hello: None, + next_generation: 1, + streams: BTreeMap::new(), + used: Usage::default(), + now_ms: 0, + lib: None, + highest_lib: None, + }) + } + + pub fn config(&self) -> &Config { + &self.config + } + pub fn is_connected(&self) -> bool { + self.hello.is_some() + } + pub fn stream_count(&self) -> usize { + self.streams.len() + } + + pub fn subscribe( + &mut self, + stream_id: u64, + request: R, + delivery: Delivery, + replay: bool, + ) -> Result, Failure> { + if stream_id == 0 || self.streams.contains_key(&stream_id) { + return Err(Failure::new( + "INVALID_REQUEST", + "Stream ID must be nonzero and unused", + false, + )); + } + if self.streams.len() >= self.config.max_streams { + return Err(Failure::new( + "STREAM_LIMIT", + "Maximum subscription count reached", + false, + )); + } + let generation = self.allocate_generation()?; + self.streams.insert( + stream_id, + Subscription { + request, + delivery, + replay, + generation, + scope: None, + started_ms: self.hello.as_ref().map(|_| self.now_ms), + resume_from: None, + anchor: None, + queue: VecDeque::new(), + used: Usage::default(), + sequence: 0, + last_block: None, + history_end: false, + ended: false, + lib_progress_ms: self.now_ms, + }, + ); + let mut result = Transition::default(); + if let Some(h) = &self.hello { + result + .commands + .push(self.subscribe_command(stream_id, h.connection_id)); + } + Ok(result) + } + + fn allocate_generation(&mut self) -> Result { + let result = self.next_generation; + self.next_generation = result + .checked_add(1) + .ok_or_else(|| Failure::new("ID_EXHAUSTED", "Generation counter exhausted", false))?; + Ok(result) + } + + fn subscribe_command(&self, stream_id: u64, connection_id: u64) -> Command { + let s = &self.streams[&stream_id]; + Command::Subscribe { + connection_id, + request_id: s.generation, + stream_id, + generation: s.generation, + request: s.request.clone(), + resume_from: s.resume_from, + window: self.config.stream_window.clone(), + } + } + + fn current(&self, scope: &Scope) -> bool { + self.hello + .as_ref() + .is_some_and(|h| h.connection_id == scope.connection_id) + && self + .streams + .get(&scope.stream_id) + .is_some_and(|s| s.scope.as_ref() == Some(scope)) + } + + fn connection_is_current(&self, connection_id: u64) -> bool { + self.hello + .as_ref() + .is_some_and(|h| h.connection_id == connection_id) + } + + fn cancel_command(stream_id: u64, s: &Subscription, connection_id: u64) -> Command { + Command::Cancel { + connection_id, + request_id: s.generation, + stream_id, + generation: s.generation, + subscription_id: s.scope.as_ref().map(|s| s.subscription_id), + } + } + + fn fail_stream(&mut self, stream_id: u64, failure: Failure) -> Transition { + let mut result = Transition::default(); + if let Some(s) = self.remove_stream(stream_id) { + if let Some(h) = &self.hello { + result + .commands + .push(Self::cancel_command(stream_id, &s, h.connection_id)); + } + result + .events + .push(Event::Error(failure.for_stream(stream_id))); + } + result + } + + fn remove_stream(&mut self, stream_id: u64) -> Option> { + let stream = self.streams.remove(&stream_id)?; + self.used.release(stream.used); + Some(stream) + } + + pub fn receive(&mut self, input: Input) -> Result, Failure> { + let mut result = Transition::default(); + match input { + Input::Hello(h) => { + if self.hello.is_some() + || h.version != 1 + || h.connection_id == 0 + || !is_hash(&h.chain_id) + || h.heartbeat_interval_ms == 0 + || h.heartbeat_interval_ms > 60_000 + { + return Err(protocol("Invalid or duplicate hello")); + } + if self.chain_id.as_ref().is_some_and(|id| id != &h.chain_id) { + return Err(Failure::new( + "CHAIN_MISMATCH", + "Server chain identity changed", + false, + )); + } + self.chain_id = Some(h.chain_id.clone()); + result.commands.push(Command::Ready { + connection_id: h.connection_id, + window: self.config.connection_window.clone(), + }); + for s in self.streams.values_mut() { + s.started_ms = Some(self.now_ms); + } + for &id in self.streams.keys() { + result + .commands + .push(self.subscribe_command(id, h.connection_id)); + } + result.events.push(Event::Connected(h.clone())); + self.hello = Some(h); + } + Input::Accepted { + scope, + request_id, + start_from, + } => { + if !self.connection_is_current(scope.connection_id) { + return Ok(result); + } + let Some(s) = self.streams.get_mut(&scope.stream_id) else { + return Ok(result); + }; + if s.generation != scope.generation { + return Ok(result); + } + if scope.subscription_id == 0 + || start_from == 0 + || request_id != s.generation + || s.scope.is_some() + || s.resume_from.is_some_and(|b| b != start_from) + { + return Ok(self.fail_stream( + scope.stream_id, + protocol("Invalid acceptance or changed replay boundary"), + )); + } + s.anchor = Some(start_from); + s.scope = Some(scope.clone()); + s.started_ms = None; + result.events.push(Event::Accepted { scope, start_from }); + } + Input::Data(batch) => { + let id = batch.scope.stream_id; + if self + .hello + .as_ref() + .is_some_and(|h| h.connection_id == batch.scope.connection_id) + && self.streams.get(&id).is_some_and(|s| { + s.generation == batch.scope.generation && s.scope.is_none() + }) + { + return Ok(self.fail_stream(id, protocol("Data arrived before acceptance"))); + } + if !self.current(&batch.scope) { + return Ok(result); + } + let s = &self.streams[&id]; + let valid = !s.ended + && s.sequence.checked_add(1) == Some(batch.sequence) + && !batch.records.is_empty() + && batch.wire_bytes > 0 + && (batch.mode == Mode::Live) == s.history_end + && batch + .records + .iter() + .all(|r| r.block_num > 0 && is_hash(&r.block_id)) + && batch + .records + .windows(2) + .all(|w| w[0].block_num <= w[1].block_num) + && s.last_block.is_none_or(|b| b <= batch.records[0].block_num) + && s.anchor.is_none_or(|b| b <= batch.records[0].block_num) + && self.highest_lib.as_ref().is_none_or(|lib| { + batch + .records + .iter() + .all(|r| r.block_num != lib.block_num || r.block_id == lib.block_id) + }); + if !valid { + return Ok(self.fail_stream( + id, + protocol("Invalid data order, phase, records or sequence"), + )); + } + let fits = |used: usize, added: usize, limit: usize| { + used.checked_add(added).is_some_and(|sum| sum <= limit) + }; + if !fits( + s.used.records, + batch.records.len(), + self.config.stream_window.records, + ) || !fits( + s.used.bytes, + batch.wire_bytes, + self.config.stream_window.bytes, + ) || !fits( + self.used.records, + batch.records.len(), + self.config.connection_window.records, + ) || !fits( + self.used.bytes, + batch.wire_bytes, + self.config.connection_window.bytes, + ) { + return Ok(self.fail_stream( + id, + Failure::new("SLOW_CONSUMER", "Peer exceeded acceptance window", true), + )); + } + let s = self.streams.get_mut(&id).expect("current scope"); + if s.queue.is_empty() { + s.lib_progress_ms = self.now_ms; + } + s.sequence = batch.sequence; + s.last_block = batch.records.last().map(|r| r.block_num); + s.used.records += batch.records.len(); + s.used.bytes += batch.wire_bytes; + self.used.records += batch.records.len(); + self.used.bytes += batch.wire_bytes; + s.queue.push_back(batch); + } + Input::HistoryEnd(scope) => { + if self.current(&scope) { + let s = self + .streams + .get_mut(&scope.stream_id) + .expect("current scope"); + if s.history_end || s.ended { + return Ok(self.fail_stream( + scope.stream_id, + protocol("Duplicate or late history_end"), + )); + } + s.history_end = true; + result.events.push(Event::HistoryEnd { + stream_id: scope.stream_id, + }); + } + } + Input::End(scope) => { + if self.current(&scope) { + let s = self + .streams + .get_mut(&scope.stream_id) + .expect("current scope"); + if !s.history_end || s.ended { + return Ok(self.fail_stream(scope.stream_id, protocol("Invalid end"))); + } + s.ended = true; + } + } + Input::Ping(id) => { + result.commands.push(Command::Pong(id)); + } + Input::Pong(_) | Input::Cancelled(_) => {} + Input::Lib(lib) => { + if !self.connection_is_current(lib.connection_id) { + return Ok(result); + } + if self.chain_id.as_ref() != Some(&lib.chain_id) + || lib.block_num == 0 + || !is_hash(&lib.block_id) + { + return Err(protocol("Invalid LIB identity")); + } + if let Some(previous) = &self.highest_lib { + if lib.block_num < previous.block_num + || (lib.block_num == previous.block_num + && lib.block_id != previous.block_id) + { + return Err(protocol("LIB regressed or changed identity")); + } + if lib.block_num == previous.block_num && self.lib.is_some() { + return Ok(result); + } + } + if self + .streams + .values() + .flat_map(|s| &s.queue) + .flat_map(|b| &b.records) + .any(|r| r.block_num == lib.block_num && r.block_id != lib.block_id) + { + return Err(protocol("LIB contradicts buffered block identity")); + } + for s in self.streams.values_mut() { + s.lib_progress_ms = self.now_ms; + } + self.lib = Some(lib.clone()); + self.highest_lib = Some(lib.clone()); + result.events.push(Event::Lib(lib)); + } + Input::Fork(fork) => { + if !self.connection_is_current(fork.connection_id) { + return Ok(result); + } + if self.chain_id.as_ref() != Some(&fork.chain_id) + || fork.starting_block == 0 + || fork.starting_block > fork.ending_block + || !is_hash(&fork.new_block_id) + || self + .highest_lib + .as_ref() + .is_some_and(|lib| fork.starting_block <= lib.block_num) + { + return Err(protocol("Invalid fork identity or irreversible fork")); + } + result.events.push(Event::Fork(fork.clone())); + let ids: Vec<_> = self.streams.keys().copied().collect(); + for id in ids { + if !self.streams[&id].replay { + let failure = self.fail_stream( + id, + Failure::new( + "STREAM_REPLAY_REQUIRED", + "Fork requires inclusive replay", + true, + ), + ); + result.commands.extend(failure.commands); + result.events.extend(failure.events); + continue; + } + let resume = Self::replay_boundary(&self.streams[&id]) + .unwrap_or(fork.starting_block) + .min(fork.starting_block); + result.commands.push(Self::cancel_command( + id, + &self.streams[&id], + fork.connection_id, + )); + let generation = self.allocate_generation()?; + let mut s = self.remove_stream(id).expect("saved stream"); + Self::reset_attempt(&mut s, generation, Some(resume), self.now_ms); + s.started_ms = Some(self.now_ms); + self.streams.insert(id, s); + result + .commands + .push(self.subscribe_command(id, fork.connection_id)); + } + } + Input::Error { + connection_id, + stream_id, + generation, + request_id, + failure, + } => { + if !self.connection_is_current(connection_id) { + return Ok(result); + } + if stream_id.is_some() != generation.is_some() { + return Err(protocol("Incomplete error scope")); + } + if let Some(id) = stream_id { + if self + .streams + .get(&id) + .is_some_and(|s| Some(s.generation) == generation) + { + if request_id.is_some_and(|r| Some(r) != generation) { + return Ok(self + .fail_stream(id, protocol("Error request correlation mismatch"))); + } + return Ok(self.fail_stream(id, failure)); + } + } else { + return Err(failure); + } + } + } + Ok(result) + } + + fn replay_boundary(s: &Subscription) -> Option { + [ + s.resume_from, + s.anchor, + s.queue + .front() + .and_then(|b| b.records.first().map(|r| r.block_num)), + ] + .into_iter() + .flatten() + .min() + } + + fn reset_attempt( + s: &mut Subscription, + generation: u64, + resume_from: Option, + now_ms: u64, + ) { + s.generation = generation; + s.resume_from = resume_from; + s.scope = None; + s.started_ms = None; + s.queue.clear(); + s.used = Usage::default(); + s.sequence = 0; + s.last_block = None; + s.history_end = false; + s.ended = false; + s.lib_progress_ms = now_ms; + } + + /// Invalidates all transport-owned scopes and unconfirmed payloads. Repeated + /// disconnects preserve the earliest recovery point until new data is handed + /// to the caller. No acknowledgement from a later connection can release an + /// earlier connection's buffered data. + pub fn disconnect(&mut self) -> Result, Failure> { + let mut result = Transition::default(); + if self.hello.take().is_none() { + return Ok(result); + } + self.lib = None; + result.events.push(Event::Disconnected); + let ids: Vec<_> = self.streams.keys().copied().collect(); + for id in ids { + if !self.streams[&id].replay { + self.remove_stream(id); + result.events.push(Event::Error( + Failure::new( + "STREAM_REPLAY_REQUIRED", + "Transport loss requires inclusive replay", + true, + ) + .for_stream(id), + )); + continue; + } + let resume = Self::replay_boundary(&self.streams[&id]); + let generation = self.allocate_generation()?; + let mut s = self.remove_stream(id).expect("saved stream"); + Self::reset_attempt(&mut s, generation, resume, self.now_ms); + self.streams.insert(id, s); + } + Ok(result) + } + + /// Stops local delivery immediately. A returned Cancel command still needs + /// bounded transport I/O; this event is not proof of server cleanup. + pub fn cancel(&mut self, stream_id: u64) -> Transition { + let mut result = Transition::default(); + if let Some(s) = self.remove_stream(stream_id) { + if let Some(h) = &self.hello { + result + .commands + .push(Self::cancel_command(stream_id, &s, h.connection_id)); + } + result.events.push(Event::Cancelled { stream_id }); + } + result + } + + /// Supply monotonic milliseconds from the adapter. No wall-clock or runtime + /// is part of the core. Equal LIB updates and heartbeats do not reset progress. + pub fn advance(&mut self, now_ms: u64) -> Result, Failure> { + if now_ms < self.now_ms { + return Err(Failure::new("INVALID_CLOCK", "Time moved backwards", false)); + } + self.now_ms = now_ms; + let expired: Vec<_> = self + .streams + .iter() + .filter_map(|(&id, s)| { + if s.started_ms.is_some_and(|start| { + now_ms.saturating_sub(start) >= self.config.request_timeout_ms + }) { + return Some(( + id, + Failure::new( + "REQUEST_TIMEOUT", + "Subscription acceptance deadline elapsed", + true, + ), + )); + } + if s.delivery == Delivery::Irreversible + && s.queue.front().is_some_and(|b| { + self.lib.as_ref().is_none_or(|lib| { + b.records + .last() + .is_some_and(|r| r.block_num > lib.block_num) + }) + }) + && now_ms.saturating_sub(s.lib_progress_ms) >= self.config.lib_timeout_ms + { + return Some(( + id, + Failure::new( + "LIB_TIMEOUT", + "Irreversible progress deadline elapsed", + true, + ), + )); + } + None + }) + .collect(); + let mut result = Transition::default(); + for (id, failure) in expired { + let failure = self.fail_stream(id, failure); + result.commands.extend(failure.commands); + result.events.extend(failure.events); + } + Ok(result) + } + + /// Hands one ready batch to the application and releases exactly its credit. + /// Event::End follows the last batch; stalled irreversible streams do not + /// block delivery for other subscriptions. + pub fn poll(&mut self) -> Transition { + let mut result = Transition::default(); + let id = self.streams.iter().find_map(|(&id, s)| { + let ready = s.queue.front().is_some_and(|b| { + s.delivery == Delivery::Head + || self.lib.as_ref().is_some_and(|lib| { + b.records + .last() + .is_some_and(|r| r.block_num <= lib.block_num) + }) + }); + (ready || (s.ended && s.queue.is_empty())).then_some(id) + }); + if let Some(id) = id { + let s = self.streams.get_mut(&id).expect("selected stream"); + if let Some(batch) = s.queue.pop_front() { + let cost = Usage { + records: batch.records.len(), + bytes: batch.wire_bytes, + }; + s.used.release(cost); + self.used.release(cost); + s.anchor = batch.records.last().map(|r| r.block_num); + s.resume_from = None; + result.commands.push(Command::Ack { + scope: batch.scope.clone(), + sequence: batch.sequence, + }); + result.events.push(Event::Data(batch)); + } else { + self.remove_stream(id); + result.events.push(Event::End { stream_id: id }); + } + } + result + } +} diff --git a/crates/hyperion-stream-core/src/types.rs b/crates/hyperion-stream-core/src/types.rs new file mode 100644 index 0000000..39035bf --- /dev/null +++ b/crates/hyperion-stream-core/src/types.rs @@ -0,0 +1,214 @@ +use std::fmt; + +pub const PROTOCOL: &str = "hyperion-stream.v1.json"; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Window { + pub records: usize, + pub bytes: usize, +} + +#[derive(Clone, Debug)] +pub struct Config { + pub stream_window: Window, + pub connection_window: Window, + pub max_streams: usize, + pub request_timeout_ms: u64, + pub lib_timeout_ms: u64, + pub expected_chain_id: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + stream_window: Window { + records: 1024, + bytes: 2 * 1024 * 1024, + }, + connection_window: Window { + records: 4096, + bytes: 8 * 1024 * 1024, + }, + max_streams: 16, + request_timeout_ms: 10_000, + lib_timeout_ms: 60_000, + expected_chain_id: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Failure { + pub code: String, + pub message: String, + pub retryable: bool, + pub stream_id: Option, +} + +impl Failure { + pub fn new(code: impl Into, message: impl Into, retryable: bool) -> Self { + Self { + code: code.into(), + message: message.into(), + retryable, + stream_id: None, + } + } + pub fn for_stream(mut self, stream_id: u64) -> Self { + self.stream_id = Some(stream_id); + self + } +} + +impl fmt::Display for Failure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: {}", self.code, self.message) + } +} +impl std::error::Error for Failure {} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Hello { + pub version: u16, + pub chain_id: String, + pub connection_id: u64, + pub heartbeat_interval_ms: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Scope { + pub connection_id: u64, + pub stream_id: u64, + pub generation: u64, + pub subscription_id: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Delivery { + Head, + Irreversible, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Mode { + History, + Live, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Record { + pub block_num: u32, + pub block_id: String, + pub cursor: Option, + pub payload: T, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Batch { + pub scope: Scope, + pub sequence: u64, + pub mode: Mode, + pub records: Vec>, + /// Measured by the codec from the complete UTF-8 data message, never trusted from the peer. + pub wire_bytes: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Lib { + pub connection_id: u64, + pub chain_id: String, + pub block_num: u32, + pub block_id: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Fork { + pub connection_id: u64, + pub chain_id: String, + pub starting_block: u32, + pub ending_block: u32, + pub new_block_id: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Input { + Hello(Hello), + Accepted { + scope: Scope, + request_id: u64, + start_from: u32, + }, + Data(Batch), + HistoryEnd(Scope), + End(Scope), + Cancelled(Scope), + Error { + connection_id: u64, + stream_id: Option, + generation: Option, + request_id: Option, + failure: Failure, + }, + Lib(Lib), + Fork(Fork), + Ping(u64), + Pong(u64), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Command { + Ready { + connection_id: u64, + window: Window, + }, + Subscribe { + connection_id: u64, + request_id: u64, + stream_id: u64, + generation: u64, + request: R, + resume_from: Option, + window: Window, + }, + Ack { + scope: Scope, + sequence: u64, + }, + Cancel { + connection_id: u64, + request_id: u64, + stream_id: u64, + generation: u64, + subscription_id: Option, + }, + Pong(u64), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Event { + Connected(Hello), + Accepted { scope: Scope, start_from: u32 }, + Data(Batch), + HistoryEnd { stream_id: u64 }, + End { stream_id: u64 }, + Cancelled { stream_id: u64 }, + Error(Failure), + Lib(Lib), + Fork(Fork), + Disconnected, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Transition { + pub commands: Vec>, + pub events: Vec>, +} + +impl Default for Transition { + fn default() -> Self { + Self { + commands: Vec::new(), + events: Vec::new(), + } + } +} diff --git a/crates/hyperion-stream-core/tests/lifecycle.rs b/crates/hyperion-stream-core/tests/lifecycle.rs new file mode 100644 index 0000000..c7c6238 --- /dev/null +++ b/crates/hyperion-stream-core/tests/lifecycle.rs @@ -0,0 +1,290 @@ +use hyperion_stream_core::*; + +fn hello(connection_id: u64) -> Hello { + Hello { + version: 1, + chain_id: "a".repeat(64), + connection_id, + heartbeat_interval_ms: 15_000, + } +} + +#[test] +fn a_consumed_batch_returns_credit_with_exact_cursor_and_scope() { + let mut client = Client::::new(Config::default()).unwrap(); + client.receive(Input::Hello(hello(1))).unwrap(); + let attempt = client + .subscribe(7, "action request".into(), Delivery::Head, true) + .unwrap(); + assert!(matches!( + &attempt.commands[0], + Command::Subscribe { + request_id: 1, + generation: 1, + .. + } + )); + let scope = Scope { + connection_id: 1, + stream_id: 7, + generation: 1, + subscription_id: 100, + }; + client + .receive(Input::Accepted { + scope: scope.clone(), + request_id: 1, + start_from: 100, + }) + .unwrap(); + client + .receive(Input::Data(Batch { + scope: scope.clone(), + sequence: 1, + mode: Mode::History, + records: vec![Record { + block_num: 100, + block_id: "b".repeat(64), + cursor: Some(u64::MAX), + payload: "arbitrary contract data".into(), + }], + wire_bytes: 400, + })) + .unwrap(); + let result = client.poll(); + assert!( + matches!(&result.events[0], Event::Data(batch) if batch.records[0].cursor == Some(u64::MAX)) + ); + assert_eq!(result.commands, vec![Command::Ack { scope, sequence: 1 }]); + assert!(client.poll().events.is_empty()); +} + +#[test] +fn outage_discards_unconfirmed_data_and_repeated_outages_keep_the_earliest_anchor() { + let mut client = Client::::new(Config::default()).unwrap(); + client.receive(Input::Hello(hello(1))).unwrap(); + client + .subscribe(1, "request".into(), Delivery::Irreversible, true) + .unwrap(); + let scope = Scope { + connection_id: 1, + stream_id: 1, + generation: 1, + subscription_id: 100, + }; + client + .receive(Input::Accepted { + scope: scope.clone(), + request_id: 1, + start_from: 100, + }) + .unwrap(); + client + .receive(Input::Data(Batch { + scope: scope.clone(), + sequence: 1, + mode: Mode::History, + records: vec![Record { + block_num: 100, + block_id: "b".repeat(64), + cursor: None, + payload: "old-fork".into(), + }], + wire_bytes: 400, + })) + .unwrap(); + assert!(client.poll().events.is_empty()); + client.disconnect().unwrap(); + let reconnect = client.receive(Input::Hello(hello(2))).unwrap(); + assert!(matches!( + &reconnect.commands[1], + Command::Subscribe { + generation: 2, + resume_from: Some(100), + .. + } + )); + client + .receive(Input::Accepted { + scope: Scope { + connection_id: 2, + generation: 2, + ..scope.clone() + }, + request_id: 2, + start_from: 100, + }) + .unwrap(); + client.disconnect().unwrap(); + let reconnect = client.receive(Input::Hello(hello(3))).unwrap(); + assert!(matches!( + &reconnect.commands[1], + Command::Subscribe { + generation: 3, + resume_from: Some(100), + .. + } + )); + client + .receive(Input::Lib(Lib { + connection_id: 3, + chain_id: "a".repeat(64), + block_num: 200, + block_id: "c".repeat(64), + })) + .unwrap(); + assert!( + client.poll().events.is_empty(), + "a later LIB must not release the old fork payload" + ); + assert!(client.receive(Input::End(scope)).unwrap().events.is_empty()); +} + +#[test] +fn fork_replacement_fences_old_data_and_cancellation_stops_new_delivery() { + let mut client = Client::::new(Config::default()).unwrap(); + client.receive(Input::Hello(hello(1))).unwrap(); + client + .subscribe(1, "request".into(), Delivery::Irreversible, true) + .unwrap(); + let scope = Scope { + connection_id: 1, + stream_id: 1, + generation: 1, + subscription_id: 100, + }; + client + .receive(Input::Accepted { + scope: scope.clone(), + request_id: 1, + start_from: 100, + }) + .unwrap(); + let batch = Batch { + scope: scope.clone(), + sequence: 1, + mode: Mode::History, + records: vec![Record { + block_num: 100, + block_id: "b".repeat(64), + cursor: None, + payload: "old".into(), + }], + wire_bytes: 400, + }; + client.receive(Input::Data(batch.clone())).unwrap(); + let fork = client + .receive(Input::Fork(Fork { + connection_id: 1, + chain_id: "a".repeat(64), + starting_block: 100, + ending_block: 101, + new_block_id: "c".repeat(64), + })) + .unwrap(); + assert!(matches!( + &fork.commands[..], + [ + Command::Cancel { generation: 1, .. }, + Command::Subscribe { + generation: 2, + resume_from: Some(100), + .. + } + ] + )); + client.receive(Input::Data(batch)).unwrap(); + client + .receive(Input::Lib(Lib { + connection_id: 1, + chain_id: "a".repeat(64), + block_num: 101, + block_id: "c".repeat(64), + })) + .unwrap(); + assert!(client.poll().events.is_empty()); + let cancel = client.cancel(1); + assert!(matches!( + &cancel.commands[..], + [Command::Cancel { + subscription_id: None, + generation: 2, + .. + }] + )); + assert_eq!(cancel.events, vec![Event::Cancelled { stream_id: 1 }]); + assert_eq!(client.stream_count(), 0); +} + +#[test] +fn equal_lib_and_heartbeats_do_not_hide_stalled_finality_or_request_acceptance() { + let config = Config { + request_timeout_ms: 20, + lib_timeout_ms: 30, + ..Config::default() + }; + let mut client = Client::::new(config).unwrap(); + client.receive(Input::Hello(hello(1))).unwrap(); + client + .subscribe(1, "pending request".into(), Delivery::Head, true) + .unwrap(); + assert!(client.advance(19).unwrap().events.is_empty()); + let timeout = client.advance(20).unwrap(); + assert!(matches!(&timeout.events[0], Event::Error(e) if e.code == "REQUEST_TIMEOUT")); + client + .subscribe(2, "LIB request".into(), Delivery::Irreversible, true) + .unwrap(); + let scope = Scope { + connection_id: 1, + stream_id: 2, + generation: 2, + subscription_id: 101, + }; + client + .receive(Input::Accepted { + scope: scope.clone(), + request_id: 2, + start_from: 100, + }) + .unwrap(); + client + .receive(Input::Lib(Lib { + connection_id: 1, + chain_id: "a".repeat(64), + block_num: 99, + block_id: "b".repeat(64), + })) + .unwrap(); + client + .receive(Input::Data(Batch { + scope, + sequence: 1, + mode: Mode::History, + records: vec![Record { + block_num: 100, + block_id: "c".repeat(64), + cursor: None, + payload: "waiting".into(), + }], + wire_bytes: 400, + })) + .unwrap(); + client.advance(49).unwrap(); + client.receive(Input::Ping(1)).unwrap(); + client + .receive(Input::Lib(Lib { + connection_id: 1, + chain_id: "a".repeat(64), + block_num: 99, + block_id: "b".repeat(64), + })) + .unwrap(); + let timeout = client.advance(50).unwrap(); + assert!( + matches!(&timeout.events[0], Event::Error(e) if e.code == "LIB_TIMEOUT" && e.retryable) + ); + assert!( + client.advance(49).is_err(), + "injected time must be monotonic" + ); +} diff --git a/crates/hyperion-stream-ws/Cargo.toml b/crates/hyperion-stream-ws/Cargo.toml new file mode 100644 index 0000000..7225078 --- /dev/null +++ b/crates/hyperion-stream-ws/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "hyperion-stream-ws" +description = "Tokio, Tungstenite, JSON and Rustls adapter for Hyperion native streaming" +readme = "README.md" +exclude = ["tests/**"] # Shared conformance runs from the repository workspace. +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +hyperion-stream-core = { path = "../hyperion-stream-core", version = "0.1.0" } +tokio = { version = "1.54.1", default-features = false, features = ["net", "time", "macros", "rt", "sync"] } +tokio-tungstenite = { version = "0.30.0", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } +rustls = { version = "0.23.46", default-features = false, features = ["ring", "std", "tls12"] } +webpki-roots = "1.0" +futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } + +[lints] +workspace = true + diff --git a/crates/hyperion-stream-ws/LICENSE b/crates/hyperion-stream-ws/LICENSE new file mode 100644 index 0000000..82c3c98 --- /dev/null +++ b/crates/hyperion-stream-ws/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 EOS Rio + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/hyperion-stream-ws/README.md b/crates/hyperion-stream-ws/README.md new file mode 100644 index 0000000..22d612e --- /dev/null +++ b/crates/hyperion-stream-ws/README.md @@ -0,0 +1,57 @@ +# hyperion-stream-ws + +Experimental Rust client for Hyperion's proposed additive native WebSocket + JSON +endpoint. Requires a compatible native server; the legacy Socket.IO `/stream` +endpoint uses a different protocol. Server integration is tracked in HAPI #294. + +This adapter explicitly depends on Tokio, tokio-tungstenite, Serde/JSON and Rustls. +The separate `hyperion-stream-core` crate has no third-party dependencies. Default +TLS selects Ring explicitly, uses Mozilla roots, and verifies certificates and +hostnames. Supply `Options::tls` for private CA roots/client certificates and +`Options::headers` for authentication/Origin headers. No TLS verification bypass +is provided by the client. + +```rust,no_run +use hyperion_stream_ws::{WsClient, Options, Request, ActionRequest}; +use hyperion_stream_ws::core::{Delivery, Event}; + +# async fn example() -> Result<(), Box> { +let mut client = WsClient::connect("wss://example.org/stream/v2", Options::default()).await?; +client.subscribe(1, Request::Action(ActionRequest { + contract: Some("eosio.token".into()), + action: Some("transfer".into()), + ..ActionRequest::default() +}), Delivery::Irreversible, true).await?; +loop { + match client.next_event().await? { + Event::Data(batch) => println!("{:?}", batch.records), + Event::Error(error) => return Err(error.into()), + Event::End { .. } => break, + _ => {} + } +} +client.cancel(1).await?; +# Ok(()) +# } +``` + +The client is single-owner and pull-driven: `next_event` drives reads, heartbeat +and deadlines. For long application work, poll it in an application-owned task +and pass batches through a bounded channel. No detached tasks survive drop. A +pending read can be cancelled and polled again. If an I/O-producing operation +(subscribe/cancel/reconnect/ACK write) is interrupted, its network outcome may be +ambiguous; reconnect explicitly before continuing that connection. + +The adapter returns one batch at a time; its acceptance ACK is flushed on the next +drive call, after the caller received it. Callers retain their own durable +checkpoint and deduplicate inclusive replay. `cancel` stops local delivery first; +success does not claim the remote cleanup acknowledgement has arrived. `reconnect` +is explicit and bounded, retains terminal stream errors, and refreshes acceptance +deadlines after network downtime. There is no infinite automatic retry policy. + +Defaults: 1 MiB message cap, 5s connect/write deadlines, 45s idle deadline, 64 pending +public events plus a fixed disconnect-outcome reserve. Core defaults: 16 streams, +1024 records/2 MiB per stream, 4096 records/8 MiB per connection, 10s acceptance and +60s irreversible-progress deadlines. Overflow fails visibly; data/control state +does not grow without a bound. Processing-time/real source/proxy/fanout acceptance +and release publication remain separate qualifications. diff --git a/crates/hyperion-stream-ws/src/client.rs b/crates/hyperion-stream-ws/src/client.rs new file mode 100644 index 0000000..5340393 --- /dev/null +++ b/crates/hyperion-stream-ws/src/client.rs @@ -0,0 +1,476 @@ +use crate::{codec, Request}; +use futures_util::{SinkExt, StreamExt}; +use hyperion_stream_core::{ + Client, Command, Config, Delivery, Event, Failure, Hello, Input, Transition, PROTOCOL, +}; +use serde_json::Value; +use std::{collections::VecDeque, sync::Arc, time::Duration}; +use tokio::{ + net::TcpStream, + time::{timeout, Instant}, +}; +use tokio_tungstenite::{ + connect_async_tls_with_config, + tungstenite::{ + client::IntoClientRequest, + http::{HeaderMap, HeaderValue}, + protocol::WebSocketConfig, + Message, + }, + Connector, MaybeTlsStream, WebSocketStream, +}; + +type Socket = WebSocketStream>; + +/// Networking dependencies are explicit and confined to this adapter. `tls` +/// permits caller-supplied CA roots/client certificates; the default uses Mozilla +/// roots and an explicitly selected Ring provider, with certificate verification. +#[derive(Clone)] +pub struct Options { + pub core: Config, + pub max_message_bytes: usize, + pub connect_timeout: Duration, + pub write_timeout: Duration, + pub idle_timeout: Duration, + pub max_pending_events: usize, + pub headers: HeaderMap, + pub tls: Option>, +} + +impl Default for Options { + fn default() -> Self { + Self { + core: Config::default(), + max_message_bytes: 1024 * 1024, + connect_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + idle_timeout: Duration::from_secs(45), + max_pending_events: 64, + headers: HeaderMap::new(), + tls: None, + } + } +} + +fn transport_error(error: impl std::fmt::Display) -> Failure { + Failure::new("TRANSPORT_ERROR", error.to_string(), true) +} + +fn socket_failure(error: tokio_tungstenite::tungstenite::Error) -> Failure { + use tokio_tungstenite::tungstenite::{error::ProtocolError, Error}; + let (code, retryable) = match &error { + Error::Http(response) => ( + "HTTP_ERROR", + response.status().is_server_error() || matches!(response.status().as_u16(), 408 | 429), + ), + Error::Tls(_) => ("TLS_ERROR", false), + Error::Io(e) if e.kind() == std::io::ErrorKind::InvalidData => { + ("TLS_OR_PROTOCOL_ERROR", false) + } + Error::Capacity(_) => ("MESSAGE_TOO_LARGE", false), + Error::Protocol(ProtocolError::ResetWithoutClosingHandshake) => ("TRANSPORT_ERROR", true), + Error::Protocol(_) | Error::Utf8(_) | Error::AttackAttempt => ("PROTOCOL_ERROR", false), + Error::Url(_) | Error::HttpFormat(_) => ("INVALID_ENDPOINT", false), + _ => ("TRANSPORT_ERROR", true), + }; + Failure::new(code, error.to_string(), retryable) +} + +fn default_tls() -> Result, Failure> { + let roots = rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let config = rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .map_err(transport_error)? + .with_root_certificates(roots) + .with_no_client_auth(); + Ok(Arc::new(config)) +} + +async fn open(endpoint: &str, options: &Options) -> Result<(Socket, Hello), Failure> { + timeout(options.connect_timeout, async { + let mut request = endpoint.into_client_request().map_err(socket_failure)?; + for (key, value) in &options.headers { + request.headers_mut().insert(key, value.clone()); + } + request + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(PROTOCOL)); + let config = WebSocketConfig::default() + .max_message_size(Some(options.max_message_bytes)) + .max_frame_size(Some(options.max_message_bytes)) + .write_buffer_size(0) + .max_write_buffer_size(options.max_message_bytes + 1024); + let tls = match &options.tls { + Some(tls) => tls.clone(), + None => default_tls()?, + }; + let (mut socket, response) = connect_async_tls_with_config( + request, + Some(config), + false, + Some(Connector::Rustls(tls)), + ) + .await + .map_err(socket_failure)?; + if response + .headers() + .get("sec-websocket-protocol") + .and_then(|h| h.to_str().ok()) + != Some(PROTOCOL) + { + return Err(Failure::new( + "PROTOCOL_ERROR", + "Server did not select the required subprotocol", + false, + )); + } + let message = socket + .next() + .await + .ok_or_else(|| transport_error("Closed before hello"))? + .map_err(socket_failure)?; + let Message::Text(text) = message else { + return Err(Failure::new( + "PROTOCOL_ERROR", + "First message must be JSON hello", + false, + )); + }; + let Input::Hello(hello) = codec::decode(&text)? else { + return Err(Failure::new( + "PROTOCOL_ERROR", + "First message must be hello", + false, + )); + }; + Ok((socket, hello)) + }) + .await + .map_err(|_| Failure::new("CONNECT_TIMEOUT", "Connection/hello deadline elapsed", true))? +} + +/// Single-owner, pull-driven native client. `next_event` drives reads, liveness +/// and deadlines; no detached tasks outlive the client. Long application work +/// should keep polling in an application-owned task with a bounded channel. +pub struct WsClient { + endpoint: String, + options: Options, + core: Client, + socket: Option, + pending: VecDeque>, + deferred: VecDeque>, + clock: Instant, + last_received: Instant, + last_ping: Instant, + ping_id: u64, + heartbeat: Duration, + control_epoch: Instant, + control_count: u32, +} + +impl WsClient { + pub async fn connect(endpoint: &str, options: Options) -> Result { + let mut core = Client::new(options.core.clone())?; + if !(128..=1024 * 1024).contains(&options.max_message_bytes) + || options.connect_timeout.is_zero() + || options.write_timeout.is_zero() + || options.idle_timeout.is_zero() + || options.max_pending_events < options.core.max_streams + 2 + || options.max_pending_events > 4096 + { + return Err(Failure::new( + "INVALID_CONFIG", + "Invalid message cap or network deadline", + false, + )); + } + let (socket, hello) = open(endpoint, &options).await?; + let heartbeat = Duration::from_millis(hello.heartbeat_interval_ms); + let transition = core.receive(Input::Hello(hello))?; + let now = Instant::now(); + let mut client = Self { + endpoint: endpoint.into(), + options, + core, + socket: Some(socket), + pending: VecDeque::new(), + deferred: VecDeque::new(), + clock: now, + last_received: now, + last_ping: now, + ping_id: 0, + heartbeat, + control_epoch: now, + control_count: 0, + }; + client.apply(transition).await?; + Ok(client) + } + + pub fn is_connected(&self) -> bool { + self.socket.is_some() && self.core.is_connected() + } + + fn detach(&mut self) -> Result<(), Failure> { + self.socket = None; + self.deferred.clear(); + // Preserve terminal outcomes even when replaced transport notifications + // become obsolete. A fixed reserve holds disconnect outcomes once; an + // additional detach cannot add them again because the core is offline. + self.pending.retain(|e| matches!(e, Event::Error(_))); + let transition = self.core.disconnect()?; + self.pending.extend(transition.events); + Ok(()) + } + + async fn broken(&mut self, error: Failure) -> Failure { + if let Err(failure) = self.detach() { + return failure; + } + error + } + + async fn send_text(&mut self, text: String) -> Result<(), Failure> { + if text.len() > self.options.max_message_bytes { + return Err(Failure::new( + "MESSAGE_TOO_LARGE", + "Outbound message exceeds configured cap", + false, + )); + } + let socket = self + .socket + .as_mut() + .ok_or_else(|| transport_error("Disconnected; call reconnect"))?; + let sent = timeout(self.options.write_timeout, socket.send(Message::text(text))).await; + let error = match sent { + Ok(Ok(())) => return Ok(()), + Ok(Err(error)) => socket_failure(error), + Err(_) => Failure::new("WRITE_TIMEOUT", "WebSocket write deadline elapsed", true), + }; + Err(self.broken(error).await) + } + + async fn flush_deferred(&mut self) -> Result<(), Failure> { + while let Some(command) = self.deferred.front() { + let text = codec::encode(command)?; + self.send_text(text).await?; + self.deferred.pop_front(); + } + Ok(()) + } + + async fn apply(&mut self, transition: Transition) -> Result<(), Failure> { + if !transition.events.is_empty() + && self.pending.len().saturating_add(transition.events.len()) + > self.options.max_pending_events + { + // Terminal errors already removed their streams in the core. Keep + // those outcomes in the reserve; each reduces the remaining active + // stream disconnect obligations one-for-one. + self.pending.extend( + transition + .events + .into_iter() + .filter(|e| matches!(e, Event::Error(_))), + ); + return Err(self + .broken(Failure::new( + "EVENT_QUEUE_OVERFLOW", + "Poll events before issuing more controls", + true, + )) + .await); + } + self.pending.extend(transition.events); + self.deferred.extend(transition.commands); + self.flush_deferred().await + } + + async fn advance(&mut self) -> Result<(), Failure> { + let elapsed = u64::try_from(self.clock.elapsed().as_millis()).unwrap_or(u64::MAX); + let transition = self.core.advance(elapsed)?; + self.apply(transition).await + } + + pub async fn subscribe( + &mut self, + stream_id: u64, + request: Request, + delivery: Delivery, + replay_on_reconnect: bool, + ) -> Result<(), Failure> { + request.validate()?; + let request_bytes = serde_json::to_vec(&request).map_err(transport_error)?.len(); + if request_bytes.saturating_add(512) > self.options.max_message_bytes { + return Err(Failure::new( + "MESSAGE_TOO_LARGE", + "Subscription request exceeds configured cap", + false, + )); + } + self.advance().await?; + // Admission must also be bounded while offline: an empty subscribe + // transition can create a future terminal error obligation. Otherwise + // repeated failed reconnects could reuse the disconnect reserve forever. + if self.pending.len() >= self.options.max_pending_events { + return Err(Failure::new( + "EVENT_QUEUE_OVERFLOW", + "Poll terminal outcomes before creating another subscription", + true, + )); + } + let transition = self + .core + .subscribe(stream_id, request, delivery, replay_on_reconnect)?; + self.apply(transition).await + } + + /// Stop local delivery first. Flush previously delivered batch acceptance + /// before cancellation; remote cleanup is a separate server responsibility. + pub async fn cancel(&mut self, stream_id: u64) -> Result<(), Failure> { + let transition = self.core.cancel(stream_id); + self.pending + .retain(|e| !matches!(e, Event::Data(b) if b.scope.stream_id == stream_id)); + self.apply(transition).await + } + + /// Explicit bounded reconnect, preserving the core's inclusive replay point. + /// Authentication, source and protocol failures are never infinitely retried. + pub async fn reconnect(&mut self) -> Result<(), Failure> { + self.detach()?; + self.pending.retain(|e| matches!(e, Event::Error(_))); + let (socket, hello) = open(&self.endpoint, &self.options).await?; + // Offline/handshake time is elapsed before a replacement attempt starts. + // It must not consume that new attempt's acceptance deadline. + let elapsed = u64::try_from(self.clock.elapsed().as_millis()).unwrap_or(u64::MAX); + let transition = self.core.advance(elapsed)?; + self.pending.extend(transition.events); + let transition = self.core.receive(Input::Hello(hello.clone()))?; + self.socket = Some(socket); + self.heartbeat = Duration::from_millis(hello.heartbeat_interval_ms); + self.last_received = Instant::now(); + self.last_ping = self.last_received; + self.control_epoch = self.last_received; + self.control_count = 0; + self.apply(transition).await + } + + /// Delivers one public event. Batch credit is sent on the next drive call, + /// after this batch has actually been returned, so an ACK write failure + /// cannot discard a never-delivered batch while advancing the replay point. + pub async fn next_event(&mut self) -> Result, Failure> { + self.flush_deferred().await?; + loop { + self.advance().await?; + if let Some(event) = self.pending.pop_front() { + return Ok(event); + } + let transition = self.core.poll(); + if !transition.events.is_empty() { + self.deferred.extend(transition.commands); + self.pending.extend(transition.events); + return Ok(self.pending.pop_front().expect("nonempty transition")); + } + let tick_ms = self + .options + .core + .request_timeout_ms + .min(self.options.core.lib_timeout_ms) + .min(self.heartbeat.as_millis() as u64) + .clamp(2, 200) + / 2; + let socket = self + .socket + .as_mut() + .ok_or_else(|| transport_error("Disconnected; call reconnect"))?; + let received = tokio::select! { + packet = socket.next() => Some(packet), + _ = tokio::time::sleep(Duration::from_millis(tick_ms)) => None, + }; + if let Some(packet) = received { + let message = match packet { + Some(Ok(message)) => message, + Some(Err(error)) => return Err(self.broken(socket_failure(error)).await), + None => return Err(self.broken(transport_error("WebSocket closed")).await), + }; + self.last_received = Instant::now(); + match message { + Message::Text(text) => { + let input = match codec::decode(&text) { + Ok(input) => input, + Err(error) => return Err(self.broken(error).await), + }; + if !matches!(&input, Input::Data(_)) { + if self.control_epoch.elapsed() >= Duration::from_secs(1) { + self.control_epoch = Instant::now(); + self.control_count = 0; + } + self.control_count += 1; + if self.control_count > 1024 { + return Err(self + .broken(Failure::new( + "CONTROL_LIMIT", + "Control message rate exceeded", + false, + )) + .await); + } + } + let transition = match self.core.receive(input) { + Ok(transition) => transition, + Err(error) => return Err(self.broken(error).await), + }; + self.apply(transition).await?; + } + Message::Ping(_) => { + let result = timeout( + self.options.write_timeout, + self.socket.as_mut().expect("connected").flush(), + ) + .await; + if !matches!(result, Ok(Ok(()))) { + return Err(self + .broken(transport_error("WebSocket pong write failed")) + .await); + } + } + Message::Pong(_) => {} + Message::Close(_) => { + return Err(self.broken(transport_error("Peer closed WebSocket")).await) + } + Message::Binary(_) | Message::Frame(_) => { + return Err(self + .broken(Failure::new( + "PROTOCOL_ERROR", + "Only JSON text messages are supported", + false, + )) + .await) + } + } + } + if self.last_received.elapsed() >= self.options.idle_timeout { + return Err(self + .broken(Failure::new( + "IDLE_TIMEOUT", + "Peer liveness deadline elapsed", + true, + )) + .await); + } + if self.last_ping.elapsed() >= self.heartbeat { + self.ping_id = self.ping_id.checked_add(1).ok_or_else(|| { + Failure::new("ID_EXHAUSTED", "Heartbeat counter exhausted", false) + })?; + self.send_text( + serde_json::json!({"type":"ping","id":self.ping_id.to_string()}).to_string(), + ) + .await?; + self.last_ping = Instant::now(); + } + } + } +} diff --git a/crates/hyperion-stream-ws/src/codec.rs b/crates/hyperion-stream-ws/src/codec.rs new file mode 100644 index 0000000..801ba2e --- /dev/null +++ b/crates/hyperion-stream-ws/src/codec.rs @@ -0,0 +1,352 @@ +//! JSON is isolated here: the core's public API has no Serde or JSON types. +use hyperion_stream_core::*; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{json, Value}; + +#[derive(Clone, Copy, Debug)] +struct Decimal(u64); + +impl<'de> Deserialize<'de> for Decimal { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + parse_decimal(&s) + .map(Self) + .ok_or_else(|| serde::de::Error::custom("Expected canonical uint64 decimal string")) + } +} + +pub fn parse_decimal(s: &str) -> Option { + if s.is_empty() + || s.len() > 20 + || (s.len() > 1 && s.starts_with('0')) + || !s.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + s.parse().ok() +} + +#[derive(Deserialize)] +struct WireScope { + connection_id: Decimal, + stream_id: Decimal, + generation: Decimal, + subscription_id: Decimal, +} + +impl WireScope { + fn into_scope(self) -> Result { + if [ + self.connection_id.0, + self.stream_id.0, + self.generation.0, + self.subscription_id.0, + ] + .contains(&0) + { + return Err(invalid("Scope identifiers must be nonzero")); + } + Ok(Scope { + connection_id: self.connection_id.0, + stream_id: self.stream_id.0, + generation: self.generation.0, + subscription_id: self.subscription_id.0, + }) + } +} + +#[derive(Deserialize)] +struct WireRecord { + block_num: u32, + block_id: String, + #[serde(default)] + cursor: Option, + payload: Value, +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Packet { + Hello { + version: u16, + chain_id: String, + connection_id: Decimal, + heartbeat_interval_ms: u64, + }, + Accepted { + #[serde(flatten)] + scope: WireScope, + request_id: Decimal, + start_from: u32, + }, + Data { + #[serde(flatten)] + scope: WireScope, + sequence: Decimal, + mode: String, + records: Vec, + }, + HistoryEnd { + #[serde(flatten)] + scope: WireScope, + }, + End { + #[serde(flatten)] + scope: WireScope, + }, + Cancelled { + #[serde(flatten)] + scope: WireScope, + }, + Error { + connection_id: Decimal, + stream_id: Option, + generation: Option, + request_id: Option, + code: String, + message: String, + retryable: bool, + }, + Lib { + connection_id: Decimal, + chain_id: String, + block_num: u32, + block_id: String, + }, + Fork { + connection_id: Decimal, + chain_id: String, + starting_block: u32, + ending_block: u32, + new_block_id: String, + }, + Ping { + id: Decimal, + }, + Pong { + id: Decimal, + }, +} + +fn invalid(message: impl Into) -> Failure { + Failure::new("PROTOCOL_ERROR", message, false) +} + +fn exact_fields(value: &Value, fields: &[&str]) -> Result<(), Failure> { + let Some(object) = value.as_object() else { + return Err(invalid("Known sequence metadata must be an object")); + }; + for key in fields { + if let Some(v) = object.get(*key) { + if v.as_str().and_then(parse_decimal).is_none() { + return Err(invalid(format!("{key} must be an exact decimal string"))); + } + } + } + Ok(()) +} + +fn exact_receipt(receipt: &Value) -> Result<(), Failure> { + exact_fields(receipt, &["global_sequence", "recv_sequence"])?; + if let Some(value) = receipt.get("auth_sequence") { + let authorizations = value + .as_array() + .ok_or_else(|| invalid("auth_sequence must be an array"))?; + for authorization in authorizations { + exact_fields(authorization, &["sequence"])?; + } + } + Ok(()) +} + +fn exact_metadata(payload: &Value) -> Result<(), Failure> { + exact_fields(payload, &["global_sequence"])?; + if let Some(receipt) = payload.get("receipt").filter(|v| !v.is_null()) { + if let Some(variant) = receipt.as_array() { + if variant.len() != 2 || !variant[0].is_string() { + return Err(invalid("Invalid SHIP receipt variant")); + } + exact_receipt(&variant[1])?; + } else { + exact_receipt(receipt)?; + } + } + if let Some(value) = payload.get("receipts") { + let receipts = value + .as_array() + .ok_or_else(|| invalid("receipts must be an array"))?; + for receipt in receipts { + exact_receipt(receipt)?; + } + } + Ok(()) +} + +/// Decode exactly one server text message. The caller bounds text before parsing. +/// Unknown fields are reserved extensions; unknown packet types fail closed. +pub fn decode(text: &str) -> Result, Failure> { + let packet: Packet = + serde_json::from_str(text).map_err(|e| invalid(format!("Invalid JSON packet: {e}")))?; + Ok(match packet { + Packet::Hello { + version, + chain_id, + connection_id, + heartbeat_interval_ms, + } => Input::Hello(Hello { + version, + chain_id, + connection_id: connection_id.0, + heartbeat_interval_ms, + }), + Packet::Accepted { + scope, + request_id, + start_from, + } => Input::Accepted { + scope: scope.into_scope()?, + request_id: request_id.0, + start_from, + }, + Packet::Data { + scope, + sequence, + mode, + records, + } => { + let mode = match mode.as_str() { + "history" => Mode::History, + "live" => Mode::Live, + _ => return Err(invalid("Unknown data mode")), + }; + let mut output = Vec::with_capacity(records.len()); + for record in records { + if record.block_num == 0 || !is_hash(&record.block_id) { + return Err(invalid("Invalid record block identity")); + } + exact_metadata(&record.payload)?; + output.push(Record { + block_num: record.block_num, + block_id: record.block_id, + cursor: record.cursor.map(|d| d.0), + payload: record.payload, + }); + } + if sequence.0 == 0 || output.is_empty() { + return Err(invalid( + "Data needs a positive sequence and nonempty records", + )); + } + Input::Data(Batch { + scope: scope.into_scope()?, + sequence: sequence.0, + mode, + records: output, + wire_bytes: text.len(), + }) + } + Packet::HistoryEnd { scope } => Input::HistoryEnd(scope.into_scope()?), + Packet::End { scope } => Input::End(scope.into_scope()?), + Packet::Cancelled { scope } => Input::Cancelled(scope.into_scope()?), + Packet::Error { + connection_id, + stream_id, + generation, + request_id, + code, + message, + retryable, + } => { + if stream_id.is_some() != generation.is_some() + || code.is_empty() + || code.len() > 128 + || message.len() > 4096 + || connection_id.0 == 0 + || stream_id.is_some_and(|d| d.0 == 0) + || generation.is_some_and(|d| d.0 == 0) + { + return Err(invalid("Invalid error scope or description")); + } + Input::Error { + connection_id: connection_id.0, + stream_id: stream_id.map(|d| d.0), + generation: generation.map(|d| d.0), + request_id: request_id.map(|d| d.0), + failure: Failure::new(code, message, retryable), + } + } + Packet::Lib { + connection_id, + chain_id, + block_num, + block_id, + } => Input::Lib(Lib { + connection_id: connection_id.0, + chain_id, + block_num, + block_id, + }), + Packet::Fork { + connection_id, + chain_id, + starting_block, + ending_block, + new_block_id, + } => Input::Fork(Fork { + connection_id: connection_id.0, + chain_id, + starting_block, + ending_block, + new_block_id, + }), + Packet::Ping { id } => Input::Ping(id.0), + Packet::Pong { id } => Input::Pong(id.0), + }) +} + +pub fn scope_json(s: &Scope) -> Value { + json!({"connection_id":s.connection_id.to_string(),"stream_id":s.stream_id.to_string(),"generation":s.generation.to_string(),"subscription_id":s.subscription_id.to_string()}) +} + +pub fn command_json(command: &Command) -> Result { + let window = |w: &Window| json!({"records":w.records,"bytes":w.bytes}); + let value = match command { + Command::Ready { + connection_id, + window: w, + } => json!({"type":"ready","connection_id":connection_id.to_string(),"window":window(w)}), + Command::Subscribe { + connection_id, + request_id, + stream_id, + generation, + request, + resume_from, + window: w, + } => { + let request = serde_json::to_value(request).map_err(|e| invalid(e.to_string()))?; + json!({"type":"subscribe","connection_id":connection_id.to_string(),"request_id":request_id.to_string(),"stream_id":stream_id.to_string(),"generation":generation.to_string(),"request":request,"resume_from":resume_from,"window":window(w)}) + } + Command::Ack { scope, sequence } => { + let mut v = scope_json(scope); + v["type"] = json!("ack"); + v["sequence"] = json!(sequence.to_string()); + v + } + Command::Cancel { + connection_id, + request_id, + stream_id, + generation, + subscription_id, + } => { + json!({"type":"cancel","connection_id":connection_id.to_string(),"request_id":request_id.to_string(),"stream_id":stream_id.to_string(),"generation":generation.to_string(),"subscription_id":subscription_id.map(|d| d.to_string())}) + } + Command::Pong(id) => json!({"type":"pong","id":id.to_string()}), + }; + Ok(value) +} + +pub fn encode(command: &Command) -> Result { + serde_json::to_string(&command_json(command)?).map_err(|e| invalid(e.to_string())) +} diff --git a/crates/hyperion-stream-ws/src/lib.rs b/crates/hyperion-stream-ws/src/lib.rs new file mode 100644 index 0000000..bd44518 --- /dev/null +++ b/crates/hyperion-stream-ws/src/lib.rs @@ -0,0 +1,9 @@ +//! Optional native WebSocket adapter. See protocol/v1 for the experimental wire contract. +#![doc = include_str!("../README.md")] + +pub mod codec; +mod request; +pub use request::*; +mod client; +pub use client::{Options, WsClient}; +pub use hyperion_stream_core as core; diff --git a/crates/hyperion-stream-ws/src/request.rs b/crates/hyperion-stream-ws/src/request.rs new file mode 100644 index 0000000..3ede47c --- /dev/null +++ b/crates/hyperion-stream-ws/src/request.rs @@ -0,0 +1,132 @@ +use hyperion_stream_core::Failure; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Bound { + Block(i64), + Timestamp(String), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FilterOperator { + Eq, + Ne, + Gt, + Lt, + Gte, + Lte, + Contains, + StartsWith, + EndsWith, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum FilterValue { + String(String), + Number(serde_json::Number), + Bool(bool), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Filter { + pub field: String, + pub value: FilterValue, + #[serde(skip_serializing_if = "Option::is_none")] + pub operator: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub asset: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FilterOp { + And, + Or, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Query { + #[serde(skip_serializing_if = "Option::is_none")] + pub start_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub read_until: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub ignore_live: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub filter_op: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub filters: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ActionRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contract: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub action: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub account: Option, + #[serde(flatten)] + pub query: Query, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct DeltaRequest { + pub code: String, + pub table: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub scope: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub payer: Option, + #[serde(flatten)] + pub query: Query, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Request { + Action(ActionRequest), + Delta(DeltaRequest), +} + +impl Request { + /// Server validation remains authoritative for selectors, ABI and filter support. + pub fn validate(&self) -> Result<(), Failure> { + let query = match self { + Self::Action(a) => &a.query, + Self::Delta(d) => &d.query, + }; + let bad_bound = |bound: &Option| match bound { + Some(Bound::Block(n)) => *n < -(u32::MAX as i64) || *n > u32::MAX as i64, + Some(Bound::Timestamp(s)) => s.is_empty() || s.len() > 64, + None => false, + }; + if bad_bound(&query.start_from) + || bad_bound(&query.read_until) + || query.filters.len() > 256 + || query + .filters + .iter() + .any(|f| f.field.is_empty() || f.field.len() > 256) + { + return Err(Failure::new( + "INVALID_REQUEST", + "Invalid query bound or filters", + false, + )); + } + if let Self::Delta(d) = self { + if d.code.is_empty() || d.table.is_empty() { + return Err(Failure::new( + "INVALID_REQUEST", + "Delta code/table are required", + false, + )); + } + } + Ok(()) + } +} diff --git a/crates/hyperion-stream-ws/tests/certs/README.md b/crates/hyperion-stream-ws/tests/certs/README.md new file mode 100644 index 0000000..173b9ab --- /dev/null +++ b/crates/hyperion-stream-ws/tests/certs/README.md @@ -0,0 +1,5 @@ +These DER files are public, local-only conformance fixtures. The server key is a +test key, never a production identity. The CA signing key was not persisted. +The ECDSA P-256 server certificate is signed by the fixture CA, valid 2020–2040, +and has only the DNS SAN `localhost`. Tests exercise trust and hostname failures +without disabling certificate verification or requiring an external TLS service. diff --git a/crates/hyperion-stream-ws/tests/certs/ca.der b/crates/hyperion-stream-ws/tests/certs/ca.der new file mode 100644 index 0000000..09f5f80 Binary files /dev/null and b/crates/hyperion-stream-ws/tests/certs/ca.der differ diff --git a/crates/hyperion-stream-ws/tests/certs/server-key.der b/crates/hyperion-stream-ws/tests/certs/server-key.der new file mode 100644 index 0000000..9ae7dce Binary files /dev/null and b/crates/hyperion-stream-ws/tests/certs/server-key.der differ diff --git a/crates/hyperion-stream-ws/tests/certs/server.der b/crates/hyperion-stream-ws/tests/certs/server.der new file mode 100644 index 0000000..3ac36eb Binary files /dev/null and b/crates/hyperion-stream-ws/tests/certs/server.der differ diff --git a/crates/hyperion-stream-ws/tests/codec.rs b/crates/hyperion-stream-ws/tests/codec.rs new file mode 100644 index 0000000..2266b7e --- /dev/null +++ b/crates/hyperion-stream-ws/tests/codec.rs @@ -0,0 +1,43 @@ +use hyperion_stream_core::{Command, Input, Scope}; +use hyperion_stream_ws::codec::{decode, encode}; + +#[test] +fn shared_cross_language_wire_vectors() { + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../../../conformance/v1/wire-vectors.json")).unwrap(); + for case in corpus["cases"].as_array().unwrap() { + let result = decode(case["text"].as_str().unwrap()); + assert_eq!( + result.is_ok(), + case["valid"].as_bool().unwrap(), + "{}: {:?}", + case["name"], + result + ); + } +} + +#[test] +fn exact_uint64_and_arbitrary_contract_json_survive_the_wire_boundary() { + let text = r#"{"type":"data","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100","sequence":"18446744073709551615","mode":"history","records":[{"block_num":100,"block_id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","cursor":"18446744073709551615","payload":{"global_sequence":"18446744073709551615","act":{"data":{"global_sequence":42,"amount":"1.0000 EOS"}}}}]}"#; + let Input::Data(batch) = decode(text).unwrap() else { + panic!("data"); + }; + assert_eq!(batch.sequence, u64::MAX); + assert_eq!(batch.wire_bytes, text.len()); + assert_eq!( + batch.records[0].payload["act"]["data"]["global_sequence"], + 42 + ); + let ack: Command = Command::Ack { + scope: Scope { + connection_id: 1, + stream_id: 1, + generation: 1, + subscription_id: 100, + }, + sequence: u64::MAX, + }; + let actual: serde_json::Value = serde_json::from_str(&encode(&ack).unwrap()).unwrap(); + assert_eq!(actual["sequence"], "18446744073709551615"); +} diff --git a/crates/hyperion-stream-ws/tests/conformance.rs b/crates/hyperion-stream-ws/tests/conformance.rs new file mode 100644 index 0000000..a609305 --- /dev/null +++ b/crates/hyperion-stream-ws/tests/conformance.rs @@ -0,0 +1,114 @@ +use hyperion_stream_core::*; +use hyperion_stream_ws::codec; +use serde_json::{json, Value}; + +fn event_json(event: Event) -> Value { + match event { + Event::Connected(h) => { + json!({"type":"connected","connection_id":h.connection_id.to_string()}) + } + Event::Accepted { scope, start_from } => { + json!({"type":"accepted","stream_id":scope.stream_id.to_string(),"generation":scope.generation.to_string(),"subscription_id":scope.subscription_id.to_string(),"start_from":start_from}) + } + Event::Data(b) => { + json!({"type":"data","stream_id":b.scope.stream_id.to_string(),"generation":b.scope.generation.to_string(),"sequence":b.sequence.to_string(),"blocks":b.records.iter().map(|r|r.block_num).collect::>(),"cursors":b.records.iter().map(|r|r.cursor.map(|n|n.to_string())).collect::>(),"payloads":b.records.into_iter().map(|r|r.payload).collect::>()}) + } + Event::HistoryEnd { stream_id } => { + json!({"type":"history_end","stream_id":stream_id.to_string()}) + } + Event::End { stream_id } => json!({"type":"end","stream_id":stream_id.to_string()}), + Event::Cancelled { stream_id } => { + json!({"type":"cancelled","stream_id":stream_id.to_string()}) + } + Event::Error(e) => { + json!({"type":"error","stream_id":e.stream_id.map(|n|n.to_string()),"code":e.code,"retryable":e.retryable}) + } + Event::Lib(lib) => json!({"type":"lib","block_num":lib.block_num}), + Event::Fork(fork) => { + json!({"type":"fork","starting_block":fork.starting_block,"ending_block":fork.ending_block}) + } + Event::Disconnected => json!({"type":"disconnected"}), + } +} + +// Expectations are literal protocol assertions, not snapshots generated by the +// implementation. Objects may select public fields; arrays always match exactly. +fn assert_fields(actual: &Value, expected: &Value, context: &str) { + match expected { + Value::Object(fields) => { + for (key, value) in fields { + assert!( + actual.get(key).is_some(), + "{context}: missing {key}: {actual}" + ); + assert_fields(&actual[key], value, context); + } + } + Value::Array(values) => { + assert_eq!( + actual.as_array().map(Vec::len), + Some(values.len()), + "{context}: {actual}" + ); + for (a, e) in actual.as_array().unwrap().iter().zip(values) { + assert_fields(a, e, context); + } + } + _ => assert_eq!(actual, expected, "{context}"), + } +} + +#[test] +fn shared_state_machine_scenarios() { + let corpus: Value = + serde_json::from_str(include_str!("../../../conformance/v1/scenarios.json")).unwrap(); + for case in corpus["cases"].as_array().unwrap() { + let mut config = Config::default(); + for (key, value) in case["config"].as_object().into_iter().flatten() { + let n = value.as_u64().unwrap(); + match key.as_str() { + "stream_records" => config.stream_window.records = n as usize, + "stream_bytes" => config.stream_window.bytes = n as usize, + "connection_records" => config.connection_window.records = n as usize, + "connection_bytes" => config.connection_window.bytes = n as usize, + "request_timeout_ms" => config.request_timeout_ms = n, + "lib_timeout_ms" => config.lib_timeout_ms = n, + _ => panic!("Unknown fixture config {key}"), + } + } + let mut client = Client::::new(config).unwrap(); + for (index, step) in case["steps"].as_array().unwrap().iter().enumerate() { + let id = || step["stream_id"].as_str().unwrap().parse::().unwrap(); + let result = match step["op"].as_str().unwrap() { + "server" => codec::decode(&step["packet"].to_string()) + .and_then(|input| client.receive(input)), + "subscribe" => client.subscribe( + id(), + step["request"].clone(), + if step["delivery"] == "irreversible" { + Delivery::Irreversible + } else { + Delivery::Head + }, + step["replay"].as_bool().unwrap_or(true), + ), + "poll" => Ok(client.poll()), + "cancel" => Ok(client.cancel(id())), + "disconnect" => client.disconnect(), + "advance" => client.advance(step["now_ms"].as_u64().unwrap()), + op => panic!("Unknown fixture operation {op}"), + }; + let actual = match result { + Ok(t) => { + json!({"commands":t.commands.iter().map(codec::command_json).collect::,_>>().unwrap(),"events":t.events.into_iter().map(event_json).collect::>()}) + } + Err(e) => json!({"fatal":e.code}), + }; + assert_fields( + &actual, + &step["expect"], + &format!("{} step {index}", case["name"]), + ); + } + } +} diff --git a/crates/hyperion-stream-ws/tests/faults.rs b/crates/hyperion-stream-ws/tests/faults.rs new file mode 100644 index 0000000..c1bef63 --- /dev/null +++ b/crates/hyperion-stream-ws/tests/faults.rs @@ -0,0 +1,206 @@ +mod support; +use futures_util::{SinkExt, StreamExt}; +use hyperion_stream_core::{Delivery, Event}; +use hyperion_stream_ws::{ActionRequest, Options, Request, WsClient}; +use serde_json::json; +use std::time::Duration; +use support::*; +use tokio::net::TcpListener; +use tokio_tungstenite::{accept_async, tungstenite::Message}; + +#[tokio::test] +async fn wrong_subprotocol_is_rejected_before_subscribing() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + let _ = accept_async(tcp).await; + }); + assert!( + WsClient::connect(&format!("ws://{address}"), Options::default()) + .await + .is_err() + ); + server.await.unwrap(); +} + +#[tokio::test] +#[allow( + clippy::result_large_err, + reason = "Tungstenite requires this external callback error type" +)] +async fn authentication_rejection_is_not_a_retryable_transport_failure() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + let _ = tokio_tungstenite::accept_hdr_async( + tcp, + |_: &tokio_tungstenite::tungstenite::handshake::server::Request, _| { + Err(tokio_tungstenite::tungstenite::http::Response::builder() + .status(401) + .body(Some("authentication required".into())) + .unwrap()) + }, + ) + .await; + }); + let error = WsClient::connect(&format!("ws://{address}"), Options::default()) + .await + .err() + .unwrap(); + assert_eq!(error.code, "HTTP_ERROR"); + assert!(!error.retryable); + server.await.unwrap(); +} + +#[tokio::test] +async fn malformed_json_binary_and_oversize_messages_fail_boundedly() { + for message in [ + Message::text("{bad"), + Message::binary(vec![1, 2, 3]), + Message::text("x".repeat(2048)), + ] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut ws = accept(&listener, "1").await; + let _ = ws.send(message).await; + let _ = ws.next().await; + }); + let options = Options { + max_message_bytes: 1024, + ..Options::default() + }; + let mut client = WsClient::connect(&format!("ws://{address}"), options) + .await + .unwrap(); + client.next_event().await.unwrap(); + let failure = tokio::time::timeout(Duration::from_secs(1), client.next_event()) + .await + .unwrap() + .unwrap_err(); + assert!(!failure.message.is_empty()); + assert!(!client.is_connected()); + drop(client); + server.await.unwrap(); + } +} + +#[tokio::test] +async fn idle_peer_has_a_finite_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut ws = accept(&listener, "1").await; + let _ = ws.next().await; + }); + let options = Options { + idle_timeout: Duration::from_millis(20), + ..Options::default() + }; + let mut client = WsClient::connect(&format!("ws://{address}"), options) + .await + .unwrap(); + client.next_event().await.unwrap(); + let error = tokio::time::timeout(Duration::from_secs(1), client.next_event()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.code, "IDLE_TIMEOUT"); + assert!(error.retryable); + drop(client); + server.await.unwrap(); +} + +#[tokio::test] +async fn cancelling_a_pending_read_does_not_cancel_or_lose_the_subscription() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (release, wait) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let mut ws = accept(&listener, "1").await; + let sub = read(&mut ws).await; + wait.await.unwrap(); + send(&mut ws, accepted(&sub, "1")).await; + let _ = ws.next().await; + }); + let mut client = WsClient::connect(&format!("ws://{address}"), Options::default()) + .await + .unwrap(); + client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + true, + ) + .await + .unwrap(); + client.next_event().await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(10), client.next_event()) + .await + .is_err() + ); + release.send(()).unwrap(); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Accepted { .. } + )); + drop(client); + server.await.unwrap(); +} + +#[tokio::test] +async fn transport_drop_during_finality_wait_replays_only_new_connection_payloads() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut first = accept(&listener, "1").await; + let sub = read(&mut first).await; + send(&mut first, accepted(&sub, "1")).await; + let mut frame = json!({"type":"data","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100","sequence":"1","mode":"history","records":[{"block_num":100,"block_id":"b".repeat(64),"payload":{"fork":"old"}}]}); + send(&mut first, frame.clone()).await; + drop(first); + let mut second = accept(&listener, "2").await; + let sub = read(&mut second).await; + assert_eq!(sub["resume_from"], 100); + assert_eq!(sub["generation"], "2"); + send(&mut second, accepted(&sub, "2")).await; + send(&mut second,json!({"type":"lib","connection_id":"2","chain_id":"a".repeat(64),"block_num":200,"block_id":"c".repeat(64)})).await; + frame["connection_id"] = json!("2"); + frame["generation"] = json!("2"); + frame["records"][0]["payload"] = json!({"fork":"new"}); + send(&mut second, frame).await; + let _ = second.next().await; + }); + let mut client = WsClient::connect(&format!("ws://{address}"), Options::default()) + .await + .unwrap(); + client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Irreversible, + true, + ) + .await + .unwrap(); + client.next_event().await.unwrap(); + client.next_event().await.unwrap(); + assert!(client.next_event().await.is_err()); + client.reconnect().await.unwrap(); + for _ in 0..3 { + assert!(!matches!( + client.next_event().await.unwrap(), + Event::Data(_) + )); + } + let Event::Data(batch) = client.next_event().await.unwrap() else { + panic!("new data") + }; + assert_eq!(batch.records[0].payload, json!({"fork":"new"})); + drop(client); + server.await.unwrap(); +} diff --git a/crates/hyperion-stream-ws/tests/interoperability.rs b/crates/hyperion-stream-ws/tests/interoperability.rs new file mode 100644 index 0000000..3634257 --- /dev/null +++ b/crates/hyperion-stream-ws/tests/interoperability.rs @@ -0,0 +1,113 @@ +mod support; +use futures_util::StreamExt; +use hyperion_stream_core::{Delivery, Event}; +use hyperion_stream_ws::{Options, Request, WsClient}; +use serde_json::{json, Value}; +use support::*; +use tokio::net::TcpListener; + +fn corpus() -> Value { + serde_json::from_str(include_str!( + "../../../conformance/v1/network-sessions.json" + )) + .unwrap() +} + +async fn serve(listener: TcpListener, fixture: Value) { + let mut ws = accept(&listener, "1").await; + let sub = read(&mut ws).await; + assert_eq!(sub["request"], fixture["request"]); + assert_eq!(sub["generation"], "1"); + for frame in fixture["server_frames"].as_array().unwrap() { + send(&mut ws, frame.clone()).await; + } + let mut acks = Vec::new(); + let mut pong = false; + while acks.len() < 2 || !pong { + let received = read(&mut ws).await; + match received["type"].as_str().unwrap() { + "ack" => { + assert_eq!(received["connection_id"], "1"); + assert_eq!(received["generation"], "1"); + assert_eq!(received["subscription_id"], "100"); + acks.push(received["sequence"].clone()); + } + "pong" => { + assert_eq!(received["id"], "18446744073709551615"); + pong = true; + } + other => panic!("Unexpected {other}"), + } + } + assert_eq!(json!(acks), fixture["expected_acks"]); + let _ = ws.close(None).await; + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), ws.next()).await; +} + +#[tokio::test] +async fn rust_adapter_runs_shared_action_and_delta_network_sessions() { + for fixture in corpus()["cases"].as_array().unwrap() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(serve(listener, fixture.clone())); + let mut client = + WsClient::connect(&format!("ws://{address}/stream/v2"), Options::default()) + .await + .unwrap(); + let request: Request = serde_json::from_value(fixture["request"].clone()).unwrap(); + client + .subscribe(1, request, Delivery::Head, true) + .await + .unwrap(); + let mut records = Vec::new(); + loop { + match client.next_event().await.unwrap() { + Event::Data(batch) => { + for record in batch.records { + records.push(json!({"block_num":record.block_num,"block_id":record.block_id,"cursor":record.cursor.map(|n|n.to_string()),"payload":record.payload})); + } + } + Event::End { .. } => break, + Event::Error(e) => panic!("{e}"), + _ => {} + } + } + assert_eq!(json!(records), fixture["expected_records"]); + drop(client); + server.await.unwrap(); + } +} + +#[tokio::test] +async fn native_javascript_peer_runs_the_same_action_and_delta_network_sessions() { + for (index, fixture) in corpus()["cases"].as_array().unwrap().iter().enumerate() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(serve(listener, fixture.clone())); + let output = tokio::task::spawn_blocking(move || { + std::process::Command::new( + std::env::var_os("HYPERION_NODE").unwrap_or_else(|| "node".into()), + ) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../conformance/v1/node-peer.mjs" + )) + .arg(format!("ws://{address}/stream/v2")) + .arg(index.to_string()) + .output() + .unwrap() + }) + .await + .unwrap(); + assert!( + output.status.success(), + "status={:?} stdout={} stderr={}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let result: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(result["records"], fixture["expected_records"]); + server.await.unwrap(); + } +} diff --git a/crates/hyperion-stream-ws/tests/reconnect.rs b/crates/hyperion-stream-ws/tests/reconnect.rs new file mode 100644 index 0000000..2b2641d --- /dev/null +++ b/crates/hyperion-stream-ws/tests/reconnect.rs @@ -0,0 +1,267 @@ +mod support; +use futures_util::StreamExt; +use hyperion_stream_core::{Delivery, Event}; +use hyperion_stream_ws::{ActionRequest, Options, Request, WsClient}; +use std::time::Duration; +use support::*; +use tokio::net::TcpListener; + +#[tokio::test] +async fn reconnect_uses_fresh_request_deadline_after_time_offline() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut first = accept(&listener, "1").await; + let sub = read(&mut first).await; + send(&mut first, accepted(&sub, "1")).await; + let mut second = accept(&listener, "2").await; + let sub = read(&mut second).await; + assert_eq!(sub["generation"], "2"); + send(&mut second, accepted(&sub, "2")).await; + let _ = second.next().await; + }); + let mut options = Options::default(); + options.core.request_timeout_ms = 100; + let mut client = WsClient::connect(&format!("ws://{address}/stream/v2"), options) + .await + .unwrap(); + client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + true, + ) + .await + .unwrap(); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Connected(_) + )); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Accepted { .. } + )); + tokio::time::sleep(Duration::from_millis(200)).await; + client.reconnect().await.unwrap(); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Connected(_) + )); + let result = client.next_event().await.unwrap(); + assert!(matches!(result, Event::Accepted { .. }), "{result:?}"); + drop(client); + server.await.unwrap(); +} + +#[tokio::test] +async fn explicit_reconnect_retains_required_error_for_replay_disabled_stream() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut first = accept(&listener, "1").await; + let sub = read(&mut first).await; + send(&mut first, accepted(&sub, "1")).await; + let mut second = accept(&listener, "2").await; + let _ = second.next().await; + }); + let mut client = WsClient::connect(&format!("ws://{address}/stream/v2"), Options::default()) + .await + .unwrap(); + client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + false, + ) + .await + .unwrap(); + client.next_event().await.unwrap(); + client.next_event().await.unwrap(); + client.reconnect().await.unwrap(); + let outcome = client.next_event().await.unwrap(); + assert!( + matches!(outcome,Event::Error(ref e) if e.code=="STREAM_REPLAY_REQUIRED" && e.retryable), + "{outcome:?}" + ); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Connected(_) + )); + drop(client); + server.await.unwrap(); +} + +#[tokio::test] +async fn repeated_local_cancellation_has_a_bounded_control_outcome_queue() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut ws = accept(&listener, "1").await; + while let Some(Ok(_)) = ws.next().await {} + }); + let mut client = WsClient::connect(&format!("ws://{address}/stream/v2"), Options::default()) + .await + .unwrap(); + let mut bounded_failure = None; + for _ in 0..256 { + match client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + true, + ) + .await + { + Ok(()) => {} + Err(e) => { + bounded_failure = Some(e); + break; + } + } + if let Err(e) = client.cancel(1).await { + bounded_failure = Some(e); + break; + } + } + assert_eq!( + bounded_failure.map(|e| e.code).as_deref(), + Some("EVENT_QUEUE_OVERFLOW") + ); + drop(client); + server.await.unwrap(); +} + +#[tokio::test] +async fn failed_reconnect_cannot_reuse_disconnect_reserve_without_draining_events() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut connection = 0; + loop { + let (tcp, _) = listener.accept().await.unwrap(); + connection += 1; + let Ok(mut ws) = tokio_tungstenite::accept_hdr_async(tcp, negotiate).await else { + continue; + }; + send(&mut ws,serde_json::json!({"type":"hello","version":1,"connection_id":connection.to_string(),"chain_id":"a".repeat(64),"heartbeat_interval_ms":15000})).await; + while let Some(Ok(_)) = ws.next().await {} + } + }); + let mut options = Options::default(); + options.core.max_streams = 1; + options.max_pending_events = 3; + let mut client = WsClient::connect(&format!("ws://{address}"), options) + .await + .unwrap(); + client.next_event().await.unwrap(); + let mut admission_failed = false; + for _ in 0..10 { + if let Err(error) = client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + false, + ) + .await + { + assert_eq!(error.code, "EVENT_QUEUE_OVERFLOW"); + admission_failed = true; + break; + } + let _ = client.reconnect().await; + } + let mut outcomes = 0; + while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(20), client.next_event()).await + { + outcomes += 1; + } + drop(client); + server.abort(); + let _ = server.await; + assert!( + admission_failed, + "Offline subscription must honor undrained event capacity" + ); + assert!( + outcomes <= 5, + "normal cap plus at most one active stream/disconnect reserve: {outcomes}" + ); +} + +#[tokio::test] +async fn simultaneous_terminal_errors_survive_normal_event_budget_overflow() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let mut connection = 0; + loop { + let (tcp, _) = listener.accept().await.unwrap(); + connection += 1; + let Ok(mut ws) = tokio_tungstenite::accept_hdr_async(tcp, negotiate).await else { + continue; + }; + send(&mut ws,serde_json::json!({"type":"hello","version":1,"connection_id":connection.to_string(),"chain_id":"a".repeat(64),"heartbeat_interval_ms":15000})).await; + while let Some(Ok(_)) = ws.next().await {} + } + }); + let mut options = Options::default(); + options.core.max_streams = 2; + options.max_pending_events = 4; + options.core.request_timeout_ms = 20; + let mut client = WsClient::connect(&format!("ws://{address}"), options) + .await + .unwrap(); + client.next_event().await.unwrap(); + for _ in 0..2 { + client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + false, + ) + .await + .unwrap(); + client.reconnect().await.unwrap(); + } + for id in 1..=2 { + client + .subscribe( + id, + Request::Action(ActionRequest::default()), + Delivery::Head, + false, + ) + .await + .unwrap(); + } + tokio::time::sleep(Duration::from_millis(30)).await; + assert_eq!( + client.next_event().await.unwrap_err().code, + "EVENT_QUEUE_OVERFLOW" + ); + let mut codes = Vec::new(); + while let Ok(Ok(event)) = + tokio::time::timeout(Duration::from_millis(20), client.next_event()).await + { + if let Event::Error(error) = event { + codes.push(error.code); + } + } + drop(client); + server.abort(); + let _ = server.await; + assert_eq!( + codes, + vec![ + "STREAM_REPLAY_REQUIRED", + "STREAM_REPLAY_REQUIRED", + "REQUEST_TIMEOUT", + "REQUEST_TIMEOUT" + ] + ); +} diff --git a/crates/hyperion-stream-ws/tests/support/mod.rs b/crates/hyperion-stream-ws/tests/support/mod.rs new file mode 100644 index 0000000..3314954 --- /dev/null +++ b/crates/hyperion-stream-ws/tests/support/mod.rs @@ -0,0 +1,74 @@ +use futures_util::{SinkExt, StreamExt}; +use hyperion_stream_core::PROTOCOL; +use serde_json::{json, Value}; +use tokio::net::{TcpListener, TcpStream}; +use tokio_tungstenite::{ + accept_hdr_async, + tungstenite::{ + handshake::server::{ErrorResponse, Request, Response}, + Message, + }, + WebSocketStream, +}; + +#[allow( + clippy::result_large_err, + reason = "Tungstenite requires this external callback error type" +)] +pub fn negotiate(request: &Request, mut response: Response) -> Result { + assert_eq!(request.headers()["sec-websocket-protocol"], PROTOCOL); + response + .headers_mut() + .insert("sec-websocket-protocol", PROTOCOL.parse().unwrap()); + Ok(response) +} + +#[allow( + dead_code, + reason = "Shared test module; TLS scenarios accept their own encrypted stream" +)] +pub async fn accept(listener: &TcpListener, connection: &str) -> WebSocketStream { + let (tcp, _) = listener.accept().await.unwrap(); + let mut ws = accept_hdr_async(tcp, negotiate).await.unwrap(); + send(&mut ws, json!({"type":"hello","version":1,"connection_id":connection,"chain_id":"a".repeat(64),"heartbeat_interval_ms":15000})).await; + assert_eq!(read(&mut ws).await["type"], "ready"); + ws +} + +pub async fn read( + ws: &mut WebSocketStream, +) -> Value { + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + match ws.next().await.unwrap().unwrap() { + Message::Text(text) => { + let packet: Value = serde_json::from_str(&text).unwrap(); + if packet["type"] == "ping" { + send(ws, json!({"type":"pong","id":packet["id"]})).await; + continue; + } + return packet; + } + Message::Ping(_) => ws.flush().await.unwrap(), + _ => panic!("Expected text"), + } + } + }) + .await + .expect("fixture read deadline") +} + +pub async fn send( + ws: &mut WebSocketStream, + packet: Value, +) { + ws.send(Message::text(packet.to_string())).await.unwrap(); +} + +#[allow( + dead_code, + reason = "Shared test module; only reconnect scenarios need this helper" +)] +pub fn accepted(subscribe: &Value, connection: &str) -> Value { + json!({"type":"accepted","connection_id":connection,"stream_id":subscribe["stream_id"],"generation":subscribe["generation"],"request_id":subscribe["request_id"],"subscription_id":"100","start_from":100}) +} diff --git a/crates/hyperion-stream-ws/tests/tls.rs b/crates/hyperion-stream-ws/tests/tls.rs new file mode 100644 index 0000000..abae9da --- /dev/null +++ b/crates/hyperion-stream-ws/tests/tls.rs @@ -0,0 +1,87 @@ +mod support; +use hyperion_stream_ws::{Options, WsClient}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; +use serde_json::json; +use std::sync::Arc; +use support::*; +use tokio::net::TcpListener; +use tokio_rustls::TlsAcceptor; +use tokio_tungstenite::accept_hdr_async; + +async fn connect_fixture( + trusted: bool, + correct_name: bool, +) -> Result<(), hyperion_stream_core::Failure> { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server_config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert( + vec![CertificateDer::from( + include_bytes!("certs/server.der").to_vec(), + )], + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + include_bytes!("certs/server-key.der").to_vec(), + )), + ) + .unwrap(); + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + let Ok(tls) = TlsAcceptor::from(Arc::new(server_config)).accept(tcp).await else { + return; + }; + let Ok(mut ws) = accept_hdr_async(tls, negotiate).await else { + return; + }; + send(&mut ws,json!({"type":"hello","version":1,"chain_id":"a".repeat(64),"connection_id":"1","heartbeat_interval_ms":15000})).await; + assert_eq!(read(&mut ws).await["type"], "ready"); + }); + let mut options = Options::default(); + if trusted { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(CertificateDer::from( + include_bytes!("certs/ca.der").to_vec(), + )) + .unwrap(); + options.tls = Some(Arc::new( + rustls::ClientConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(), + )); + } + let host = if correct_name { + "localhost" + } else { + "127.0.0.1" + }; + let result = WsClient::connect( + &format!("wss://{host}:{}/stream/v2", address.port()), + options, + ) + .await; + let outcome = result.map(drop); + server.await.unwrap(); + outcome +} + +#[tokio::test] +async fn wss_accepts_an_explicitly_trusted_matching_certificate() { + connect_fixture(true, true).await.unwrap(); +} +#[tokio::test] +async fn wss_rejects_an_untrusted_certificate() { + assert!(!connect_fixture(false, true).await.unwrap_err().retryable); +} +#[tokio::test] +async fn wss_rejects_a_trusted_certificate_for_the_wrong_hostname() { + assert!(!connect_fixture(true, false).await.unwrap_err().retryable); +} diff --git a/crates/hyperion-stream-ws/tests/transport.rs b/crates/hyperion-stream-ws/tests/transport.rs new file mode 100644 index 0000000..670b83a --- /dev/null +++ b/crates/hyperion-stream-ws/tests/transport.rs @@ -0,0 +1,92 @@ +use futures_util::{SinkExt, StreamExt}; +use hyperion_stream_core::{Delivery, Event, PROTOCOL}; +use hyperion_stream_ws::{ActionRequest, Options, Request, WsClient}; +use serde_json::{json, Value}; +use tokio::net::TcpListener; +use tokio_tungstenite::{ + accept_hdr_async, + tungstenite::{ + handshake::server::{Request as HandshakeRequest, Response}, + Message, + }, +}; + +#[tokio::test] +#[allow( + clippy::result_large_err, + reason = "Tungstenite requires this external callback error type" +)] +async fn actual_websocket_delivers_json_and_acknowledges_after_application_acceptance() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + let mut ws = accept_hdr_async(tcp, |req: &HandshakeRequest, mut response: Response| { + assert_eq!(req.headers()["sec-websocket-protocol"], PROTOCOL); + response + .headers_mut() + .insert("sec-websocket-protocol", PROTOCOL.parse().unwrap()); + Ok(response) + }) + .await + .unwrap(); + ws.send(Message::text(json!({"type":"hello","version":1,"connection_id":"1","chain_id":"a".repeat(64),"heartbeat_interval_ms":15000}).to_string())).await.unwrap(); + let ready: Value = + serde_json::from_str(ws.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + assert_eq!(ready["type"], "ready"); + let sub: Value = + serde_json::from_str(ws.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + assert_eq!(sub["request"]["kind"], "action"); + let mut scope = + json!({"connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100"}); + scope["type"] = json!("accepted"); + scope["request_id"] = json!("1"); + scope["start_from"] = json!(100); + ws.send(Message::text(scope.to_string())).await.unwrap(); + scope.as_object_mut().unwrap().remove("request_id"); + scope.as_object_mut().unwrap().remove("start_from"); + scope["type"] = json!("data"); + scope["sequence"] = json!("1"); + scope["mode"] = json!("history"); + scope["records"] = json!([{"block_num":100,"block_id":"b".repeat(64),"cursor":"18446744073709551615","payload":{"global_sequence":"18446744073709551615"}}]); + ws.send(Message::text(scope.to_string())).await.unwrap(); + let ack: Value = + serde_json::from_str(ws.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + assert_eq!( + ack, + json!({"type":"ack","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100","sequence":"1"}) + ); + let cancel: Value = + serde_json::from_str(ws.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + assert_eq!(cancel["type"], "cancel"); + }); + let mut client = WsClient::connect(&format!("ws://{address}/stream/v2"), Options::default()) + .await + .unwrap(); + client + .subscribe( + 1, + Request::Action(ActionRequest::default()), + Delivery::Head, + true, + ) + .await + .unwrap(); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Connected(_) + )); + assert!(matches!( + client.next_event().await.unwrap(), + Event::Accepted { .. } + )); + let Event::Data(data) = client.next_event().await.unwrap() else { + panic!("data"); + }; + assert_eq!(data.records[0].cursor, Some(u64::MAX)); + client.cancel(1).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); +} diff --git a/docs/LAUNCH.md b/docs/LAUNCH.md index 4e686cb..668fdf8 100644 --- a/docs/LAUNCH.md +++ b/docs/LAUNCH.md @@ -23,7 +23,7 @@ production deployment, or universal server compatibility follows from that merge types and tested at neighboring 2^53 values, 2^63 and uint64 max through history/live packets. Server projection error retryability, status, and code survive setup and delivery failures. - TypeScript 7.0.2 emits ESM/declarations, CommonJS bundles emitted JS, and UMD bundles the - emitted browser entry. Bun 1.4.3 checks the complete source project, including examples. + emitted browser entry. Bun 1.4.4 checks the complete source project, including examples. - npm lockfile/pins, prepack build, explicit publish files, conditional declaration exports, external ESM/CJS/TypeScript consumer checks, and Chromium ESM/UMD acceptance are present. - CI runs Node 24/26 on Linux and Windows and real Chromium on Linux/Node24. diff --git a/docs/reviews/rust-native-stream-architecture.md b/docs/reviews/rust-native-stream-architecture.md new file mode 100644 index 0000000..afc100c --- /dev/null +++ b/docs/reviews/rust-native-stream-architecture.md @@ -0,0 +1,149 @@ +# Rust-first native streaming: source audit and proposed migration + +Investigation date: 2026-10-22. Status: proposal, not a production implementation or accepted +wire protocol. Accepted user direction: Rust first, Python/others only as needed; a zero-deps +core with a separate transport/TLS adapter is acceptable. Existing 4.5 qualification and +the architecture chat's frozen measurement campaign remain independent. + +## Recommendation + +Replace the public Socket.IO protocol with a versioned Hyperion protocol over standard +WebSocket, initially alongside the existing route. Keep the Rust subscription state machine +free of mandatory third-party crates. Supply runtime, WebSocket, wire codec and TLS through +an explicitly separate adapter. A Tokio + tokio-tungstenite + Rustls adapter is the baseline +candidate, not a zero-dependency complete connection stack. + +For JavaScript on supported Node/browser versions, native WebSocket plus native JSON can +support a package with no runtime dependencies. Keep any legacy Socket.IO compatibility +package separate: an installed optional or mandatory Socket.IO dependency would still count. +The current npm package has one direct runtime dependency, `socket.io-client@4.8.4`, with +Engine.IO, parsers and other dependencies underneath it. + +Transport and encoding are separate choices. Compare native WebSocket with JSON, bounded +batching and explicit delivery windows first; then evaluate binary codecs against the same +records and correctness checks. No performance improvement has yet been demonstrated. +The [transport research](socketio-removal-transport-research.md) compares Rust crates, +alternative transports, browser constraints and codec candidates. + +Rust adapter candidates are Tokio/tokio-tungstenite/Rustls as the initial implementation, +fastwebsockets as a measured alternative, and WebTransport as a separate experiment. These +are not competing choices for the zero-deps core. In local runtime probes, Node 22.23.3 and +24.22.0 both expose `WebSocket`, but have neither a `WebTransport` global nor a `node:quic` +builtin. Current browser WebTransport support is broader ([Baseline 2026](https://developer.mozilla.org/en-US/docs/Web/API/WebTransport)); +the Node results are a concrete constraint on using it as the single zero-runtime-deps +JavaScript transport for our present baseline, not a claim about every Node release. + +## Actual Socket.IO scope in Hyperion + +Source audit pins: stream client `99ad2cf1ee47ee7c8c5cf3ea7a4f35bfbdf4ff11`; History API +`release/4.5` at `37035935facb306207194d8309d29542d68b8ad6`. These are investigation +references, not a newly qualified release combination. + +| Use | Observed implementation | Replacement work | +| --- | --- | --- | +| Public `/stream` | `SocketManager` constructs a WebSocket-only Socket.IO server on a uWebSockets.js app. The current client also selects WebSocket-only mode. | Add a distinct native endpoint/subprotocol, application handshake and connection registry. HTTP polling fallback is not part of this public path today. | +| API to indexer `/router` | The API is a Socket.IO client; the router is a Socket.IO server. It carries registrations, cancellations, client counts, block/action/delta data and fork/LIB notifications. | Migrate this separately; a native public endpoint alone does not remove internal Socket.IO. Preserve relay replacement, request ownership and disconnect cleanup. | +| Cross-API broadcast | The public Socket.IO server uses the Redis adapter keyed by chain ID. Status, fork and LIB events use broadcasts; live records address local socket IDs directly. | Define native fanout ownership and test multiple API workers, duplicate events, restarts and slow clients. Do not silently remove this behavior with the adapter. | +| History delivery | `streamPastCommon` accepts a Socket.IO `Socket`, waits for each batch ACK with a 120-second deadline, and clears the Elasticsearch scroll in `finally`. | Extract a transport-independent history sink with cancellation and bounded acceptance. Preserve source/projection proof guards and resource cleanup. | +| Hub publication | `base-publisher.ts` separately connects to an external query hub over Socket.IO with a fresh authenticated session. The master also retains a Socket.IO hub type. | Coordinate that peer protocol separately if the goal includes removing Socket.IO from the entire HAPI dependency tree. | + +Pinned source: [public listener and Redis adapter](https://github.com/eosrio/hyperion-history-api/blob/37035935facb306207194d8309d29542d68b8ad6/src/api/socketManager.ts#L81-L101), +[API relay](https://github.com/eosrio/hyperion-history-api/blob/37035935facb306207194d8309d29542d68b8ad6/src/api/socketManager.ts#L209-L281), +[internal router](https://github.com/eosrio/hyperion-history-api/blob/37035935facb306207194d8309d29542d68b8ad6/src/indexer/workers/ws-router.ts#L424-L491), +[history delivery and cleanup](https://github.com/eosrio/hyperion-history-api/blob/37035935facb306207194d8309d29542d68b8ad6/src/api/helpers/functions.ts#L279-L340), +[hub connection](https://github.com/eosrio/hyperion-history-api/blob/37035935facb306207194d8309d29542d68b8ad6/src/api/qry-hub/base-publisher.ts#L141-L158), +[current client](https://github.com/eosrio/hyperion-stream-client/blob/99ad2cf1ee47ee7c8c5cf3ea7a4f35bfbdf4ff11/src/hyperion-stream-client.ts). + +The server already has a native WebSocket implementation through uWebSockets.js. This +proposal does not require rewriting the API server in Rust or adding another broker. + +## Reproduced wire compatibility constraint + +The [diagnostic probe](../../scripts/probes/socketio-wire-probe.mjs) runs a real loopback +Socket.IO 4.8.4 server with the same public path/WebSocket transport profile and a native +Node WebSocket client. It is a protocol fixture, not the production SocketManager. + +Executed on native Node 22.23.3 and 24.22.0: + +| Probe | Both runtimes observed | +| --- | --- | +| Plain WebSocket to `/stream/` | No WebSocket connection; no subscription request reached the server. | +| Engine.IO query parameters, then a plain JSON subscribe command | WebSocket opened and returned the Engine.IO session/heartbeat envelope, then closed without registering a request. | +| Minimal Engine.IO/Socket.IO framing | Namespace connection, Hyperion-style handshake and a correlated request ACK succeeded. | + +The last probe explicitly sends `40` and a `427[...]` event and receives `437[...]`. +It illustrates why replacing the npm import with native WebSocket while retaining the old +endpoint keeps the Engine.IO/Socket.IO protocol obligation. The probe is intentionally not +a maintained Socket.IO reimplementation. Run with `node scripts/probes/socketio-wire-probe.mjs`. + +As a local artifact baseline, the current UMD is 62,157 bytes, or 18,382 bytes with Node's +default gzip. There is no native replacement artifact to compare yet; these numbers are not +a claimed saving or evidence that transport is the current indexing bottleneck. + +## Proposed module shape + +```text +protocol/ schemas, semantics, versioned scenarios +crates/hyperion-stream-core/ zero mandatory third-party crates +crates/hyperion-stream-ws/ optional runtime/WebSocket/codec/TLS adapter +packages/typescript-native/ platform WebSocket + JSON +packages/legacy-socketio/ explicit compatibility installation +conformance/ common fixture runner and language drivers +``` + +The core owns subscription transitions, inclusive recovery boundaries, history/live ordering, +stale-generation rejection, bounded application queues, fork/LIB invalidation, errors and +cancellation. Its inputs/outputs are typed Rust values and explicit effects, independent of +networking, clocks, Serde and Tokio. The adapter performs I/O, wire encoding/decoding, timers +and certificate verification. Adapter dependencies and feature selections remain visible. +Existing JavaScript paths need not move during the investigation. + +On HAPI, extract a `StreamSession`/history-sink seam so both adapters share request validation, +history/filter logic, canonical-source/projection guards and lifecycle cleanup. Merely adding +a second sender around the current Socket.IO-specific maps is insufficient for native +registration, cancellation, cross-worker broadcast and relay recovery. + +## Protocol requirements before implementation + +A candidate `/stream/v2` route with a versioned subprotocol is a proposal, not a reserved +or implemented endpoint. Specify: + +- Chain identity, supported wire version, capabilities, request correlation IDs, logical + subscription IDs and connection/subscription generations. Do not use a Socket.IO socket ID + as a durable consumer identity. +- Subscribe/accepted, data batches, history completion, fork/LIB notifications, cancel and + terminal failures. Preserve exact decimal-string sequences and arbitrary contract fields. +- Bounded acceptance windows in records and bytes for both history and live data, deadlines, + and explicit slow-consumer failure. Reserve control capacity so stalled data does not block + cancellation, errors or heartbeat processing. Shared indexing must remain protected from + one slow subscription. +- Inclusive replay from the earliest unread/unconfirmed position, canonical fork replacement, + and scoped failure when retained inputs are unavailable. Delivery ACKs remain distinct from + application durability; transport changes do not create exactly-once delivery. +- TLS/origin/authentication policy, heartbeat behavior compatible with browser interfaces, + proxy upgrade/idle settings, frame/message limits, and validated error/version handling. + +These semantics are inferred requirements for the replacement. [WebSocket](https://www.rfc-editor.org/rfc/rfc6455.html) +does not supply them. [Socket.IO](https://socket.io/docs/v4/delivery-guarantees/) also leaves +additional arrival guarantees to the application. + +## Staged migration and acceptance + +1. Freeze the contract, limits and shared conformance scenarios; keep legacy 4.5 work moving. +2. Build the zero-deps Rust core and optional adapter against a bounded native fixture, with + a platform-JavaScript peer to check cross-language behavior and the zero-runtime-deps path. +3. Integrate a native HAPI adapter through the shared session/history seam. Run legacy and + native routes against the same actual retained dataset, proxy and recovery/finality cases. +4. Compare JSON/binary encodings and candidate Rust transports using identical payloads, + TLS/proxy settings, fault schedules and memory bounds. Measure parsing/allocation, wire + bytes, throughput, tail latency, CPU and peak queue occupancy; include overlapping and + slow subscriptions. Accept only runs with matching source completeness and identities. +5. Migrate the internal router and replace cross-worker broadcast deliberately. Retire legacy + public transport only after an explicit supported-server/client migration decision. Any + whole-HAPI dependency removal also needs a compatible external hub migration. + +[HAPI #231](https://github.com/eosrio/hyperion-history-api/issues/231), +[release matrix #20](https://github.com/eosrio/hyperion-tools/issues/20), and +[release coordination #24](https://github.com/eosrio/hyperion-tools/issues/24) remain the +qualification owners. Neither this report, the diagnostic fixture nor a codec benchmark +closes those gates. No production source, lockfile, package dependency or service changed. diff --git a/docs/reviews/socketio-removal-transport-research.md b/docs/reviews/socketio-removal-transport-research.md new file mode 100644 index 0000000..439a51b --- /dev/null +++ b/docs/reviews/socketio-removal-transport-research.md @@ -0,0 +1,209 @@ +# Rust-first streaming without Socket.IO: transport research + +Research date: 2026-10-22. This is an implementation proposal, not an accepted protocol or +release qualification. No runtime, dependencies, release pins, or running services were changed. + +## Recommendation + +Use standard WebSocket with a small, versioned Hyperion application protocol for the new +Rust-first client. Retain the existing Socket.IO route during migration and run both routes +against the same behavioral conformance suite. JavaScript and later Python clients can use +their own native WebSocket implementations; neither needs to load a Rust shared library. +This recommendation is an architectural inference from the sources below, not a measured +throughput or memory result. The API/relay integration scope needs the separate source audit. +Accepted dependency policy: the user permits a **zero-dependency core with a separate +transport/TLS adapter**. Keep third-party runtime, TLS, codec and ecosystem types out of the +core's required dependency graph and public interface. Adapter dependencies remain explicit. + +## Options + +| Option | Verified properties | Assessment | +| --- | --- | --- | +| Standard WebSocket | RFC 6455 defines bidirectional text/binary messages, control frames, closing, an opening handshake, and subprotocol selection. | Best fit for subscription commands, data, cancellation, and acknowledgements on one connection. Hyperion must define application semantics. | +| Keep the Socket.IO wire protocol but remove its library | Engine.IO defines connection establishment, heartbeat and packet encoding; Socket.IO adds namespace/event/ACK and binary-attachment framing. | Preserves endpoint compatibility but leaves us maintaining those protocols. It removes the package, not the protocol dependency. | +| SSE plus HTTP control requests | EventSource receives UTF-8 event streams, reconnects, and sends `Last-Event-ID`; its interface is receive-oriented. | Feasible, but subscribe/cancel/ACK traffic needs additional HTTP requests and correlation. Binary payloads need an application encoding. It is less direct for this bidirectional workload. | +| gRPC streaming | Native gRPC supports streaming flow control. The official grpc-web implementation currently supports unary and server streaming, but not client or bidirectional streaming. | Attractive for a native-only interface; browser parity requires another pattern or transport. Not the first common transport. | + +Sources: [RFC 6455](https://www.rfc-editor.org/rfc/rfc6455.html), +[Engine.IO protocol](https://socket.io/docs/v4/engine-io-protocol/), +[Socket.IO protocol](https://socket.io/docs/v4/socket-io-protocol/), +[EventSource specification](https://html.spec.whatwg.org/multipage/server-sent-events.html), +[gRPC flow control](https://grpc.io/docs/guides/flow-control/), +[grpc-web streaming support](https://github.com/grpc/grpc-web#streaming-support). +Socket.IO protocol revision numbers and package major versions are distinct; interoperability +must pin the actual wire revision rather than guessing from a library's version label. + +## Rust, Node and browser feasibility + +Recommended optional native adapter: **Tokio + tokio-tungstenite + Rustls**. Tokio supplies +scheduling/network I/O; Tungstenite implements WebSocket; tokio-tungstenite integrates it +with Tokio. These are explicit adapter dependencies, not dependencies of the core. + +| Crate | Current stable reference, 2026-10-22 | Role | +| --- | --- | --- | +| Tokio | 1.54.1 | Async runtime, network I/O, timers and synchronization. | +| tokio-tungstenite / Tungstenite | 0.30.0 / 0.30.0 | Async adapter / RFC 6455 implementation. | +| Rustls | 0.23.46 | TLS, with explicit crypto-provider and trust-root configuration. | +| fastwebsockets | 0.10.0 | Alternate frame-oriented implementation to benchmark. | + +Versions were checked against the [Tokio](https://crates.io/api/v1/crates/tokio), +[tokio-tungstenite](https://crates.io/api/v1/crates/tokio-tungstenite), +[Tungstenite](https://crates.io/api/v1/crates/tungstenite), +[Rustls](https://crates.io/api/v1/crates/rustls) and +[fastwebsockets](https://crates.io/api/v1/crates/fastwebsockets) registry metadata. They are +candidate pins, not a compiled or qualified combination. tokio-tungstenite declares Tokio +`^1.0.0`, optional Rustls `^0.23.0` and tokio-rustls `^0.26.0`. +[Adapter manifest](https://github.com/snapview/tokio-tungstenite/blob/v0.30.0/Cargo.toml). + +Select minimal Tokio features, a Rustls crypto provider, and OS or bundled trust roots +deliberately; verify certificate/hostname rejection and proxy support. Rustls's default +provider is `aws-lc-rs`, with its own platform/build requirements. TLS is not automatically +enabled merely by choosing a WebSocket crate. +[Rustls capabilities](https://docs.rs/rustls/latest/rustls/). + +Tungstenite **0.30.0** exposes frame/message/write-buffer limits; defaults include a 64 MiB +message limit and unlimited maximum write buffer. TLS must be selected, and the crate +documents no `permessage-deflate` support. These require deployment qualification. +[Limits](https://docs.rs/tungstenite/0.30.0/tungstenite/protocol/struct.WebSocketConfig.html), +[crate capabilities](https://docs.rs/tungstenite/0.30.0/tungstenite/). + +`fastwebsockets` exposes raw frames; use `FragmentCollector` for complete messages. Its +`upgrade` feature supplies client/server HTTP handshakes through Hyper, and its interface +offers automatic Pong/Close handling. Supply TLS separately and qualify masking, message +limits, fragmented/control-frame interleaving, close races and cancellation. It also lacks +`permessage-deflate`. Neither its name nor upstream benchmarks establish superiority for +Hyperion's payloads or reconnect workload. +[Pinned interface](https://docs.rs/fastwebsockets/0.10.0/fastwebsockets/). + +Node's built-in, browser-compatible `WebSocket` became non-experimental in **22.4.0**. +A new client targeting supported Node 22/24 versions and browsers can therefore use the +platform client instead of `socket.io-client`. +[Node 22.23.3 source documentation](https://github.com/nodejs/node/blob/v22.23.3/doc/api/globals.md#class-websocket). +This does not provide a Node WebSocket server or remove server-side dependencies. + +Rust/WASM needs browser I/O integration; `web-sys::WebSocket` is an external-crate option, +not zero Cargo dependencies. WASM is optional for the JavaScript client. +[wasm-bindgen browser example](https://rustwasm.github.io/docs/wasm-bindgen/examples/websockets.html). + +The standard browser WebSocket interface exposes outgoing `bufferedAmount`, but no read-side +backpressure interface. Ping/Pong control frames are not exposed to browser JavaScript. +Use server-side RFC liveness checks and define an application heartbeat if browser code must +observe liveness/progress; do not require JavaScript to send RFC Ping frames. +[WebSocket interface and heartbeat specification](https://websockets.spec.whatwg.org/). + +## Accepted zero-dependency core scope + +Rust's standard library exposes TCP/UDP networking and `Future`, but no TLS, WebSocket or +JSON client/codec interface. Async does not inherently require Tokio: a caller-driven +`Future`/poll interface or an owned executor can use standard primitives. Requiring +`futures_core::Stream`, however, introduces an external crate. A strict implementation must +own and qualify its envelope codec or explicitly receive those capabilities. Network +framing and TLS can live in the separately permitted adapter. +[std catalog](https://doc.rust-lang.org/std/), [networking](https://doc.rust-lang.org/std/net/), +[Future](https://doc.rust-lang.org/std/future/), +[external Stream trait](https://docs.rs/futures-core/0.3.34/futures_core/stream/trait.Stream.html). + +A caller-driven core with an empty Cargo dependency graph does not independently open +`wss://`, authenticate certificates or configure proxies. The accepted adapter supplies those +functions. This distinction remains visible in packaging and documentation. + +Windows offers OS WinHTTP WebSocket upgrade/send/receive APIs; `WINHTTP_FLAG_SECURE` +selects TLS. Apple offers `URLSessionWebSocketTask` over TCP/TLS, while its current networking +guidance favors Network.framework for new WebSocket code. Direct Rust FFI is feasible in +principle, not implemented or qualified here. +[WinHTTP upgrade](https://learn.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpwebsocketcompleteupgrade), +[secure requests](https://learn.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest), +[Apple task](https://developer.apple.com/documentation/foundation/urlsessionwebsockettask), +[Apple guidance](https://developer.apple.com/documentation/technotes/tn3151-choosing-the-right-networking-api), +[Rust FFI](https://doc.rust-lang.org/reference/items/external-blocks.html). + +Linux kTLS does not fill the gap: kernel record encryption is configured after a userspace +TLS handshake. A system OpenSSL/libcurl dependency would still be an external requirement. +Do not implement TLS, silently vendor a TLS library, or invoke a subprocess and label the +result zero dependencies. Under the accepted policy, an explicit TLS adapter is legitimate. +[Kernel TLS interface](https://docs.kernel.org/networking/tls.html). + +## Transport versus codec optimization + +WebSocket/HTTP3 is a transport choice; JSON/MessagePack/CBOR/Protobuf is a payload-codec +choice. Sending UTF-8 JSON in a binary WebSocket message does not compress or change its +representation. Compare these dimensions independently against the same semantic tests. + +| Candidate | Benefit to investigate | Compatibility obligations | +| --- | --- | --- | +| WebSocket + JSON | Existing payload shape, native browser/Node JSON support, simple inspection. | Establish the baseline; batch messages and bound parsing/queues. | +| WebSocket + MessagePack or CBOR | Binary maps/arrays/integers and byte strings. | Specify a JSON-compatible profile, string keys, supported values, size/depth limits and unknown fields. Preserve public decimal-string sequence semantics even if the codec supports native uint64. | +| WebSocket + Protobuf envelope | Typed control fields and generated schema tooling. | Arbitrary contract data needs a defined representation: raw JSON bytes are one option. Avoid projecting exact integers through `google.protobuf.Value.number_value`, which is double. Generated code may require a runtime. | +| Native gRPC | Typed RPC and streaming flow control. | Protocol and infrastructure change; official grpc-web still lacks client/bidirectional streaming. | +| WebTransport | Independent reliable streams; separate unreliable datagrams. | Ledger events require reliable delivery. Qualify browser versions, server infrastructure, certificates and fallback behavior. | + +Sources: [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md), +[CBOR RFC 8949](https://www.rfc-editor.org/rfc/rfc8949.html), +[Protobuf encoding](https://protobuf.dev/programming-guides/encoding/), +[ProtoJSON mappings](https://protobuf.dev/programming-guides/json/). +Browser JavaScript has no built-in MessagePack/CBOR/Protobuf codec. A new codec requires +runtime code/dependencies or an owned/generated self-contained implementation; code +generation alone does not guarantee zero runtime dependencies. Keep JSON supported unless +an alternative passes parity and shows a workload benefit, including server encoding cost. + +WebTransport is now listed as **Baseline 2026**, newly available since March 2026; older +browsers and individual features still vary. The July 2026 W3C snapshot specifies HTTP/3 +and HTTP/2 reliable-only modes. Its QUIC/HTTP3 path needs UDP and suitable proxy/listener +support; do not assume HTTP/2 fallback is implemented end to end. Rust `wtransport` **0.7.2** +is a QUIC/HTTP3 candidate, not a general WebSocket replacement. Treat it as a later measured +experiment rather than making datagram loss or a new deployment requirement part of the +baseline. +[Browser status](https://developer.mozilla.org/en-US/docs/Web/API/WebTransport), +[W3C snapshot](https://www.w3.org/TR/2026/CR-webtransport-20260730/), +[Rust interface](https://docs.rs/wtransport/0.7.2/wtransport/). + +Root observed in the actual supported Node **22.23.3/24.22.0** binaries: +`typeof WebTransport === 'undefined'`, `typeof WebSocket === 'function'`, and +`isBuiltin('node:quic') === false`. A WebTransport adapter needs an additional implementation +on that tested baseline. This local observation does not characterize every Node release. + +## What the Hyperion protocol still must own + +WebSocket replaces framing and connection handling, not subscription correctness. Proposed +requirements: + +- Negotiate a protocol version and capabilities through a distinct route/subprotocol and a + bounded application handshake. Specify request IDs, logical stream IDs and connection + generations so a stale reply cannot target a replacement subscription. +- Define subscribe/accepted/data/history-end/fork/LIB/cancel/cancelled/error semantics, + per-request terminal failures, and unknown-field behavior. Start with a JSON envelope and + canonical decimal-string sequence identifiers. Add a binary codec only after measurement. +- Bound individual frames, reassembled messages, batch sizes, inbound/outbound queues and + reversible buffers in both bytes and records. Define a per-stream credit/acceptance policy + and explicit overflow failure. Keep control/heartbeat/cancel handling operational when a + data consumer stalls; one stalled subscription must not deadlock all control traffic. +- Separate delivery acceptance from application durability. Reconnection alone cannot + establish retained-history completeness, exactly-once processing, a durable consumer + checkpoint, canonical fork replacement or LIB correctness. Define inclusive replay and + explicit unavailable-history errors, then test them against actual Hyperion sources. +- Define timeout, reconnect backoff, cancellation and cleanup behavior; reject old UUIDs or + generations and invalidate unconfirmed fork data before replay. +- Specify `wss://`, authentication, allowed browser origins, subprotocol validation, proxy + upgrade forwarding and idle deadlines. Browser constructors expose URL/subprotocols rather + than arbitrary request headers; choose an authentication mechanism that also works there. + +These are proposed application responsibilities inferred from the scope of +[RFC 6455](https://www.rfc-editor.org/rfc/rfc6455.html) and the +[browser interface](https://websockets.spec.whatwg.org/), not guarantees supplied by either. + +## Rust-first implementation shape and acceptance + +Keep protocol types/validation and subscription transitions separate from network I/O. +Use bounded queues and explicit close semantics. Keep Tokio/Tungstenite/Rustls behind the +accepted separate adapter, with no mandatory third-party types or codecs in the core. +Browser and later language implementations must satisfy the same conformance contract. + +The first spike should run Rust and platform JavaScript against the same loopback endpoint, +then the actual Hyperion route/proxy. Test binary/text frames, malformed/oversized messages, +heartbeat timeout, stalled consumers, cancellation, partial-block drops, repeated recovery, +fork replacement, LIB advancement, exact uint64 strings and scoped retryable errors. Run +package and TLS tests on Windows/Linux. Measure bytes, CPU, throughput, latency and peak +buffer occupancy with identical data and correctness checks before making performance claims. +Existing server/client qualification remains +[History API #231](https://github.com/eosrio/hyperion-history-api/issues/231); successful +transport experiments alone do not close it. diff --git a/protocol/v1/README.md b/protocol/v1/README.md new file mode 100644 index 0000000..5a64751 --- /dev/null +++ b/protocol/v1/README.md @@ -0,0 +1,197 @@ +# Hyperion native streaming v1 + +Working implementation contract, 2026-10-23. This is an additive, experimental +protocol; the legacy Socket.IO endpoint remains supported. Server route ownership: +the Hyperion architecture task. Client implementation: this repository. + +## Transport and identity + +One UTF-8 JSON object per WebSocket text message, subprotocol +`hyperion-stream.v1.json`. Proposed listener path: `/stream/v2` on the existing +stream listener, subject to the server's route/proxy integration tests. No binary +messages, compression requirement, or Socket.IO/Engine.IO framing. The client +requires the server to select the exact subprotocol. Production uses `wss://`. + +All IDs, generations, frame sequences and uint64 cursors are canonical decimal +strings in `0..=18446744073709551615` (no leading zeros except `"0"`). Block numbers +are JSON integers in `1..=4294967295`. Chain/block IDs are 64 lowercase hex digits. +Request/stream/subscription/connection IDs and generations are nonzero. Request +IDs and generations are fresh for every subscribe attempt; stream IDs identify a +logical client subscription and survive reconnect. Subscription IDs are assigned +by the server and unique within a connection. Connection IDs identify a server +connection incarnation. Every scoped packet carries the full scope: + +```json +{"connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100"} +``` + +Connection identity plus generation fences every data, ACK, error, cancel and +completion packet. Old/unknown scopes are ignored without granting credit, +delivering data or changing current state. Request IDs correlate subscribe and +accepted/rejected outcomes; they are distinct from server subscription IDs. + +## Handshake and subscription + +Server first: + +```json +{"type":"hello","version":1,"chain_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","connection_id":"1","heartbeat_interval_ms":15000} +``` + +Client responds with its connection acceptance budget before subscribing: + +```json +{"type":"ready","connection_id":"1","window":{"records":4096,"bytes":8388608}} +{"type":"subscribe","connection_id":"1","request_id":"1","stream_id":"1","generation":"1","request":{"kind":"action","contract":"eosio.token","action":"transfer","start_from":100},"resume_from":null,"window":{"records":1024,"bytes":2097152}} +``` + +`request` is an action or delta request with `kind`, the existing selectors, +`start_from`, `read_until`, `ignore_live`, `filter_op`, and filters. `replayOnReconnect` +is a local client policy, not a server request field. Falsy filter values must +survive serialization. `resume_from` overrides the original start bound with an +inclusive absolute block on recovery; a null value uses the original request. +The server retains the original end bound and filters. Initial start bounds may +be zero (live), absolute/relative block integers, or existing timestamp strings. +`read_until` keeps the existing history-only meaning; `ignore_live` requests a +finite stream. The existing server validates selectors and supported filters. + +```json +{"type":"accepted","connection_id":"1","request_id":"1","stream_id":"1","generation":"1","subscription_id":"100","start_from":100} +``` + +The server resolves the actual absolute inclusive replay anchor in `start_from`, +including live/relative/timestamp requests. No data before accepted. Clients pin +the first chain identity (or configure one) and reject a different chain on a +later connection. Unknown version, malformed packets, wrong current request +correlation and sequence gaps fail visibly; they never advance replay. + +## Data, credit and completion + +```json +{"type":"data","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100","sequence":"1","mode":"history","records":[{"block_num":100,"block_id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","cursor":"18446744073709551615","payload":{"global_sequence":"18446744073709551615","act":{"data":{"amount":"1.0000 EOS"}}}}]} +{"type":"ack","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100","sequence":"1"} +``` + +Sequences start at `"1"` and increase by exactly one per data frame in each scope. +A frame is nonempty, contains one mode, and has nondecreasing block numbers. +Blocks cannot move backwards within a generation. `cursor` is optional exact +uint64 metadata, not a durable checkpoint. Arbitrary contract payload is retained; +known uint64 metadata (`global_sequence`, receipt sequences) uses decimal strings, +including values beyond JavaScript's safe integer range. Delta primary/secondary +keys retain their complete existing strings, including compound/wider identities. +Every record requires a real block_id; unavailable identity fails explicitly. +Optional receipt is an object, null, or a SHIP variant `[string, object]`. +Grouped receipts is an array of objects; each receipt's auth_sequence, when +present, is an array of objects with optional exact decimal-string sequence. +Malformed known container shapes fail; arbitrary contract act.data/delta data +and unrelated sequence-named fields remain opaque. + +Both per-stream and per-connection windows bound **unacknowledged** record counts +and UTF-8 encoded bytes of complete data JSON messages (including envelope). +They are hard maxima, not incremental grants. ACK releases exactly that frame's +count/bytes in both budgets. ACKs are once per frame, in receive order; no repeated +or fabricated ACK grants credit. A client sends ACK when the batch is handed to +its caller. This is acceptance, not application durability. A caller that needs +durable recovery saves its own inclusive checkpoint and deduplicates records. + +The server bounds its own unsent queues and history/live overlap too. It must +pause or fail a slow stream without blocking unrelated streams, silently dropping +records, or advancing its replay boundary. Control messages do not consume data +credit, but have bounded sizes, rates and deadlines. The server splits history +pages into bounded nonempty frames, charges its own retained frame's actual bytes +and records, and never trusts an ACK to declare those costs. A single record that +cannot fit the frame cap and both windows fails its stream. Unknown, repeated or +out-of-order ACK never releases credit. Active and lifetime control/subscription +state are bounded. Oversize frames and exhausted +budgets fail explicitly. Initial client maximum WebSocket/JSON message: 1 MiB; +maximum streams: 16; all budgets and deadlines are configurable within local bounds. + +```json +{"type":"history_end","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100"} +{"type":"end","connection_id":"1","stream_id":"1","generation":"1","subscription_id":"100"} +``` + +Every stream emits history_end, including a live-only stream with empty history. +No live data before history_end; no history data after it. The server must buffer +live overlap within a bound or fail/replay, preserving history/live ordering. +history_end describes the server's history phase, not independently qualified +source completeness. end forbids further data; the client drains accepted queued +batches before reporting End. HistoryEnd can be observed while LIB-mode history +is still waiting for irreversibility. End also waits for those batches. + +## Cancellation, failure, heartbeat + +Client cancel uses the full current scope. Server cancelled acknowledges cleanup +of routing ownership, active history search, ACK waits and scroll resources. +Client local cancellation immediately stops delivery; its completion does not +claim the remote acknowledgement has arrived. Cancelling before acceptance uses +`connection_id`, `stream_id`, `generation`, `request_id` with `subscription_id:null`. +The server must cancel that pending attempt too. +For pending cancellation the server may allocate its subscription ID before +accepted, and return cancelled with the full scope and optional original +request_id. No artificial accepted packet is required. Locally cancelled clients +ignore that late acknowledgement; confirmed server cleanup is tested separately. + +```json +{"type":"error","connection_id":"1","stream_id":"1","generation":"1","request_id":"1","code":"HISTORY_UNAVAILABLE","message":"Retained history unavailable","retryable":true} +{"type":"ping","id":"1"} +{"type":"pong","id":"1"} +``` + +Stream errors require stream_id + generation and matching connection_id. Optional +request_id must match the attempt. A connection error has neither stream_id nor +generation. Stream failure clears only that stream and is terminal for that +attempt; callers explicitly decide whether to retry retryable terminal errors. +Transport loss can be retried through reconnect with the original subscriptions. +No automatic infinite retry of protocol/authentication/source errors. + +Either peer can send application ping; the other echoes its id in pong. Native +WebSocket Ping/Pong may supplement this but is not the browser-facing liveness +contract. Handshake, request acceptance, writes, idle connections, irreversible +progress and server cancellation/ACK waits have finite deadlines. A stalled +LIB cannot be hidden by repeated equal LIB notifications or heartbeats. + +## Fork, LIB and recovery + +```json +{"type":"lib","connection_id":"1","chain_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","block_num":100,"block_id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} +{"type":"fork","connection_id":"1","chain_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","starting_block":101,"ending_block":103,"new_block_id":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"} +``` + +These controls also require `connection_id` and are ordered within that connection. +Controls from an old connection are ignored. LIB never regresses; an equal LIB +must have the same block ID. Fork start must be above LIB. Irreversible delivery +buffers entire batches until every record is at/below LIB; control messages must +remain deliverable without data credit. To avoid deadlock, the server sends LIB, +history_end, fork and terminal controls independently of data ACK waits. + +An outage invalidates all unconfirmed records and current server scopes. Fork +also invalidates the affected connection's outstanding delivery buffers. The +client cancels/replaces subscriptions with fresh request IDs/generations, retains +the earliest unread/unconfirmed recovery block across repeated outages, and +replays inclusively. Already accepted head-mode records can be replayed too. +With replay disabled, emit retryable STREAM_REPLAY_REQUIRED. Never confirm old +buffer contents using a LIB from a later connection. Relay replacement/outage +closes the native WebSocket. Reconnect gets a new hello/connection incarnation; +an in-band second hello is invalid. Old public scopes cannot silently stay active. + +The accepted start_from is a real resolved lower bound, never the first matching +row or an optimistic indexed head. A live-only anchor uses the observed router +handoff block inclusively. If a timestamp/relative/live anchor cannot be resolved, +the server rejects retryably. Native history writes complete on admission into +the bounded in-flight window, allowing finality controls to arrive independently +of client-consumption ACKs. A replacement retires old history/routing/ACK ownership +before becoming active. cancelled follows actual history and confirmed router +cleanup, or a terminal retryable cleanup error. + +## Shared conformance + +Language-neutral scenarios under `conformance/v1/` describe server wire inputs, +client actions and literal expected public events/outbound packets. The Rust +driver uses the public core and JSON codec; native WebSocket integration exercises +the real adapter against a second implementation peer. A platform-JSON/BigInt +JavaScript driver validates the same wire corpus without runtime dependencies. +Run limits, stale-scope, completion, fork/LIB, outage, exact uint64, malformed JSON, +request correlation, cancellation and timeout cases on every supported platform. +These fixtures establish protocol/component behavior; actual HAPI retained-history, +filter/proxy/fanout and known-chain recovery acceptance remains owned by #231. diff --git a/protocol/v1/implementation-plan.md b/protocol/v1/implementation-plan.md new file mode 100644 index 0000000..6d50002 --- /dev/null +++ b/protocol/v1/implementation-plan.md @@ -0,0 +1,29 @@ +# Rust native stream client and shared conformance + +Implement the user-approved Rust-first client with a dependency-free core and a +separate Tokio/tokio-tungstenite/Rustls/JSON adapter. The initial transport is +native WebSocket + JSON. This is additive to the legacy Socket.IO client. + +The working wire contract is protocol/v1/README.md. The server lane is +eosrio/hyperion-history-api#294, owned by the architecture task. Changes to the +contract must be coordinated between both lanes and recorded in the shared +conformance corpus. Server source/readiness/filter/cancellation guards and the +existing public/internal streaming routes remain required. + +Deliverables: +- A standalone Cargo core with no third-party dependencies in any feature. +- Typed public state transitions, bounded per-stream/connection delivery windows, + generation/request/connection fencing, exact counters, scoped failures, + cancellation, finite deadlines, inclusive replay and fork/LIB invalidation. +- An optional Tokio adapter with native text WebSocket framing, strict JSON, + bounded frame/write buffers, deadlines, chain/subprotocol verification and TLS. +- Language-neutral scenarios and wire vectors run by Rust and native JavaScript + drivers, plus real WebSocket integration including fault/slow-consumer cases. +- Windows/Linux and supported Rust/Node CI, examples and honest scope/limits docs. +- Review, package checks and a concrete PR. Publication and actual retained-chain + source/fanout/proxy/recovery acceptance remain separate release work under #231. + +Acceptance focuses on discovering bugs: stale scopes, partial/duplicate frames, +sequence gaps, integer precision, queue/byte exhaustion, cancellation races, +request/idle/LIB timeout, repeated outages, fork-before-LIB, cross-stream isolation, +draining completion, malformed JSON and TLS identity validation. diff --git a/scripts/probes/socketio-wire-probe.mjs b/scripts/probes/socketio-wire-probe.mjs new file mode 100644 index 0000000..f23f5b3 --- /dev/null +++ b/scripts/probes/socketio-wire-probe.mjs @@ -0,0 +1,95 @@ +import { createServer } from 'node:http'; +import assert from 'node:assert/strict'; +import { Server } from 'socket.io'; + +// Diagnostic only: show why a native WebSocket cannot use the existing endpoint +// without also speaking Engine.IO and Socket.IO. This is not a client implementation. +const http = createServer(); +const io = new Server(http, { path: '/stream', transports: ['websocket'], connectTimeout: 1000 }); +let requests = 0; +io.on('connection', (socket) => { + socket.emit('handshake', { chain_id: 'test-chain' }); + socket.on('action_stream_request', (_request, ack) => { + requests++; + ack({ status: 'OK', reqUUID: 'request-1' }); + }); +}); +await new Promise((resolve) => http.listen(0, '127.0.0.1', resolve)); +const origin = `ws://127.0.0.1:${http.address().port}`; +const results = []; +try { + for (const mode of ['plain-websocket', 'engineio-with-plain-json', 'minimal-socketio']) { + const result = { mode, opened: false, frames: [], requestsBefore: requests }; + await new Promise((resolve) => { + const endpoint = + mode === 'plain-websocket' ? '/stream/' : '/stream/?EIO=4&transport=websocket'; + const socket = new WebSocket(origin + endpoint); + let finished = false; + const finish = () => { + if (finished) return; + finished = true; + clearTimeout(timer); + socket.close(); + resolve(); + }; + const timer = setTimeout(() => { + result.timedOut = true; + finish(); + }, 2000); + socket.addEventListener('open', () => { + result.opened = true; + }); + socket.addEventListener('error', () => { + result.error = true; + finish(); + }); + socket.addEventListener('close', (event) => { + result.closeCode = event.code; + finish(); + }); + socket.addEventListener('message', (event) => { + const frame = String(event.data); + result.frames.push( + frame.startsWith('0') ? '0{Engine.IO session and heartbeat settings}' : frame + ); + if (frame.startsWith('0')) { + if (mode === 'engineio-with-plain-json') + socket.send( + JSON.stringify({ + op: 'subscribe', + type: 'action', + contract: 'eosio.token', + action: 'transfer' + }) + ); + else socket.send('40'); + } + if (frame.startsWith('42["handshake"')) { + socket.send( + '427' + + JSON.stringify([ + 'action_stream_request', + { contract: 'eosio.token', action: 'transfer' } + ]) + ); + } + if (frame.startsWith('437')) { + result.ack = JSON.parse(frame.slice(3))[0]; + finish(); + } + }); + }); + result.requestsAfter = requests; + results.push(result); + } + assert.equal(results[0].opened, false); + assert.equal(results[0].error, true); + assert.equal(results[1].opened, true); + assert.equal(results[1].requestsAfter, results[1].requestsBefore); + assert.ok(!results[1].timedOut); + assert.deepEqual(results[2].ack, { status: 'OK', reqUUID: 'request-1' }); + assert.equal(requests, 1); + console.log(JSON.stringify({ runtime: process.version, socketIO: '4.8.4', results }, null, 2)); +} finally { + await new Promise((resolve) => io.close(resolve)); +} diff --git a/test/adversarial.test.js b/test/adversarial.test.js index afca601..4d58f8d 100644 --- a/test/adversarial.test.js +++ b/test/adversarial.test.js @@ -70,7 +70,15 @@ test( 'LIB delivery after reconnect never confirms a forked record buffered on the old connection', { timeout: 10000 }, async (t) => { - const f = await fixture(t, { client: { libStream: true }, resend: true }); + const f = await fixture(t, { + client: { libStream: true }, + resend: true, + request({ uuid, ack }) { + const response = { status: 'OK', reqUUID: uuid }; + if (uuid === 'request-1') ack(response); + else setTimeout(() => ack(response), 25); + } + }); await f.client.connect(); const stream = await f.client.streamActions({ ...actionRequest, replayOnReconnect: true }); const delivered = []; @@ -82,7 +90,7 @@ test( messages: [{ ...action(10), block_id: 'orphaned-fork' }] }); f.sockets[0].conn.close(); - await until(() => f.requests.length === 2); + await until(() => f.requests.length === 2 && stream.reqUUID === f.requests[1].uuid); await f.send({ type: 'action_trace', mode: 'history', diff --git a/test/browser.test.mjs b/test/browser.test.mjs index f4594ca..f03a759 100644 --- a/test/browser.test.mjs +++ b/test/browser.test.mjs @@ -46,8 +46,8 @@ for (const format of ['esm', 'umd']) const iterator=stream[Symbol.asyncIterator](); const {value}=await iterator.next(); const {done}=await iterator.return(); + window.disconnectClient=()=>client.disconnect(); window.result={sequence:value.content.global_sequence,block:value.content.block_num,done}; - client.disconnect(); ` } } catch(error) {window.failure=String(error);} @@ -133,6 +133,11 @@ for (const format of ['esm', 'umd']) } : { sequence: '18446744073709551615', block: 42, done: true } ); - if (scenario === 'binary') await until(() => f.cancellations.length === 1); + if (scenario === 'binary') { + // stop/iterator return completes local consumption; remote cancellation + // is asynchronous. Verify it before deliberately closing the transport. + await until(() => f.cancellations.length === 1); + await page.evaluate(() => window.disconnectClient()); + } assert.deepEqual(errors, []); });