Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: MichaelOrlov/mcap
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: main
Choose a base ref
...
head repository: foxglove/mcap
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref
Able to merge. These branches can be automatically merged.
26 contributors
Loading
Showing with 10,480 additions and 7,733 deletions.
  1. +0 −2 .gitattributes
  2. +30 −0 .github/CODEOWNERS
  3. +63 −58 .github/workflows/ci.yml
  4. +8 −8 .github/workflows/website.yml
  5. +0 −3 .gitignore
  6. +1 −0 .swiftlint.yml
  7. +0 −3 .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
  8. +0 −4 .yarnrc.yml
  9. +14 −5 Package.resolved
  10. +3 −3 Package.swift
  11. +3 −3 README.md
  12. +1 −1 cpp/bench/conanfile.py
  13. +1 −1 cpp/build-docs.sh
  14. +1 −1 cpp/build.sh
  15. +1 −1 cpp/docs/conanfile.py
  16. +1 −1 cpp/examples/conanfile.py
  17. +1 −1 cpp/mcap/conanfile.py
  18. +5 −1 cpp/mcap/include/mcap/internal.hpp
  19. +1 −1 cpp/mcap/include/mcap/types.hpp
  20. +20 −7 cpp/mcap/include/mcap/writer.hpp
  21. +29 −5 cpp/mcap/include/mcap/writer.inl
  22. +1 −1 cpp/test/conanfile.py
  23. +7 −5 cpp/test/unit_tests.cpp
  24. +4 −0 cspell.config.yaml
  25. +26 −10 go/cli/mcap/cmd/attachment.go
  26. +28 −24 go/cli/mcap/cmd/cat.go
  27. +301 −0 go/cli/mcap/cmd/du.go
  28. +68 −20 go/cli/mcap/cmd/filter.go
  29. +47 −0 go/cli/mcap/cmd/filter_test.go
  30. +2 −0 go/cli/mcap/cmd/info.go
  31. +3 −0 go/go.work.sum
  32. +90 −0 go/mcap/errors.go
  33. +1 −1 go/mcap/go.mod
  34. +15 −4 go/mcap/indexed_message_iterator.go
  35. +0 −49 go/mcap/lexer.go
  36. +0 −3 go/mcap/mcap.go
  37. +2 −4 go/mcap/reader.go
  38. +2 −2 go/mcap/reader_test.go
  39. +1 −1 go/mcap/version.go
  40. +0 −7 go/mcap/writer.go
  41. +1 −1 package.json
  42. +1 −1 python/mcap-protobuf-support/mcap_protobuf/__init__.py
  43. +22 −2 python/mcap-protobuf-support/mcap_protobuf/decoder.py
  44. +0 −1 python/mcap-protobuf-support/setup.cfg
  45. +20 −0 python/mcap-protobuf-support/tests/generate.py
  46. +21 −0 python/mcap-protobuf-support/tests/proto/test_proto/nested_type_message.proto
  47. +6 −1 python/mcap-protobuf-support/tests/test_decoder.py
  48. +31 −0 python/mcap-protobuf-support/tests/test_proto/nested_type_message_pb2.py
  49. +1 −1 python/mcap-ros1-support/mcap_ros1/__init__.py
  50. +1 −1 python/mcap-ros1-support/mcap_ros1/decoder.py
  51. +1 −2 python/mcap-ros1-support/setup.cfg
  52. +1 −1 python/mcap-ros2-support/mcap_ros2/__init__.py
  53. +1 −1 python/mcap-ros2-support/mcap_ros2/decoder.py
  54. +1 −2 python/mcap-ros2-support/setup.cfg
  55. +1 −1 python/mcap/mcap/__init__.py
  56. +0 −2 python/mcap/mcap/_chunk_builder.py
  57. +57 −21 python/mcap/mcap/_message_queue.py
  58. +1 −1 python/mcap/mcap/data_stream.py
  59. +13 −11 python/mcap/mcap/reader.py
  60. +1 −1 python/mcap/mcap/stream_reader.py
  61. +1 −1 python/mcap/mcap/summary.py
  62. +1 −2 python/mcap/setup.cfg
  63. +29 −5 python/mcap/tests/test_message_queue.py
  64. +34 −12 python/mcap/tests/test_reader.py
  65. +8 −7 rust/Cargo.toml
  66. +3 −1 rust/benches/reader.rs
  67. +26 −29 rust/examples/common/conformance_writer_spec.rs
  68. +1 −1 rust/examples/conformance_reader_async.rs
  69. +78 −44 rust/examples/conformance_writer.rs
  70. +1 −1 rust/examples/mcapcat.rs
  71. +1 −1 rust/examples/mcapcopy.rs
  72. +1 −1 rust/examples/recover.rs
  73. +92 −0 rust/src/chunk_sink.rs
  74. +34 −42 rust/src/io_utils.rs
  75. +37 −9 rust/src/lib.rs
  76. +119 −475 rust/src/read.rs
  77. +23 −0 rust/src/records.rs
  78. +11 −0 rust/src/sans_io.rs
  79. +20 −0 rust/src/sans_io/decompressor.rs
  80. +1,136 −0 rust/src/sans_io/linear_reader.rs
  81. +67 −0 rust/src/sans_io/lz4.rs
  82. +52 −0 rust/src/sans_io/zstd.rs
  83. +2 −6 rust/src/tokio.rs
  84. +75 −0 rust/src/tokio/linear_reader.rs
  85. +0 −146 rust/src/tokio/lz4.rs
  86. +0 −434 rust/src/tokio/read.rs
  87. +0 −103 rust/src/tokio/read_exact_or_zero.rs
  88. +1,305 −344 rust/src/write.rs
  89. +65 −1 rust/tests/attachment.rs
  90. +1 −1 rust/tests/common.rs
  91. +1 −1 rust/tests/compression.rs
  92. +3 −0 rust/tests/data/zstd_chunk_with_padding.mcap
  93. +2 −9 rust/tests/handles_time0_messages.rs
  94. +7 −3 rust/tests/message.rs
  95. +1 −1 rust/tests/metadata.rs
  96. +17 −7 rust/tests/round_trip.rs
  97. +9 −7 swift/conformance/ReadConformanceRunner.swift
  98. +7 −7 swift/crc/CRC32.swift
  99. +6 −6 swift/mcap/MCAPRandomAccessReader.swift
  100. +2 −2 swift/mcap/MCAPStreamedReader.swift
  101. +10 −9 swift/mcap/Records.swift
  102. +1 −1 swift/test/MCAPRandomAccessReaderTests.swift
  103. +2 −2 testdata/bags/demo.bag
  104. +2 −2 testdata/mcap/demo.mcap
  105. +3 −3 tests/conformance/scripts/run-tests/runners/RustWriterTestRunner.ts
  106. +4 −1 typescript/browser/package.json
  107. +1 −1 typescript/core/package.json
  108. +18 −16 typescript/core/src/ChunkCursor.ts
  109. +56 −0 typescript/core/src/McapIndexedReader.test.ts
  110. +70 −89 typescript/core/src/McapIndexedReader.ts
  111. +61 −0 typescript/core/src/McapStreamReader.test.ts
  112. +78 −43 typescript/core/src/McapStreamReader.ts
  113. +6 −6 typescript/core/src/McapWriter.test.ts
  114. +32 −6 typescript/core/src/Reader.ts
  115. +0 −47 typescript/core/src/StreamBuffer.test.ts
  116. +0 −58 typescript/core/src/StreamBuffer.ts
  117. +381 −336 typescript/core/src/parse.ts
  118. +0 −14 typescript/examples/bag2mcap/typings/protobufjs.d.ts
  119. +4 −1 typescript/nodejs/package.json
  120. +3 −2 typescript/support/package.json
  121. +12 −4 typescript/support/src/protobufDescriptors.ts
  122. +0 −14 typescript/support/typings/protobufjs.d.ts
  123. +1 −1 website/docs/guides/cli.md
  124. +4 −0 website/docs/spec/registry.md
  125. +81 −21 website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
  126. +50 −11 website/src/components/McapRecordingDemo/Recorder.ts
  127. +323 −90 website/src/components/McapRecordingDemo/videoCapture.ts
  128. +3 −0 website/src/icons/companies/aescape-dark.svg
  129. +3 −0 website/src/icons/companies/aescape-light.svg
  130. +18 −0 website/src/icons/companies/saronic-dark.svg
  131. +18 −0 website/src/icons/companies/saronic-light.svg
  132. +8 −0 website/src/icons/companies/wayve-dark.svg
  133. +8 −0 website/src/icons/companies/wayve-light.svg
  134. +21 −0 website/src/icons/index.ts
  135. +4,958 −4,955 yarn.lock
2 changes: 0 additions & 2 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
.yarn/**/*.cjs filter=lfs diff=lfs merge=lfs -text
*.bag filter=lfs diff=lfs merge=lfs -text
*.bfbs filter=lfs diff=lfs merge=lfs -text
*.csv linguist-generated=true
@@ -11,4 +10,3 @@ cpp/examples/protobuf/proto/** linguist-vendored=true linguist-generated=true
python/mcap-ros1-support/mcap_ros1/vendor/** linguist-vendored=true
tests/conformance/data/** linguist-generated=true
typescript/examples/flatbuffer/output/** linguist-generated=true
go/ros/testdata/markers.bz2.bag filter=lfs diff=lfs merge=lfs -text
30 changes: 30 additions & 0 deletions .github/CODEOWNERS
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# With a monorepo, it can be hard to know who should review your change. This file defines responsible
# individuals for various packages and files to aid finding reviewers or those with appropriate context.
#
# Updating this file:
# There is no formal policy to update this file. If you feel like you want to "stay updated" on changes to specific
# folders, packages, or glob patterns, make a PR and get a review from someone on the team to add yourself.
#
# NOTE: the file format uses "later match takes precedence" structure so be mindful that glob patterns and other paths you
# add later in the file don't accidentally remove ownership.
#
# File syntax:
# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners

# Any file(s) or path(s) not assigned to someone specific falls back to these owners
* @jtbandes @james-rms

# Sanity check changes to the owners file for syntax and ordering
/.github/CODEOWNERS @defunctzombie @jtbandes @james-rms

/cpp @jtbandes @james-rms @indirectlylit

/go @james-rms

/python @jtbandes @james-rms @indirectlylit

/rust @james-rms @bennetthardwick

/swift @jtbandes

/typescript @jtbandes @achim-k
121 changes: 63 additions & 58 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
@@ -11,9 +11,9 @@ jobs:
spellcheck:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- run: corepack enable
- uses: actions/setup-node@v4
with:
@@ -26,9 +26,9 @@ jobs:
conformance-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- run: corepack enable
- uses: actions/setup-node@v4
with:
@@ -41,7 +41,7 @@ jobs:
conformance-cpp:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -61,7 +61,7 @@ jobs:
conformance-go:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -78,9 +78,9 @@ jobs:
- run: yarn test:conformance --runner go-

conformance-python:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04 # required for python 3.8
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -90,7 +90,7 @@ jobs:
cache: yarn
- uses: actions/setup-python@v5
with:
python-version: 3.7
python-version: 3.8
- run: cd python && pip install -e mcap
- run: yarn install --immutable
- run: yarn test:conformance:generate-inputs --verify
@@ -99,7 +99,7 @@ jobs:
conformance-typescript:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -114,7 +114,7 @@ jobs:
conformance-kaitai-struct:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -127,9 +127,9 @@ jobs:
- run: yarn test:conformance --runner ksy-

conformance-swift:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04 # https://github.com/swift-actions/setup-swift/issues/677
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -148,7 +148,7 @@ jobs:
conformance-rust:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- run: corepack enable
@@ -160,7 +160,7 @@ jobs:
with:
toolchain: stable
default: true
- run: cargo build --example=conformance_reader --example=conformance_reader_async --features=tokio
- run: cargo build --example=conformance_reader --example=conformance_reader_async --example=conformance_writer --features=tokio
working-directory: rust
- run: yarn install --immutable
- run: yarn test:conformance:generate-inputs --verify
@@ -172,9 +172,9 @@ jobs:
run:
working-directory: cpp
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- uses: actions/cache@v4
with:
path: ~/.conan/data
@@ -191,16 +191,16 @@ jobs:
run:
working-directory: cpp
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- uses: actions/cache@v4
with:
path: ~/.conan/data
key: ${{ runner.os }}-${{ hashFiles('cpp/**/conanfile.py') }}
- uses: actions/setup-python@v5
with:
python-version: 3.7
python-version: 3.8
- run: pip install conan~=1.0
- run: bash build.sh --build-tests-only
- run: ./test/build/Debug/bin/unit-tests
@@ -213,9 +213,9 @@ jobs:
id-token: write

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- run: corepack enable
- uses: actions/setup-node@v4
with:
@@ -270,9 +270,9 @@ jobs:
typescript-examples:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- run: corepack enable
- uses: actions/setup-node@v4
with:
@@ -293,14 +293,14 @@ jobs:
- run: yarn workspace @foxglove/mcap-example-text-annotation-demo typecheck

python:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04 # required for python 3.8
defaults:
run:
working-directory: python
permissions:
id-token: write
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- uses: actions/setup-python@v5
@@ -315,73 +315,77 @@ jobs:
uses: pypa/gh-action-pypi-publish@release/v1
if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
with:
packages_dir: python/mcap/dist
repository_url: https://test.pypi.org/legacy/
skip_existing: true
packages-dir: python/mcap/dist
repository-url: https://test.pypi.org/legacy/
skip-existing: true
attestations: false # https://github.com/pypa/gh-action-pypi-publish/issues/283#issuecomment-2499296440

- name: Publish mcap-protobuf-support to TestPyPI
uses: pypa/gh-action-pypi-publish@release/v1
if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
with:
repository_url: https://test.pypi.org/legacy/
skip_existing: true
packages_dir: python/mcap-protobuf-support/dist
repository-url: https://test.pypi.org/legacy/
skip-existing: true
packages-dir: python/mcap-protobuf-support/dist
attestations: false # https://github.com/pypa/gh-action-pypi-publish/issues/283#issuecomment-2499296440

- name: Publish mcap-ros1-support to TestPyPI
uses: pypa/gh-action-pypi-publish@release/v1
if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
with:
repository_url: https://test.pypi.org/legacy/
skip_existing: true
packages_dir: python/mcap-ros1-support/dist
repository-url: https://test.pypi.org/legacy/
skip-existing: true
packages-dir: python/mcap-ros1-support/dist
attestations: false # https://github.com/pypa/gh-action-pypi-publish/issues/283#issuecomment-2499296440

- name: Publish mcap-ros2-support to TestPyPI
uses: pypa/gh-action-pypi-publish@release/v1
if: ${{ !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
with:
repository_url: https://test.pypi.org/legacy/
skip_existing: true
packages_dir: python/mcap-ros2-support/dist
repository-url: https://test.pypi.org/legacy/
skip-existing: true
packages-dir: python/mcap-ros2-support/dist
attestations: false # https://github.com/pypa/gh-action-pypi-publish/issues/283#issuecomment-2499296440

- name: Publish mcap to PyPI
if: |
!github.event.pull_request.head.repo.fork &&
startsWith(github.ref, 'refs/tags/releases/python/mcap/v')
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages_dir: python/mcap/dist
packages-dir: python/mcap/dist

- name: Publish mcap-protobuf-support to PyPI
if: |
!github.event.pull_request.head.repo.fork &&
startsWith(github.ref, 'refs/tags/releases/python/mcap-protobuf-support/v')
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages_dir: python/mcap-protobuf-support/dist
packages-dir: python/mcap-protobuf-support/dist

- name: Publish mcap-ros1-support to PyPI
if: |
!github.event.pull_request.head.repo.fork &&
startsWith(github.ref, 'refs/tags/releases/python/mcap-ros1-support/v')
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages_dir: python/mcap-ros1-support/dist
packages-dir: python/mcap-ros1-support/dist

- name: Publish mcap-ros2-support to PyPI
if: |
!github.event.pull_request.head.repo.fork &&
startsWith(github.ref, 'refs/tags/releases/python/mcap-ros2-support/v')
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages_dir: python/mcap-ros2-support/dist
packages-dir: python/mcap-ros2-support/dist

go:
runs-on: ubuntu-latest
defaults:
run:
working-directory: go
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- uses: actions/setup-go@v5
@@ -439,7 +443,7 @@ jobs:
env: ${{ matrix.env }}

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: "0"
- run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
@@ -464,16 +468,16 @@ jobs:
draft: false

swift:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04 # https://github.com/swift-actions/setup-swift/issues/677
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
lfs: false
- uses: swift-actions/setup-swift@v2
with:
swift-version: "5.7"
- run: docker run -t --rm -v $(pwd):/work -w /work ghcr.io/realm/swiftlint:0.49.1
- run: docker run -t --rm -v $(pwd):/work ghcr.io/nicklockwood/swiftformat:0.49.18 --lint /work
swift-version: "6.0"
- run: docker run -t --rm -v $(pwd):/work -w /work ghcr.io/realm/swiftlint:0.58.2
- run: docker run -t --rm -v $(pwd):/work ghcr.io/nicklockwood/swiftformat:0.55.5 --lint /work
- run: swift build
- run: swift test

@@ -483,7 +487,7 @@ jobs:
run:
working-directory: rust
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
lfs: true
- uses: actions-rs/toolchain@v1
@@ -493,17 +497,18 @@ jobs:
components: "rustfmt, clippy"
- run: rustup target add wasm32-unknown-unknown
- run: cargo fmt --all -- --check
- run: cargo clippy -- --no-deps
- run: cargo clippy --no-default-features -- --no-deps
- run: cargo clippy --no-default-features --features lz4 -- --no-deps
- run: cargo clippy --no-default-features --features zstd -- --no-deps
- run: cargo clippy --no-default-features --features tokio -- --no-deps
- run: cargo clippy --no-default-features --features tokio,lz4 -- --no-deps
- run: cargo clippy --no-default-features --features tokio,zstd -- --no-deps
- run: cargo clippy --all-targets -- --no-deps -D warnings
- run: cargo clippy --no-default-features -- --no-deps -D warnings
- run: cargo clippy --no-default-features --features lz4 -- --no-deps -D warnings
- run: cargo clippy --no-default-features --features zstd -- --no-deps -D warnings
- run: cargo clippy --no-default-features --features tokio -- --no-deps -D warnings
- run: cargo clippy --no-default-features --features tokio,lz4 -- --no-deps -D warnings
- run: cargo clippy --no-default-features --features tokio,zstd -- --no-deps -D warnings
- run: cargo build --all-features
- run: cargo test --all-features
- run: cargo build --all-features --target wasm32-unknown-unknown
- run: cargo check --all-features --target wasm32-unknown-unknown
- run: cargo publish --dry-run
- name: "publish to crates.io"
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/releases/rust/v')
run: cargo publish --token ${{ secrets.RUST_CRATES_IO_TOKEN }}
Loading