Generate Kaitai Struct Format Descriptions and run Kaitai Struct Compiler to generate language bindings (#1336)

Kaitai Struct is a declarative language used to describe binary data structures. KSY files containing format specifications for all supported SBP messages can be generated using the kaitai target. The resulting files can be found in the kaitai/ksy directory. This target also runs the Kaitai Struct compiler to generate bindings for python and perl. These bindings (along with test cases) can be found in the kaitai/python and kaitai/perl  subdirectories respectively.

Due to the nature of Kaitai Struct, the new bindings do not currently allow for serialisation (i.e. construction of SBP objects). Serialisation for java and python targets is supported as an experimental feature in the development version of Kaitai Struct (>0.10) so this restriction should hopefully be removed in the future.

It should also be noted that the field names used by the KSY bindings are not 1:1 identical to the native bindings due constraints which Kaitai Struct places on various token types, e.g. it is forbidden to use upper case characters in field names.
---------
Co-authored-by: Jason Mobarak <[email protected]>

Author: dgburr

.github/workflows/benchmark.yaml

@@ -12,8 +12,11 @@ on:
       - "python/**"
       - "haskell/**"
       - "rust/**"
+      - "kaitai/python/**"
+      - "kaitai/perl/**"
       - scripts/ci_benchmark.bash
       - scripts/ci_benchmark_install.bash
+      - test_data/benchmark_main.py
       - .github/workflows/benchmark.yaml
 jobs:
   benchmark:

Dockerfile

@@ -85,6 +85,8 @@ RUN \
       python3.10 python3.10-dev python3.10-distutils \
       dpkg-dev \
       cmake \
+      libjson-perl \
+      libdigest-crc-perl \
   && curl -sSL https://get.haskellstack.org/ | sh \
   && rm -rf /var/lib/apt/lists/* /tmp/* \
   && curl -s "https://get.sdkman.io" | bash \
@@ -115,6 +117,26 @@ ENV PATH=$NVM_DIR/versions/node/$NODE_VERSION/bin:${PATH}
 
 RUN npm install npm@latest mocha quicktype -g && sudo rm -rf /tmp/*
 
+# install kaitai struct compiler
+RUN \
+     cd /tmp \
+  && curl -LO https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/0.10/kaitai-struct-compiler-0.10.zip \
+  && unzip kaitai-struct-compiler-0.10.zip \
+  && mv kaitai-struct-compiler-0.10 /opt \
+  && ln -sf /opt/kaitai-struct-compiler-0.10/bin/kaitai-struct-compiler /usr/local/bin \
+  && kaitai-struct-compiler --version \
+  && rm -rf /tmp/*
+
+# install perl runtime for kaitai struct
+RUN \
+     cd /tmp \
+  && curl -LO https://github.com/kaitai-io/kaitai_struct_perl_runtime/archive/refs/tags/0.10.zip \
+  && unzip 0.10.zip \
+  && cd kaitai_struct_perl_runtime-0.10 \
+  && perl Makefile.PL \
+  && make install \
+  && rm -rf /tmp/*
+
 ARG UID=1000
 
 # Add a "dockerdev" user with sudo capabilities

Makefile

@@ -19,7 +19,7 @@ SBP_STAGING := $(shell git describe --match 'libsbp-staging*' --always --tags |
 
 CHANGELOG_MAX_ISSUES := 100
 
-.PHONY: help test release dist clean all docs pdf html c deps-c gen-c test-c python deps-python gen-python test-python javascript deps-javascript gen-javascript test-javascript java deps-java gen-java test-java haskell deps-haskell gen-haskell test-haskell haskell deps-protobuf gen-protobuf test-protobuf verify-prereq-generator verify-prereq-c verify-prereq-javascript verify-prereq-python verify-prereq-java verify-prereq-haskell verify-prereq-protobuf mapping rust deps-rust gen-rust test-rust deps-jsonschema gen-jsonschema test-jsonschema verify-prereq-jsonschema deps-quicktype-typescript gen-quicktype-typescript test-quicktype-typescript verify-prereq-quicktype-typescript deps-quicktype-javascript gen-quicktype-javascript test-quicktype-javascript verify-prereq-quicktype-javascript deps-quicktype-elm gen-quicktype-elm test-quicktype-elm verify-prereq-quicktype-elm
+.PHONY: help test release dist clean all docs pdf html c deps-c gen-c test-c python deps-python gen-python test-python javascript deps-javascript gen-javascript test-javascript java deps-java gen-java test-java haskell deps-haskell gen-haskell test-haskell haskell deps-protobuf gen-protobuf test-protobuf deps-kaitai gen-kaitai test-kaitai verify-prereq-generator verify-prereq-c verify-prereq-javascript verify-prereq-python verify-prereq-java verify-prereq-haskell verify-prereq-protobuf verify-prereq-kaitai mapping rust deps-rust gen-rust test-rust deps-jsonschema gen-jsonschema test-jsonschema verify-prereq-jsonschema deps-quicktype-typescript gen-quicktype-typescript test-quicktype-typescript verify-prereq-quicktype-typescript deps-quicktype-javascript gen-quicktype-javascript test-quicktype-javascript verify-prereq-quicktype-javascript deps-quicktype-elm gen-quicktype-elm test-quicktype-elm verify-prereq-quicktype-elm
 
 # Functions
 define announce-begin
@@ -55,6 +55,7 @@ help:
 	@echo "  java         to make Java bindings"
 	@echo "  rust         to make Rust bindings"
 	@echo "  protobuf     to make Protocol Buffer bindings"
+	@echo "  kaitai       to make Kaitai Struct format descriptions"
 	@echo "  jsonschema   to make JSON Schema definitions"
 	@echo "  release      to handle some release tasks"
 	@echo "  test         to run all tests"
@@ -67,7 +68,7 @@ help:
 
 packaged-languages: c python haskell rust javascript
 
-non-packaged-languages: java protobuf jsonschema quicktype
+non-packaged-languages: java protobuf kaitai jsonschema quicktype
 
 all: packaged-languages docs non-packaged-languages
 
@@ -87,6 +88,7 @@ java:       deps-java       gen-java       test-java       java-examples
 haskell:    deps-haskell    gen-haskell    test-haskell
 rust:       deps-rust       gen-rust       test-rust
 protobuf:   deps-protobuf   gen-protobuf   test-protobuf
+kaitai  :   deps-kaitai     gen-kaitai     test-kaitai
 jsonschema: deps-jsonschema gen-jsonschema test-jsonschema
 
 quicktype-typescript: deps-quicktype-typescript gen-quicktype-typescript test-quicktype-typescript
@@ -136,6 +138,9 @@ verify-prereq-rust:
 
 verify-prereq-protobuf: ;
 
+verify-prereq-kaitai:
+	@command -v kaitai-struct-compiler 1>/dev/null 2>/dev/null || { echo >&2 -e "I require \`kaitai-struct-compiler\` but it's not installed. Aborting.\n"; exit 1; }
+
 verify-prereq-jsonschema: ;
 
 verify-prereq-quicktype:
@@ -169,6 +174,8 @@ deps-rust: verify-prereq-rust
 
 deps-protobuf: verify-prereq-protobuf
 
+deps-kaitai: verify-prereq-kaitai
+
 deps-jsonschema: verify-prereq-jsonschema
 
 deps-quicktype-typescript: verify-prereq-quicktype
@@ -179,7 +186,7 @@ deps-quicktype-elm: verify-prereq-quicktype
 
 # Generators
 
-gen: gen-c gen-python gen-javascript gen-java gen-haskell gen-rust gen-protobuf gen-jsonschema gen-quicktype
+gen: gen-c gen-python gen-javascript gen-java gen-haskell gen-rust gen-protobuf gen-kaitai gen-jsonschema gen-quicktype
 gen-quicktype: gen-quicktype-typescript gen-quicktype-elm
 
 gen-c_args = -i $(SBP_SPEC_DIR) \
@@ -298,8 +305,6 @@ gen-rust:
 
 	$(call announce-end,"Finished formatting Rust code")
 
-	$(call announce-end,"Finished generating Rust bindings")
-
 gen-protobuf:
 	$(call announce-begin,"Generating Protocol Buffers bindings")
 	cd $(SWIFTNAV_ROOT)/generator; \
@@ -309,6 +314,31 @@ gen-protobuf:
 					--protobuf
 	$(call announce-begin,"Finished generating Protocol Buffers bindings")
 
+gen-kaitai:
+	$(call announce-begin,"Generating Kaitai Struct Format Description")
+	cd $(SWIFTNAV_ROOT)/generator; \
+	$(SBP_GEN_BIN) -i $(SBP_SPEC_DIR) \
+		-o $(SWIFTNAV_ROOT)/kaitai/ \
+		-r $(SBP_VERSION) \
+		--kaitai
+	$(call announce-begin,"Finished generating Kaitai Struct Format Description")
+
+	$(call announce-begin,"Generating Kaitai Struct Python tests")
+	cd $(SWIFTNAV_ROOT)/generator; \
+	$(SBP_GEN_BIN) -i $(SBP_TESTS_SPEC_DIR) \
+		-o $(SWIFTNAV_ROOT)/kaitai/python/kaitai_sbp/tests/ \
+		-r $(SBP_VERSION) \
+		--test-kaitai-python
+	$(call announce-end,"Finished generating Kaitai Struct Python tests")
+
+	$(call announce-begin,"Generating Kaitai Struct Perl tests")
+	cd $(SWIFTNAV_ROOT)/generator; \
+	$(SBP_GEN_BIN) -i $(SBP_TESTS_SPEC_DIR) \
+		-o $(SWIFTNAV_ROOT)/kaitai/perl/KaitaiSbp/t/ \
+		-r $(SBP_VERSION) \
+		--test-kaitai-perl
+	$(call announce-end,"Finished generating Kaitai Struct Perl tests")
+
 gen-jsonschema:
 	$(call announce-begin,"Generating JSON Schema definitions")
 	cd $(SWIFTNAV_ROOT)/generator; \
@@ -335,7 +365,7 @@ gen-quicktype-elm:
 
 # Testers
 
-test: test-all-begin test-c test-c-v4 test-java test-python test-haskell test-javascript test-rust test-all-end
+test: test-all-begin test-c test-c-v4 test-java test-python test-haskell test-javascript test-rust test-kaitai test-all-end
 
 test-all-begin:
 	$(call announce-begin,"Running all tests")
@@ -395,6 +425,15 @@ test-protobuf:
 	$(call announce-begin,"Running Protocol Buffer tests")
 	$(call announce-end,"Finished running Protocol Buffer tests")
 
+test-kaitai:
+	$(call announce-begin,"Running Kaitai Struct Python tests")
+	cd $(SWIFTNAV_ROOT) && tox -c kaitai/python/kaitai_sbp/tests/tox.ini kaitai/python/kaitai_sbp/tests/test_*.py
+	$(call announce-end,"Finished running Kaitai Struct Python tests")
+
+	$(call announce-begin,"Running Kaitai Struct Perl tests")
+	cd $(SWIFTNAV_ROOT) && perl -MExtUtils::Command::MM -MTest::Harness -e test_harness kaitai/perl/KaitaiSbp/t/*.t
+	$(call announce-end,"Finished running Kaitai Struct Perl tests")
+
 test-jsonschema:
 	$(call announce-begin,"Running JSON Schema tests")
 	$(call announce-end,"Finished running JSON Schema tests")

README.md

@@ -15,6 +15,7 @@
 - [SBP Development Procedures](#sbp-development-procedures)  
 - [SBP Protocol Specification](#sbp-protocol-specification)
 - [JSON Schema Definitions](#json-schema-definitions)
+- [Kaitai Struct Format Descriptions](#kaitai-struct-format-descriptions)
 - [LICENSE](#license)
 
 <!-- tocstop -->
@@ -43,6 +44,7 @@ following directory structure:
 * [`javascript`](./javascript): JavaScript client library and examples.
 * [`rust`](./rust): Rust client library and examples.
 * [`sbpjson`](./sbpjson): Tools for parsing SBP-JSON.
+* [`kaitai`](./kaitai): Kaitai Struct Format Description and generated code.
 
 Except for the `generator`, all of the above are generated and should not be modified directly.
 
@@ -263,8 +265,54 @@ provided.  Libraries for JavaScript, TypeScript, and Elm generated by the
 [QuickType](https://github.com/quicktype/quicktype) tool are provided.  See the
 HOWTO for instructions on updating these schemas.
 
+## Kaitai Struct Format Descriptions
+
+[Kaitai Struct](https://kaitai.io/) is a declarative language used to describe
+binary data structures.  KSY files containing format specifications for all
+supported SBP messages can be generated using the `kaitai` target.  The
+resulting files can be found in the [`kaitai/ksy`](./kaitai/ksy) directory.
+This target also runs the Kaitai Struct compiler to generate bindings for
+perl and python.  These bindings (along with test cases) can be found in the
+[`kaitai/perl`](./kaitai/perl) and [`kaitai/python`](./kaitai/python) subdirectories
+respectively.
+
+The Kaitai Struct format description files can also potentially be used to
+generate bindings for the following targets:
+
+* graphviz
+* csharp
+* rust
+* java
+* go
+* cpp_stl
+* php
+* lua
+* nim
+* html
+* ruby
+* construct
+* javascript
+
+### Notes on python bindings
+
+The python bindings generated by the Kaitai Struct compiler allow for
+significantly faster parsing in comparison to the construct-based bindings
+included in libsbp.  However, due to the nature of Kaitai Struct, these
+bindings do not allow for serialisation (i.e. construction of SBP objects).
+Serialisation for java and python targets is supported as an experimental
+feature in the development version of Kaitai Struct (> 0.10) so this
+restriction should hopefully be removed in the future.
+
+It should also be noted that the names used by the KSY bindings are not 1:1
+identical to the construct bindings due constraints which Kaitai Struct places
+on various token types (e.g. it is forbidden to use upper case characters in
+field names).  The test cases in the [`kaitai/ksy/tests`](./kaitai/ksy/tests) directory
+(particularly those in `test_parsers.py`) can be used as a starting point for
+working with these bindings, as well as demonstrating the minor differences
+between the output generated by each set of bindings.
+
 ## LICENSE
 
-Copyright © 2015-2022 Swift Navigation
+Copyright © 2015-2023 Swift Navigation
 
 Distributed under the [MIT open source license](LICENSE).

generator/sbpg/generator.py

@@ -69,6 +69,15 @@ def get_args():
   parser.add_argument('--protobuf',
                       action="store_true",
                       help='Target language: Protocol Buffers.')
+  parser.add_argument('--kaitai',
+                      action="store_true",
+                      help='Target language: Kaitai Struct Format Description.')
+  parser.add_argument('--test-kaitai-python',
+                      action="store_true",
+                      help='Target language: Kaitai Struct Python tests.')
+  parser.add_argument('--test-kaitai-perl',
+                      action="store_true",
+                      help='Target language: Kaitai Struct Perl tests.')
   parser.add_argument('--jsonschema',
                       action="store_true",
                       help='Target language: JSON Schema.')
@@ -112,7 +121,7 @@ def main():
     # Parse and validate arguments.
     args = get_args().parse_args()
     verbose = args.verbose
-    assert args.jsonschema or args.python or args.javascript or args.c or args.test_c or args.c_sources or args.haskell or args.latex or args.protobuf or args.java or args.test_java or args.rust or args.test_rust, \
+    assert args.jsonschema or args.python or args.javascript or args.c or args.test_c or args.c_sources or args.haskell or args.latex or args.protobuf or args.kaitai or args.test_kaitai_python or args.test_kaitai_perl or args.java or args.test_java or args.rust or args.test_rust, \
         "Please specify a target language."
     input_file = os.path.abspath(args.input_file[0])
     assert len(args.input_file) == 1
@@ -123,7 +132,7 @@ def main():
     assert os.path.exists(output_dir), \
         "Invalid output directory: %s. Exiting!" % output_dir
     # Ingest, parse, and validate.
-    test_mode = args.test_c or args.test_rust or args.test_java
+    test_mode = args.test_c or args.test_kaitai_python or args.test_kaitai_perl or args.test_java or args.test_rust
 
     if test_mode:
       file_index = yaml.resolve_test_deps(*yaml.get_files(input_file))
@@ -184,6 +193,15 @@ def main():
         elif args.protobuf:
           import sbpg.targets.protobuf as pb
           pb.render_source(output_dir, parsed)
+        elif args.kaitai:
+          import sbpg.targets.kaitai as kaitai
+          kaitai.render_source(output_dir, parsed)
+        elif args.test_kaitai_python:
+          import sbpg.targets.test_kaitai_python as test_kaitai_python
+          test_kaitai_python.render_source(output_dir, parsed)
+        elif args.test_kaitai_perl:
+          import sbpg.targets.test_kaitai_perl as test_kaitai_perl
+          test_kaitai_perl.render_source(output_dir, parsed)
         elif args.jsonschema:
           import sbpg.targets.jsonschema as jsonschema
           jsonschema.render_source(output_dir, parsed)
@@ -210,9 +228,11 @@ def main():
       elif args.test_c:
         test_c.render_check_suites(output_dir, all_specs)
         test_c.render_check_main(output_dir, all_specs)
-      if args.test_rust:
+      elif args.test_rust:
         import sbpg.targets.test_rust as test_rs
         test_rs.render_main(output_dir, all_specs)
+      elif args.kaitai:
+        kaitai.render_main(output_dir, all_specs, release)
 
 
   except KeyboardInterrupt:

generator/sbpg/targets/common.py

@@ -10,6 +10,8 @@
 # WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
 
 import base64
+import re
+import json
 
 def string_type(value):
     return type(value) == str
@@ -31,3 +33,38 @@ def to_str(value):
 
 def b64_decode(field):
     return base64.standard_b64decode(field)
+
+# convert CamelCase to snake_case
+def snake_case(s):
+    if "_" in s:
+        return "_".join(snake_case(p) for p in s.split("_"))
+    if len(s) == 1:
+        return s.lower()
+    s = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", s)
+    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s).lower()
+
+# convert snake_case to CamelCase
+def camel_case(s):
+    s = re.sub("([a-z])([A-Z])", r"\1_\2", s)
+    return "".join(w.title() for w in s.split("_"))
+
+# convert all keys in a dict to snake_case
+def snake_case_keys(x):
+    if isinstance(x, list):
+        return [snake_case_keys(v) for v in x]
+    elif isinstance(x, dict):
+        return dict((snake_case(k), snake_case_keys(v)) for k, v in x.items())
+    else:
+        return x
+
+# convert JSON to dict
+def decode_json(x):
+    return json.loads(x)
+
+# Convert dict to canonical JSON representation
+def encode_json(x):
+    return json.dumps(x, allow_nan=False, sort_keys=True, separators=(',', ':'))
+
+# escape non-printable characters
+def str_escape(x):
+    return repr(x)[1:-1]

generator/sbpg/targets/haskell.py

@@ -19,6 +19,7 @@
 from jinja2.utils import pass_environment
 
 from sbpg.targets.templating import JENV, indented_wordwrap
+from sbpg.targets.common import camel_case
 from sbpg import ReleaseVersion
 
 MESSAGES_TEMPLATE_NAME = "SbpMessagesTemplate.hs"
@@ -68,12 +69,6 @@
   'string': 'putByteString $ encodeUtf8',
 }
 
-def camel_case(s):
-  """
-  Convert snake_case to camel_case.
-  """
-  return "".join([i.capitalize() for i in s.split("_")])
-
 def to_global(s):
   """
   Format a global variable name.