From 59a654b343342be6b6b80d4a6cbf3f2ca1114cdc Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Thu, 14 Mar 2024 20:34:02 +0100 Subject: [PATCH 1/2] Add API for deserializing envelopes from JSON --- libbroker/CMakeLists.txt | 1 + libbroker/broker/data_envelope.cc | 5 ++ libbroker/broker/envelope.cc | 37 +++++++++ libbroker/broker/envelope.hh | 4 + libbroker/broker/envelope.test.cc | 127 ++++++++++++++++++++++++++++++ libbroker/broker/error.hh | 2 + 6 files changed, 176 insertions(+) create mode 100644 libbroker/broker/envelope.test.cc diff --git a/libbroker/CMakeLists.txt b/libbroker/CMakeLists.txt index ed4dab7a..51509ea1 100644 --- a/libbroker/CMakeLists.txt +++ b/libbroker/CMakeLists.txt @@ -165,6 +165,7 @@ set(BROKER_TEST_SRC broker/data.test.cc broker/detail/peer_status_map.test.cc broker/domain_options.test.cc + broker/envelope.test.cc broker/error.test.cc broker/filter_type.test.cc broker/format/bin.test.cc diff --git a/libbroker/broker/data_envelope.cc b/libbroker/broker/data_envelope.cc index b064b36c..9fea40cb 100644 --- a/libbroker/broker/data_envelope.cc +++ b/libbroker/broker/data_envelope.cc @@ -5,11 +5,16 @@ #include "broker/error.hh" #include "broker/expected.hh" #include "broker/format/bin.hh" +#include "broker/internal/json.hh" +#include "broker/internal/native.hh" #include "broker/internal/type_id.hh" #include "broker/topic.hh" #include #include +#include +#include +#include using namespace std::literals; diff --git a/libbroker/broker/envelope.cc b/libbroker/broker/envelope.cc index 100d043f..bcb9f35b 100644 --- a/libbroker/broker/envelope.cc +++ b/libbroker/broker/envelope.cc @@ -7,6 +7,7 @@ #include "broker/error.hh" #include "broker/expected.hh" #include "broker/format/bin.hh" +#include "broker/internal/json.hh" #include "broker/internal/logger.hh" #include "broker/internal/type_id.hh" #include "broker/p2p_message_type.hh" @@ -20,6 +21,9 @@ #include #include #include +#include +#include +#include namespace broker { @@ -148,6 +152,39 @@ expected envelope::deserialize(const std::byte* data, } } +expected envelope::deserialize_json(const char* data, + size_t size) { + // Parse the JSON text into a JSON object. + auto val = caf::json_value::parse_shallow(std::string_view{data, size}); + if (!val) + return error{ec::invalid_json}; + auto obj = val->to_object(); + // Type-checking. + if (obj.value("type").to_string() != "data-message") + return error{ec::deserialization_failed}; + // Read the topic. + auto topic = obj.value("topic").to_string(); + if (topic.empty()) + return error{ec::deserialization_failed}; + auto stl_topic = std::string_view{topic.data(), topic.size()}; + // Try to convert the JSON structure into our binary serialization format. + std::vector buf; + buf.reserve(512); // Allocate some memory to avoid small allocations. + if (auto err = internal::json::data_message_to_binary(obj, buf)) + return err; + // Turn the binary data into a data envelope. TTL and sender/receiver are + // not part of the JSON representation, so we use defaults values. + auto res = data_envelope::deserialize(endpoint_id::nil(), endpoint_id::nil(), + defaults::ttl, stl_topic, buf.data(), + buf.size()); + // Note: must manually "unbox" the expected to convert from + // expected to expected. + if (res) + return *res; + else + return res.error(); +} + data_envelope_ptr envelope::as_data() const { BROKER_ASSERT(type() == envelope_type::data); return {new_ref, static_cast(this)}; diff --git a/libbroker/broker/envelope.hh b/libbroker/broker/envelope.hh index ad1d4260..bd458f54 100644 --- a/libbroker/broker/envelope.hh +++ b/libbroker/broker/envelope.hh @@ -80,6 +80,10 @@ public: /// write format. static expected deserialize(const std::byte* data, size_t size); + /// Attempts to deserialize an envelope from the given message in Broker's + /// JSON format. + static expected deserialize_json(const char* data, size_t size); + /// @pre `type == envelope_type::data` data_envelope_ptr as_data() const; diff --git a/libbroker/broker/envelope.test.cc b/libbroker/broker/envelope.test.cc new file mode 100644 index 00000000..bee5df92 --- /dev/null +++ b/libbroker/broker/envelope.test.cc @@ -0,0 +1,127 @@ +#include "broker/data_envelope.hh" + +#include "broker/broker-test.test.hh" + +using namespace broker; +using namespace std::literals; + +namespace { + +// A data message that has one of everything. +constexpr std::string_view json = R"_({ + "type": "data-message", + "topic": "/foo/bar", + "@data-type": "vector", + "data": [ + { + "@data-type": "none", + "data": {} + }, + { + "@data-type": "boolean", + "data": true + }, + { + "@data-type": "count", + "data": 42 + }, + { + "@data-type": "integer", + "data": 23 + }, + { + "@data-type": "real", + "data": 12.48 + }, + { + "@data-type": "string", + "data": "this is a string" + }, + { + "@data-type": "address", + "data": "2001:db8::" + }, + { + "@data-type": "subnet", + "data": "255.255.255.0/24" + }, + { + "@data-type": "port", + "data": "8080/tcp" + }, + { + "@data-type": "timestamp", + "data": "2022-04-10T16:07:00.000" + }, + { + "@data-type": "timespan", + "data": "23s" + }, + { + "@data-type": "enum-value", + "data": "foo" + }, + { + "@data-type": "set", + "data": [ + { + "@data-type": "integer", + "data": 1 + }, + { + "@data-type": "integer", + "data": 2 + }, + { + "@data-type": "integer", + "data": 3 + } + ] + }, + { + "@data-type": "table", + "data": [ + { + "key": { + "@data-type": "string", + "data": "first-name" + }, + "value": { + "@data-type": "string", + "data": "John" + } + }, + { + "key": { + "@data-type": "string", + "data": "last-name" + }, + "value": { + "@data-type": "string", + "data": "Doe" + } + } + ] + } + ] +})_"; + +} // namespace + +TEST(JSON can be deserialized to a data message) { + auto maybe_envelope = envelope::deserialize_json(json.data(), json.size()); + REQUIRE(maybe_envelope); +} + +TEST(an invalid JSON payload results in an error) { + auto no_json = "this is not json!"sv; + auto maybe_envelope = envelope::deserialize_json(no_json.data(), + no_json.size()); + CHECK(!maybe_envelope); +} + +TEST(a JSON payload that does not contain a broker data results in an error) { + std::string_view obj = R"_({"foo": "bar"})_"; + auto maybe_envelope = envelope::deserialize_json(obj.data(), obj.size()); + CHECK(!maybe_envelope); +} diff --git a/libbroker/broker/error.hh b/libbroker/broker/error.hh index 815fa969..1c5cad8c 100644 --- a/libbroker/broker/error.hh +++ b/libbroker/broker/error.hh @@ -103,6 +103,8 @@ enum class ec : uint8_t { redundant_connection, /// Broker encountered a logic_error = 40, + /// Broker failed to parse a JSON object. + invalid_json, }; // --ec-enum-end From 1bc0d93151228865e16f6b836929f19bc9bdc56e Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Sat, 23 Mar 2024 10:37:26 +0100 Subject: [PATCH 2/2] Implement new decode function for the JSON format --- libbroker/broker/envelope.test.cc | 2 +- libbroker/broker/format/json.cc | 32 +++++++ libbroker/broker/format/json.hh | 7 ++ libbroker/broker/format/json.test.cc | 130 +++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 1 deletion(-) diff --git a/libbroker/broker/envelope.test.cc b/libbroker/broker/envelope.test.cc index bee5df92..765d7d95 100644 --- a/libbroker/broker/envelope.test.cc +++ b/libbroker/broker/envelope.test.cc @@ -51,7 +51,7 @@ constexpr std::string_view json = R"_({ }, { "@data-type": "timestamp", - "data": "2022-04-10T16:07:00.000" + "data": "2014-07-09T10:16:44.000" }, { "@data-type": "timespan", diff --git a/libbroker/broker/format/json.cc b/libbroker/broker/format/json.cc index abb72544..59a80434 100644 --- a/libbroker/broker/format/json.cc +++ b/libbroker/broker/format/json.cc @@ -1,5 +1,15 @@ #include "broker/format/json.hh" +#include "broker/defaults.hh" +#include "broker/error.hh" +#include "broker/expected.hh" +#include "broker/internal/json.hh" +#include "broker/variant.hh" + +#include +#include +#include + #include #include #include @@ -40,4 +50,26 @@ size_t encode_to_buf(timestamp value, std::array& buf) { return pos; } +error decode(std::string_view str, variant& result) { + // Parse the JSON text into a JSON object. + auto val = caf::json_value::parse_shallow(str); + if (!val) + return error{ec::invalid_json}; + auto obj = val->to_object(); + // Try to convert the JSON structure into our binary serialization format. + std::vector buf; + buf.reserve(512); // Allocate some memory to avoid small allocations. + if (auto err = internal::json::data_message_to_binary(obj, buf)) + return err; + // Turn the binary data into a data envelope. TTL and sender/receiver are + // not part of the JSON representation, so we use defaults values. + auto res = data_envelope::deserialize(endpoint_id::nil(), endpoint_id::nil(), + defaults::ttl, topic::reserved, + buf.data(), buf.size()); + if (!res) + return res.error(); + result = (*res)->value(); + return {}; +} + } // namespace broker::format::json::v1 diff --git a/libbroker/broker/format/json.hh b/libbroker/broker/format/json.hh index 48688782..0d893959 100644 --- a/libbroker/broker/format/json.hh +++ b/libbroker/broker/format/json.hh @@ -2,6 +2,7 @@ #include "broker/config.hh" #include "broker/data.hh" +#include "broker/fwd.hh" #include "broker/message.hh" #include @@ -385,4 +386,10 @@ OutIter encode(const data_message& msg, OutIter out) { return out; } +/// Tries to decode a JSON object from `str`. On success, the result is stored +/// in `result` and the functions a default-constructed `error`. Otherwise, the +/// function returns a non-empty error and leaves `result` in an unspecified +/// state. +error decode(std::string_view str, variant& result); + } // namespace broker::format::json::v1 diff --git a/libbroker/broker/format/json.test.cc b/libbroker/broker/format/json.test.cc index e6a5549f..654c5b9f 100644 --- a/libbroker/broker/format/json.test.cc +++ b/libbroker/broker/format/json.test.cc @@ -1,6 +1,10 @@ #include "broker/format/json.hh" #include "broker/broker-test.test.hh" +#include "broker/variant.hh" +#include "broker/variant_list.hh" +#include "broker/variant_set.hh" +#include "broker/variant_table.hh" using namespace broker; using namespace std::literals; @@ -588,3 +592,129 @@ TEST(data_message) { CHECK_EQUAL(to_v1(msg), baseline); } } + +TEST(decode JSON into a variant) { + constexpr std::string_view json = R"_({ + "@data-type": "vector", + "data": [ + { + "@data-type": "none", + "data": {} + }, + { + "@data-type": "boolean", + "data": true + }, + { + "@data-type": "count", + "data": 42 + }, + { + "@data-type": "integer", + "data": 23 + }, + { + "@data-type": "real", + "data": 12.48 + }, + { + "@data-type": "string", + "data": "this is a string" + }, + { + "@data-type": "address", + "data": "2001:db8::" + }, + { + "@data-type": "subnet", + "data": "255.255.255.0/24" + }, + { + "@data-type": "port", + "data": "8080/tcp" + }, + { + "@data-type": "timestamp", + "data": "2014-07-09T10:16:44.000" + }, + { + "@data-type": "timespan", + "data": "23s" + }, + { + "@data-type": "enum-value", + "data": "foo" + }, + { + "@data-type": "set", + "data": [ + { + "@data-type": "integer", + "data": 1 + }, + { + "@data-type": "integer", + "data": 2 + }, + { + "@data-type": "integer", + "data": 3 + } + ] + }, + { + "@data-type": "table", + "data": [ + { + "key": { + "@data-type": "string", + "data": "first-name" + }, + "value": { + "@data-type": "string", + "data": "John" + } + }, + { + "key": { + "@data-type": "string", + "data": "last-name" + }, + "value": { + "@data-type": "string", + "data": "Doe" + } + } + ] + } + ] + })_"; + variant res; + auto err = format::json::v1::decode(json, res); + REQUIRE(!err); + auto xs = res.to_list(); + REQUIRE_EQ(xs.size(), 14u); + CHECK(xs.at(0).is_none()); + CHECK_EQ(xs.at(1).to_boolean(), true); + CHECK_EQ(xs.at(2).to_count(), 42u); + CHECK_EQ(xs.at(3).to_integer(), 23); + CHECK_EQ(xs.at(4).to_real(), 12.48); + CHECK_EQ(xs.at(5).to_string(), "this is a string"sv); + CHECK_EQ(xs.at(6).to_address(), addr("2001:db8::")); + CHECK_EQ(xs.at(7).to_subnet(), snet("255.255.255.0/24")); + CHECK_EQ(xs.at(8).to_port(), port(8080, port::protocol::tcp)); + CHECK_EQ(xs.at(9).to_timestamp(), broker_genesis()); + CHECK_EQ(xs.at(10).to_timespan(), timespan{23s}); + CHECK_EQ(xs.at(11).to_enum_value(), enum_value{"foo"}); + auto set = xs.at(12).to_set(); + REQUIRE_EQ(set.size(), 3u); + CHECK(!set.contains(integer{0})); + CHECK(set.contains(integer{1})); + CHECK(set.contains(integer{2})); + CHECK(set.contains(integer{3})); + CHECK(!set.contains(integer{4})); + auto tbl = xs.at(13).to_table(); + REQUIRE_EQ(tbl.size(), 2u); + CHECK_EQ(tbl["first-name"].to_string(), "John"sv); + CHECK_EQ(tbl["last-name"].to_string(), "Doe"sv); +}