diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index 2f9c2f05e..38d897270 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -set(ICEBERG_REST_SOURCES rest_catalog.cc) +set(ICEBERG_REST_SOURCES rest_catalog.cc json_internal.cc) set(ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS) diff --git a/src/iceberg/catalog/rest/json_internal.cc b/src/iceberg/catalog/rest/json_internal.cc new file mode 100644 index 000000000..229001493 --- /dev/null +++ b/src/iceberg/catalog/rest/json_internal.cc @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/json_internal.h" + +#include +#include +#include +#include +#include + +#include + +#include "iceberg/catalog/rest/types.h" +#include "iceberg/json_internal.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_metadata.h" +#include "iceberg/util/json_util_internal.h" +#include "iceberg/util/macros.h" + +namespace iceberg::rest { + +namespace { + +// REST API JSON field constants +constexpr std::string_view kNamespace = "namespace"; +constexpr std::string_view kNamespaces = "namespaces"; +constexpr std::string_view kProperties = "properties"; +constexpr std::string_view kRemovals = "removals"; +constexpr std::string_view kUpdates = "updates"; +constexpr std::string_view kUpdated = "updated"; +constexpr std::string_view kRemoved = "removed"; +constexpr std::string_view kMissing = "missing"; +constexpr std::string_view kNextPageToken = "next-page-token"; +constexpr std::string_view kName = "name"; +constexpr std::string_view kLocation = "location"; +constexpr std::string_view kSchema = "schema"; +constexpr std::string_view kPartitionSpec = "partition-spec"; +constexpr std::string_view kWriteOrder = "write-order"; +constexpr std::string_view kStageCreate = "stage-create"; +constexpr std::string_view kMetadataLocation = "metadata-location"; +constexpr std::string_view kOverwrite = "overwrite"; +constexpr std::string_view kSource = "source"; +constexpr std::string_view kDestination = "destination"; +constexpr std::string_view kMetadata = "metadata"; +constexpr std::string_view kConfig = "config"; +constexpr std::string_view kIdentifiers = "identifiers"; + +} // namespace + +// CreateNamespaceRequest +nlohmann::json ToJson(const CreateNamespaceRequest& request) { + nlohmann::json json; + json[kNamespace] = request.namespace_.levels; + if (!request.properties.empty()) { + json[kProperties] = request.properties; + } + return json; +} + +Result CreateNamespaceRequestFromJson( + const nlohmann::json& json) { + CreateNamespaceRequest request; + ICEBERG_ASSIGN_OR_RAISE(request.namespace_.levels, + GetJsonValue>(json, kNamespace)); + ICEBERG_ASSIGN_OR_RAISE( + request.properties, + (GetJsonValueOrDefault>(json, + kProperties))); + return request; +} + +// UpdateNamespacePropertiesRequest +nlohmann::json ToJson(const UpdateNamespacePropertiesRequest& request) { + // Initialize as an empty object so that when all optional fields are absent we return + // {} instead of null + nlohmann::json json = nlohmann::json::object(); + if (!request.removals.empty()) { + json[kRemovals] = request.removals; + } + if (!request.updates.empty()) { + json[kUpdates] = request.updates; + } + return json; +} + +Result UpdateNamespacePropertiesRequestFromJson( + const nlohmann::json& json) { + UpdateNamespacePropertiesRequest request; + ICEBERG_ASSIGN_OR_RAISE( + request.removals, GetJsonValueOrDefault>(json, kRemovals)); + ICEBERG_ASSIGN_OR_RAISE( + request.updates, + (GetJsonValueOrDefault>(json, + kUpdates))); + return request; +} + +// CreateTableRequest +nlohmann::json ToJson(const CreateTableRequest& request) { + nlohmann::json json; + json[kName] = request.name; + if (!request.location.empty()) { + json[kLocation] = request.location; + } + json[kSchema] = ToJson(*request.schema); + if (request.partition_spec) { + json[kPartitionSpec] = ToJson(*request.partition_spec); + } + if (request.write_order) { + json[kWriteOrder] = ToJson(*request.write_order); + } + SetOptionalField(json, kStageCreate, request.stage_create); + if (!request.properties.empty()) { + json[kProperties] = request.properties; + } + return json; +} + +Result CreateTableRequestFromJson(const nlohmann::json& json) { + CreateTableRequest request; + ICEBERG_ASSIGN_OR_RAISE(request.name, GetJsonValue(json, kName)); + ICEBERG_ASSIGN_OR_RAISE(request.location, + GetJsonValueOrDefault(json, kLocation, "")); + ICEBERG_ASSIGN_OR_RAISE(auto schema_json, GetJsonValue(json, kSchema)); + ICEBERG_ASSIGN_OR_RAISE(request.schema, SchemaFromJson(schema_json)); + if (json.contains(kPartitionSpec)) { + ICEBERG_ASSIGN_OR_RAISE(auto partition_spec_json, + GetJsonValue(json, kPartitionSpec)); + ICEBERG_ASSIGN_OR_RAISE(request.partition_spec, + PartitionSpecFromJson(request.schema, partition_spec_json)); + } else { + request.partition_spec = nullptr; + } + if (json.contains(kWriteOrder)) { + ICEBERG_ASSIGN_OR_RAISE(auto write_order_json, + GetJsonValue(json, kWriteOrder)); + ICEBERG_ASSIGN_OR_RAISE(request.write_order, SortOrderFromJson(write_order_json)); + } else { + request.write_order = nullptr; + } + ICEBERG_ASSIGN_OR_RAISE(request.stage_create, + GetJsonValueOptional(json, kStageCreate)); + ICEBERG_ASSIGN_OR_RAISE( + request.properties, + (GetJsonValueOrDefault>(json, + kProperties))); + return request; +} + +// RegisterTableRequest +nlohmann::json ToJson(const RegisterTableRequest& request) { + nlohmann::json json; + json[kName] = request.name; + json[kMetadataLocation] = request.metadata_location; + if (request.overwrite) { + json[kOverwrite] = request.overwrite; + } + return json; +} + +Result RegisterTableRequestFromJson(const nlohmann::json& json) { + RegisterTableRequest request; + ICEBERG_ASSIGN_OR_RAISE(request.name, GetJsonValue(json, kName)); + ICEBERG_ASSIGN_OR_RAISE(request.metadata_location, + GetJsonValue(json, kMetadataLocation)); + // Default to false if not present + ICEBERG_ASSIGN_OR_RAISE(auto overwrite_opt, + GetJsonValueOptional(json, kOverwrite)); + request.overwrite = overwrite_opt.value_or(false); + return request; +} + +// RenameTableRequest +nlohmann::json ToJson(const RenameTableRequest& request) { + nlohmann::json json; + json[kSource] = ToJson(request.source); + json[kDestination] = ToJson(request.destination); + return json; +} + +Result RenameTableRequestFromJson(const nlohmann::json& json) { + RenameTableRequest request; + ICEBERG_ASSIGN_OR_RAISE(auto source_json, GetJsonValue(json, kSource)); + ICEBERG_ASSIGN_OR_RAISE(request.source, TableIdentifierFromJson(source_json)); + ICEBERG_ASSIGN_OR_RAISE(auto dest_json, + GetJsonValue(json, kDestination)); + ICEBERG_ASSIGN_OR_RAISE(request.destination, TableIdentifierFromJson(dest_json)); + return request; +} + +// LoadTableResult (used by CreateTableResponse, LoadTableResponse) +nlohmann::json ToJson(const LoadTableResult& result) { + nlohmann::json json; + if (!result.metadata_location.empty()) { + json[kMetadataLocation] = result.metadata_location; + } + json[kMetadata] = ToJson(*result.metadata); + if (!result.config.empty()) { + json[kConfig] = result.config; + } + return json; +} + +Result LoadTableResultFromJson(const nlohmann::json& json) { + LoadTableResult result; + ICEBERG_ASSIGN_OR_RAISE(result.metadata_location, GetJsonValueOrDefault( + json, kMetadataLocation, "")); + ICEBERG_ASSIGN_OR_RAISE(auto metadata_json, + GetJsonValue(json, kMetadata)); + ICEBERG_ASSIGN_OR_RAISE(result.metadata, TableMetadataFromJson(metadata_json)); + ICEBERG_ASSIGN_OR_RAISE( + result.config, (GetJsonValueOrDefault>( + json, kConfig))); + return result; +} + +// ListNamespacesResponse +nlohmann::json ToJson(const ListNamespacesResponse& response) { + nlohmann::json json; + if (!response.next_page_token.empty()) { + json[kNextPageToken] = response.next_page_token; + } + nlohmann::json namespaces = nlohmann::json::array(); + for (const auto& ns : response.namespaces) { + namespaces.push_back(ToJson(ns)); + } + json[kNamespaces] = std::move(namespaces); + return json; +} + +Result ListNamespacesResponseFromJson( + const nlohmann::json& json) { + ListNamespacesResponse response; + ICEBERG_ASSIGN_OR_RAISE(response.next_page_token, + GetJsonValueOrDefault(json, kNextPageToken, "")); + ICEBERG_ASSIGN_OR_RAISE(auto namespaces_json, + GetJsonValue(json, kNamespaces)); + for (const auto& ns_json : namespaces_json) { + ICEBERG_ASSIGN_OR_RAISE(auto ns, NamespaceFromJson(ns_json)); + response.namespaces.push_back(std::move(ns)); + } + return response; +} + +// CreateNamespaceResponse +nlohmann::json ToJson(const CreateNamespaceResponse& response) { + nlohmann::json json; + json[kNamespace] = response.namespace_.levels; + if (!response.properties.empty()) { + json[kProperties] = response.properties; + } + return json; +} + +Result CreateNamespaceResponseFromJson( + const nlohmann::json& json) { + CreateNamespaceResponse response; + ICEBERG_ASSIGN_OR_RAISE(response.namespace_.levels, + GetJsonValue>(json, kNamespace)); + ICEBERG_ASSIGN_OR_RAISE( + response.properties, + (GetJsonValueOrDefault>(json, + kProperties))); + return response; +} + +// GetNamespaceResponse +nlohmann::json ToJson(const GetNamespaceResponse& response) { + nlohmann::json json; + json[kNamespace] = response.namespace_.levels; + if (!response.properties.empty()) { + json[kProperties] = response.properties; + } + return json; +} + +Result GetNamespaceResponseFromJson(const nlohmann::json& json) { + GetNamespaceResponse response; + ICEBERG_ASSIGN_OR_RAISE(response.namespace_.levels, + GetJsonValue>(json, kNamespace)); + ICEBERG_ASSIGN_OR_RAISE( + response.properties, + (GetJsonValueOrDefault>(json, + kProperties))); + return response; +} + +// UpdateNamespacePropertiesResponse +nlohmann::json ToJson(const UpdateNamespacePropertiesResponse& response) { + nlohmann::json json; + json[kUpdated] = response.updated; + json[kRemoved] = response.removed; + if (!response.missing.empty()) { + json[kMissing] = response.missing; + } + return json; +} + +Result UpdateNamespacePropertiesResponseFromJson( + const nlohmann::json& json) { + UpdateNamespacePropertiesResponse response; + ICEBERG_ASSIGN_OR_RAISE(response.updated, + GetJsonValue>(json, kUpdated)); + ICEBERG_ASSIGN_OR_RAISE(response.removed, + GetJsonValue>(json, kRemoved)); + ICEBERG_ASSIGN_OR_RAISE( + response.missing, GetJsonValueOrDefault>(json, kMissing)); + return response; +} + +// ListTablesResponse +nlohmann::json ToJson(const ListTablesResponse& response) { + nlohmann::json json; + if (!response.next_page_token.empty()) { + json[kNextPageToken] = response.next_page_token; + } + nlohmann::json identifiers_json = nlohmann::json::array(); + for (const auto& identifier : response.identifiers) { + identifiers_json.push_back(ToJson(identifier)); + } + json[kIdentifiers] = identifiers_json; + return json; +} + +Result ListTablesResponseFromJson(const nlohmann::json& json) { + ListTablesResponse response; + ICEBERG_ASSIGN_OR_RAISE(response.next_page_token, + GetJsonValueOrDefault(json, kNextPageToken, "")); + ICEBERG_ASSIGN_OR_RAISE(auto identifiers_json, + GetJsonValue(json, kIdentifiers)); + for (const auto& id_json : identifiers_json) { + ICEBERG_ASSIGN_OR_RAISE(auto identifier, TableIdentifierFromJson(id_json)); + response.identifiers.push_back(std::move(identifier)); + } + return response; +} + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/json_internal.h b/src/iceberg/catalog/rest/json_internal.h new file mode 100644 index 000000000..3286e1c1d --- /dev/null +++ b/src/iceberg/catalog/rest/json_internal.h @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/catalog/rest/types.h" +#include "iceberg/result.h" + +namespace iceberg::rest { + +/// \brief Serializes a `ListNamespacesResponse` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const ListNamespacesResponse& response); + +/// \brief Deserializes a JSON object into a `ListNamespacesResponse` object. +ICEBERG_REST_EXPORT Result ListNamespacesResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `CreateNamespaceRequest` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const CreateNamespaceRequest& request); + +/// \brief Deserializes a JSON object into a `CreateNamespaceRequest` object. +ICEBERG_REST_EXPORT Result CreateNamespaceRequestFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `CreateNamespaceResponse` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const CreateNamespaceResponse& response); + +/// \brief Deserializes a JSON object into a `CreateNamespaceResponse` object. +ICEBERG_REST_EXPORT Result CreateNamespaceResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `GetNamespaceResponse` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const GetNamespaceResponse& response); + +/// \brief Deserializes a JSON object into a `GetNamespaceResponse` object. +ICEBERG_REST_EXPORT Result GetNamespaceResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes an `UpdateNamespacePropertiesRequest` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson( + const UpdateNamespacePropertiesRequest& request); + +/// \brief Deserializes a JSON object into an `UpdateNamespacePropertiesRequest` object. +ICEBERG_REST_EXPORT Result +UpdateNamespacePropertiesRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes an `UpdateNamespacePropertiesResponse` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson( + const UpdateNamespacePropertiesResponse& response); + +/// \brief Deserializes a JSON object into an `UpdateNamespacePropertiesResponse` object. +ICEBERG_REST_EXPORT Result +UpdateNamespacePropertiesResponseFromJson(const nlohmann::json& json); + +/// \brief Serializes a `ListTablesResponse` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const ListTablesResponse& response); + +/// \brief Deserializes a JSON object into a `ListTablesResponse` object. +ICEBERG_REST_EXPORT Result ListTablesResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `CreateTableRequest` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const CreateTableRequest& request); + +/// \brief Deserializes a JSON object into a `CreateTableRequest` object. +ICEBERG_REST_EXPORT Result CreateTableRequestFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `LoadTableResult` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const LoadTableResult& result); + +/// \brief Deserializes a JSON object into a `LoadTableResult` object. +ICEBERG_REST_EXPORT Result LoadTableResultFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `RegisterTableRequest` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const RegisterTableRequest& request); + +/// \brief Deserializes a JSON object into a `RegisterTableRequest` object. +ICEBERG_REST_EXPORT Result RegisterTableRequestFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `RenameTableRequest` object to JSON. +ICEBERG_REST_EXPORT nlohmann::json ToJson(const RenameTableRequest& request); + +/// \brief Deserializes a JSON object into a `RenameTableRequest` object. +ICEBERG_REST_EXPORT Result RenameTableRequestFromJson( + const nlohmann::json& json); + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/meson.build b/src/iceberg/catalog/rest/meson.build index 9d8a7d384..93247d05d 100644 --- a/src/iceberg/catalog/rest/meson.build +++ b/src/iceberg/catalog/rest/meson.build @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -iceberg_rest_sources = files('rest_catalog.cc') +iceberg_rest_sources = files('json_internal.cc', 'rest_catalog.cc') # cpr does not export symbols, so on Windows it must # be used as a static lib cpr_needs_static = ( @@ -46,4 +46,7 @@ iceberg_rest_dep = declare_dependency( meson.override_dependency('iceberg-rest', iceberg_rest_dep) pkg.generate(iceberg_rest_lib) -install_headers(['rest_catalog.h', 'types.h'], subdir: 'iceberg/catalog/rest') +install_headers( + ['rest_catalog.h', 'types.h', 'json_internal.h'], + subdir: 'iceberg/catalog/rest', +) diff --git a/src/iceberg/catalog/rest/types.h b/src/iceberg/catalog/rest/types.h index 4c50ab268..62b9f7116 100644 --- a/src/iceberg/catalog/rest/types.h +++ b/src/iceberg/catalog/rest/types.h @@ -75,8 +75,8 @@ using PageToken = std::string; /// \brief Result body for table create/load/register APIs. struct ICEBERG_REST_EXPORT LoadTableResult { - std::optional metadata_location; - std::shared_ptr metadata; // required // required + std::string metadata_location; + std::shared_ptr metadata; // required std::unordered_map config; // TODO(Li Feiyang): Add std::shared_ptr storage_credential; }; diff --git a/src/iceberg/json_internal.cc b/src/iceberg/json_internal.cc index 0ad546197..1bad8cc4b 100644 --- a/src/iceberg/json_internal.cc +++ b/src/iceberg/json_internal.cc @@ -37,6 +37,7 @@ #include "iceberg/snapshot.h" #include "iceberg/sort_order.h" #include "iceberg/statistics_file.h" +#include "iceberg/table_identifier.h" #include "iceberg/table_metadata.h" #include "iceberg/transform.h" #include "iceberg/type.h" @@ -73,6 +74,7 @@ constexpr std::string_view kKey = "key"; constexpr std::string_view kValue = "value"; constexpr std::string_view kDoc = "doc"; constexpr std::string_view kName = "name"; +constexpr std::string_view kNamespace = "namespace"; constexpr std::string_view kNames = "names"; constexpr std::string_view kId = "id"; constexpr std::string_view kInitialDefault = "initial-default"; @@ -1147,4 +1149,32 @@ Result> NameMappingFromJson(const nlohmann::json& j return NameMapping::Make(std::move(mapped_fields)); } +nlohmann::json ToJson(const TableIdentifier& identifier) { + nlohmann::json json; + json[kNamespace] = identifier.ns.levels; + json[kName] = identifier.name; + return json; +} + +Result TableIdentifierFromJson(const nlohmann::json& json) { + TableIdentifier identifier; + ICEBERG_ASSIGN_OR_RAISE( + identifier.ns.levels, + GetJsonValueOrDefault>(json, kNamespace)); + ICEBERG_ASSIGN_OR_RAISE(identifier.name, GetJsonValue(json, kName)); + + return identifier; +} + +nlohmann::json ToJson(const Namespace& ns) { return ns.levels; } + +Result NamespaceFromJson(const nlohmann::json& json) { + if (!json.is_array()) [[unlikely]] { + return JsonParseError("Cannot parse namespace from non-array:{}", SafeDumpJson(json)); + } + Namespace ns; + ICEBERG_ASSIGN_OR_RAISE(ns.levels, GetTypedJsonValue>(json)); + return ns; +} + } // namespace iceberg diff --git a/src/iceberg/json_internal.h b/src/iceberg/json_internal.h index d5eb5bcd4..894bc6eb3 100644 --- a/src/iceberg/json_internal.h +++ b/src/iceberg/json_internal.h @@ -327,4 +327,30 @@ ICEBERG_EXPORT nlohmann::json ToJson(const NameMapping& name_mapping); ICEBERG_EXPORT Result> NameMappingFromJson( const nlohmann::json& json); +/// \brief Serializes a `TableIdentifier` object to JSON. +/// +/// \param[in] identifier The `TableIdentifier` object to be serialized. +/// \return A JSON object representing the `TableIdentifier` in the form of key-value +/// pairs. +ICEBERG_EXPORT nlohmann::json ToJson(const TableIdentifier& identifier); + +/// \brief Deserializes a JSON object into a `TableIdentifier` object. +/// +/// \param[in] json The JSON object representing a `TableIdentifier`. +/// \return A `TableIdentifier` object or an error if the conversion fails. +ICEBERG_EXPORT Result TableIdentifierFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `Namespace` object to JSON. +/// +/// \param[in] ns The `Namespace` object to be serialized. +/// \return A JSON array representing the namespace levels. +ICEBERG_EXPORT nlohmann::json ToJson(const Namespace& ns); + +/// \brief Deserializes a JSON array into a `Namespace` object. +/// +/// \param[in] json The JSON array representing a `Namespace`. +/// \return A `Namespace` object or an error if the conversion fails. +ICEBERG_EXPORT Result NamespaceFromJson(const nlohmann::json& json); + } // namespace iceberg diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 68af62bf1..ec07a4347 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -150,7 +150,8 @@ if(ICEBERG_BUILD_BUNDLE) endif() if(ICEBERG_BUILD_REST) - add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc) + add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc + rest_json_internal_test.cc) target_link_libraries(rest_catalog_test PRIVATE iceberg_rest_static) target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR}) endif() diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index 88b16325c..5f5db0900 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -86,7 +86,10 @@ if get_option('rest').enabled() cpp_httplib_dep = dependency('cpp-httplib') iceberg_tests += { 'rest_catalog_test': { - 'sources': files('rest_catalog_test.cc'), + 'sources': files( + 'rest_catalog_test.cc', + 'rest_json_internal_test.cc', + ), 'dependencies': [iceberg_rest_dep, cpp_httplib_dep], }, } diff --git a/src/iceberg/test/rest_json_internal_test.cc b/src/iceberg/test/rest_json_internal_test.cc new file mode 100644 index 000000000..3e7b7ac27 --- /dev/null +++ b/src/iceberg/test/rest_json_internal_test.cc @@ -0,0 +1,664 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "iceberg/catalog/rest/json_internal.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" +#include "iceberg/test/matchers.h" +#include "iceberg/transform.h" + +namespace iceberg::rest { + +namespace { + +// Helper templates for type-specific deserialization +template +Result FromJsonHelper(const nlohmann::json& json); + +template <> +Result FromJsonHelper( + const nlohmann::json& j) { + return CreateNamespaceRequestFromJson(j); +} + +template <> +Result FromJsonHelper( + const nlohmann::json& j) { + return UpdateNamespacePropertiesRequestFromJson(j); +} + +template <> +Result FromJsonHelper(const nlohmann::json& j) { + return CreateTableRequestFromJson(j); +} + +template <> +Result FromJsonHelper( + const nlohmann::json& j) { + return RegisterTableRequestFromJson(j); +} + +template <> +Result FromJsonHelper(const nlohmann::json& j) { + return RenameTableRequestFromJson(j); +} + +template <> +Result FromJsonHelper(const nlohmann::json& j) { + return LoadTableResultFromJson(j); +} + +template <> +Result FromJsonHelper( + const nlohmann::json& j) { + return ListNamespacesResponseFromJson(j); +} + +template <> +Result FromJsonHelper( + const nlohmann::json& j) { + return CreateNamespaceResponseFromJson(j); +} + +template <> +Result FromJsonHelper( + const nlohmann::json& j) { + return GetNamespaceResponseFromJson(j); +} + +template <> +Result +FromJsonHelper(const nlohmann::json& j) { + return UpdateNamespacePropertiesResponseFromJson(j); +} + +template <> +Result FromJsonHelper(const nlohmann::json& j) { + return ListTablesResponseFromJson(j); +} + +// Test helper functions +template +void TestJsonRoundTrip(const T& obj) { + auto j = ToJson(obj); + auto parsed = FromJsonHelper(j); + ASSERT_TRUE(parsed.has_value()) << parsed.error().message; + auto j2 = ToJson(parsed.value()); + EXPECT_EQ(j, j2) << "Round-trip JSON mismatch."; +} + +template +void TestJsonConversion(const T& obj, const nlohmann::json& expected_json) { + auto got = ToJson(obj); + EXPECT_EQ(expected_json, got) << "JSON conversion mismatch."; + + auto parsed = FromJsonHelper(expected_json); + ASSERT_TRUE(parsed.has_value()) << parsed.error().message; + auto back = ToJson(parsed.value()); + EXPECT_EQ(expected_json, back) << "JSON conversion mismatch."; +} + +template +void TestMissingRequiredField(const nlohmann::json& invalid_json, + const std::string& expected_field_name) { + auto result = FromJsonHelper(invalid_json); + EXPECT_FALSE(result.has_value()); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(result, HasErrorMessage("Missing '" + expected_field_name + "'")); +} + +template +void TestUnknownFieldIgnored(const nlohmann::json& minimal_valid) { + auto j = minimal_valid; + j["__unknown__"] = {{"nested", 1}}; + auto parsed = FromJsonHelper(j); + ASSERT_TRUE(parsed.has_value()) << parsed.error().message; + auto out = ToJson(parsed.value()); + EXPECT_FALSE(out.contains("__unknown__")); +} + +std::shared_ptr CreateTableMetadata() { + std::vector schema_fields{ + SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "name", iceberg::string(), true), + SchemaField(3, "ts", iceberg::timestamp(), false), + }; + auto schema = std::make_shared(std::move(schema_fields), /*schema_id=*/1); + + auto spec = std::make_shared( + schema, 1, + std::vector{PartitionField(3, 1000, "ts_day", Transform::Day())}); + + auto order = std::make_shared( + 1, std::vector{SortField(1, Transform::Identity(), + SortDirection::kAscending, NullOrder::kFirst)}); + + return std::make_shared(TableMetadata{ + .format_version = 2, + .table_uuid = "test-uuid-12345", + .location = "s3://bucket/warehouse/table", + .last_sequence_number = 42, + .last_updated_ms = TimePointMsFromUnixMs(1700000000000).value(), + .last_column_id = 3, + .schemas = {schema}, + .current_schema_id = 1, + .partition_specs = {spec}, + .default_spec_id = 1, + .last_partition_id = 1000, + .properties = {{"engine.version", "1.2.3"}, {"write.format.default", "parquet"}}, + .current_snapshot_id = 1234567890, + .snapshots = {std::make_shared( + Snapshot{.snapshot_id = 1234567890, + .sequence_number = 1, + .timestamp_ms = TimePointMsFromUnixMs(1699999999999).value(), + .manifest_list = "s3://bucket/metadata/snap-1234567890.avro", + .summary = {{"operation", "append"}, {"added-files", "10"}}})}, + .snapshot_log = {}, + .metadata_log = {}, + .sort_orders = {order}, + .default_sort_order_id = 1, + .refs = {}, + .statistics = {}, + .partition_statistics = {}, + .next_row_id = 100}); +} + +TableIdentifier CreateTableIdentifier(std::vector ns, std::string name) { + return TableIdentifier{.ns = Namespace{std::move(ns)}, .name = std::move(name)}; +} + +} // namespace + +TEST(JsonRestNewTest, CreateNamespaceRequestFull) { + CreateNamespaceRequest req{ + .namespace_ = Namespace{.levels = {"db1", "schema1"}}, + .properties = std::unordered_map{ + {"owner", "user1"}, {"location", "/warehouse/db1/schema1"}}}; + + nlohmann::json golden = R"({ + "namespace": ["db1","schema1"], + "properties": {"owner":"user1","location":"/warehouse/db1/schema1"} + })"_json; + + TestJsonConversion(req, golden); + TestJsonRoundTrip(req); +} + +TEST(JsonRestNewTest, CreateNamespaceRequestMinimal) { + CreateNamespaceRequest req{.namespace_ = Namespace{.levels = {"db_only"}}, + .properties = {}}; + + nlohmann::json golden = R"({ + "namespace": ["db_only"] + })"_json; + + TestJsonConversion(req, golden); +} + +TEST(JsonRestNewTest, CreateNamespaceRequestMissingNamespace) { + nlohmann::json invalid = R"({ + "properties": {"key": "value"} + })"_json; + + TestMissingRequiredField(invalid, "namespace"); +} + +TEST(JsonRestNewTest, CreateNamespaceRequestIgnoreUnknown) { + nlohmann::json minimal = R"({"namespace":["db"]})"_json; + TestUnknownFieldIgnored(minimal); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesRequestBoth) { + UpdateNamespacePropertiesRequest req{ + .removals = std::vector{"k1", "k2"}, + .updates = std::unordered_map{{"a", "b"}}}; + + nlohmann::json golden = R"({ + "removals": ["k1","k2"], + "updates": {"a":"b"} + })"_json; + + TestJsonConversion(req, golden); + TestJsonRoundTrip(req); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesRequestRemovalsOnly) { + UpdateNamespacePropertiesRequest req{.removals = std::vector{"k"}, + .updates = {}}; + + nlohmann::json golden = R"({ + "removals": ["k"] + })"_json; + + TestJsonConversion(req, golden); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesRequestUpdatesOnly) { + UpdateNamespacePropertiesRequest req{ + .removals = {}, + .updates = std::unordered_map{{"x", "y"}}}; + + nlohmann::json golden = R"({ + "updates": {"x":"y"} + })"_json; + + TestJsonConversion(req, golden); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesRequestEmpty) { + UpdateNamespacePropertiesRequest req{.removals = {}, .updates = {}}; + + nlohmann::json golden = R"({})"_json; + + TestJsonConversion(req, golden); +} + +TEST(JsonRestNewTest, CreateTableRequestFull) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "ts", iceberg::timestamp(), false), + SchemaField(3, "data", iceberg::string(), true)}, + 0); + + auto partition_spec = + std::make_shared(schema, 1, + std::vector{PartitionField( + 2, 1000, "ts_month", Transform::Month())}); + + SortField sort_field(1, Transform::Identity(), SortDirection::kAscending, + NullOrder::kFirst); + auto write_order = std::make_shared(1, std::vector{sort_field}); + + CreateTableRequest request{ + .name = "test_table", + .location = "s3://bucket/warehouse/test_table", + .schema = schema, + .partition_spec = partition_spec, + .write_order = write_order, + .stage_create = true, + .properties = std::unordered_map{ + {"write.format.default", "parquet"}, {"commit.retry.num-retries", "10"}}}; + + TestJsonRoundTrip(request); +} + +TEST(JsonRestNewTest, CreateTableRequestMinimal) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false)}, 0); + + CreateTableRequest request{.name = "minimal_table", + .location = "", + .schema = schema, + .partition_spec = nullptr, + .write_order = nullptr, + .stage_create = std::nullopt, + .properties = {}}; + + TestJsonRoundTrip(request); + + auto json = ToJson(request); + EXPECT_EQ(json["name"], "minimal_table"); + EXPECT_FALSE(json.contains("location")); + EXPECT_TRUE(json.contains("schema")); + EXPECT_FALSE(json.contains("partition-spec")); + EXPECT_FALSE(json.contains("write-order")); + EXPECT_FALSE(json.contains("stage-create")); + EXPECT_FALSE(json.contains("properties")); +} + +TEST(JsonRestNewTest, CreateTableRequestMissingRequiredFields) { + nlohmann::json invalid_json = R"({ + "location": "/tmp/test" + })"_json; + + TestMissingRequiredField(invalid_json, "name"); + + invalid_json = R"({ + "name": "test_table" + })"_json; + + TestMissingRequiredField(invalid_json, "schema"); +} + +TEST(JsonRestNewTest, RegisterTableRequestWithOverwriteTrue) { + RegisterTableRequest r1{ + .name = "t", .metadata_location = "s3://m/v1.json", .overwrite = true}; + + nlohmann::json g1 = R"({ + "name":"t", + "metadata-location":"s3://m/v1.json", + "overwrite": true + })"_json; + + TestJsonConversion(r1, g1); + TestJsonRoundTrip(r1); +} + +TEST(JsonRestNewTest, RegisterTableRequestWithOverwriteFalse) { + RegisterTableRequest r2{ + .name = "t2", .metadata_location = "/tmp/m.json", .overwrite = false}; + + nlohmann::json g2 = R"({ + "name":"t2", + "metadata-location":"/tmp/m.json" + })"_json; + + TestJsonConversion(r2, g2); +} + +TEST(JsonRestNewTest, RegisterTableRequestMissingFields) { + TestMissingRequiredField( + R"({"metadata-location":"s3://m/v1.json"})"_json, "name"); + TestMissingRequiredField(R"({"name":"t"})"_json, + "metadata-location"); +} + +TEST(JsonRestNewTest, RenameTableRequestBasic) { + RenameTableRequest req{.source = CreateTableIdentifier({"old"}, "t0"), + .destination = CreateTableIdentifier({"new", "s"}, "t1")}; + + nlohmann::json golden = R"({ + "source": {"namespace":["old"], "name":"t0"}, + "destination": {"namespace":["new","s"], "name":"t1"} + })"_json; + + TestJsonConversion(req, golden); + TestJsonRoundTrip(req); +} + +TEST(JsonRestNewTest, RenameTableRequestDecodeWithUnknownField) { + nlohmann::json golden = R"({ + "source": {"namespace":["a"], "name":"b"}, + "destination": {"namespace":["x","y"], "name":"z"}, + "__extra__": true + })"_json; + + auto parsed = RenameTableRequestFromJson(golden); + ASSERT_TRUE(parsed.has_value()); + EXPECT_EQ(parsed->source.ns.levels, (std::vector{"a"})); + EXPECT_EQ(parsed->destination.ns.levels, (std::vector{"x", "y"})); +} + +TEST(JsonRestNewTest, RenameTableRequestInvalidNested) { + nlohmann::json invalid = R"({ + "source": { + "namespace": "should be array", + "name": "table" + }, + "destination": { + "namespace": ["db"], + "name": "new_table" + } + })"_json; + + auto result = RenameTableRequestFromJson(invalid); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); +} + +TEST(JsonRestNewTest, LoadTableResultFull) { + LoadTableResult r{.metadata_location = "s3://bucket/metadata/v1.json", + .metadata = CreateTableMetadata(), + .config = std::unordered_map{ + {"catalog-impl", "org.apache.iceberg.rest.RESTCatalog"}, + {"warehouse", "s3://bucket/warehouse"}}}; + + TestJsonRoundTrip(r); + + auto j = ToJson(r); + EXPECT_TRUE(j.contains("metadata-location")); + EXPECT_TRUE(j.contains("metadata")); + EXPECT_TRUE(j.contains("config")); + EXPECT_EQ(j["metadata"]["table-uuid"], "test-uuid-12345"); +} + +TEST(JsonRestNewTest, LoadTableResultMinimal) { + LoadTableResult r{ + .metadata_location = "", .metadata = CreateTableMetadata(), .config = {}}; + + TestJsonRoundTrip(r); + + auto j = ToJson(r); + EXPECT_FALSE(j.contains("metadata-location")); + EXPECT_TRUE(j.contains("metadata")); + EXPECT_FALSE(j.contains("config")); +} + +TEST(JsonRestNewTest, LoadTableResultWithEmptyMetadataLocation) { + LoadTableResult r{ + .metadata_location = "", .metadata = CreateTableMetadata(), .config = {}}; + + auto j = ToJson(r); + EXPECT_FALSE(j.contains("metadata-location")); +} + +TEST(JsonRestNewTest, ListNamespacesResponseWithPageToken) { + ListNamespacesResponse r{ + .next_page_token = "token123", + .namespaces = {Namespace{.levels = {"db1"}}, Namespace{.levels = {"db2", "s1"}}, + Namespace{.levels = {"db3", "s2", "sub"}}}}; + + nlohmann::json golden = R"({ + "next-page-token": "token123", + "namespaces": [ + ["db1"], ["db2","s1"], ["db3","s2","sub"] + ] + })"_json; + + TestJsonConversion(r, golden); + TestJsonRoundTrip(r); +} + +TEST(JsonRestNewTest, ListNamespacesResponseWithoutPageToken) { + ListNamespacesResponse r{ + .next_page_token = "", + .namespaces = {Namespace{.levels = {"db1"}}, Namespace{.levels = {"db2"}}}}; + + nlohmann::json golden = R"({ + "namespaces": [["db1"], ["db2"]] + })"_json; + + TestJsonConversion(r, golden); +} + +TEST(JsonRestNewTest, CreateNamespaceResponseFull) { + CreateNamespaceResponse resp{ + .namespace_ = Namespace{.levels = {"db1", "s1"}}, + .properties = std::unordered_map{{"created", "true"}}}; + + nlohmann::json golden = R"({ + "namespace": ["db1","s1"], + "properties": {"created":"true"} + })"_json; + + TestJsonConversion(resp, golden); + TestJsonRoundTrip(resp); +} + +TEST(JsonRestNewTest, CreateNamespaceResponseMinimal) { + CreateNamespaceResponse resp{.namespace_ = Namespace{.levels = {"db1", "s1"}}, + .properties = {}}; + + nlohmann::json golden = R"({"namespace":["db1","s1"]})"_json; + + TestJsonConversion(resp, golden); +} + +TEST(JsonRestNewTest, CreateNamespaceResponseIgnoreUnknown) { + nlohmann::json minimal = R"({"namespace":["db","s"]})"_json; + TestUnknownFieldIgnored(minimal); +} + +TEST(JsonRestNewTest, GetNamespaceResponseFull) { + GetNamespaceResponse r{.namespace_ = Namespace{.levels = {"prod", "analytics"}}, + .properties = std::unordered_map{ + {"owner", "team-analytics"}, {"retention", "90days"}}}; + + nlohmann::json golden = R"({ + "namespace": ["prod","analytics"], + "properties": {"owner":"team-analytics","retention":"90days"} + })"_json; + + TestJsonConversion(r, golden); + TestJsonRoundTrip(r); +} + +TEST(JsonRestNewTest, GetNamespaceResponseMinimal) { + GetNamespaceResponse r{.namespace_ = Namespace{.levels = {"db"}}, .properties = {}}; + + nlohmann::json golden = R"({"namespace":["db"]})"_json; + + TestJsonConversion(r, golden); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesResponseFull) { + UpdateNamespacePropertiesResponse full{ + .updated = {"u1", "u2"}, .removed = {"r1"}, .missing = {"m1"}}; + + nlohmann::json g_full = R"({ + "updated": ["u1","u2"], + "removed": ["r1"], + "missing": ["m1"] + })"_json; + + TestJsonConversion(full, g_full); + TestJsonRoundTrip(full); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesResponseMinimal) { + UpdateNamespacePropertiesResponse minimal{ + .updated = {"u"}, .removed = {}, .missing = {}}; + + nlohmann::json g_min = R"({ + "updated": ["u"], + "removed": [] + })"_json; + + TestJsonConversion(minimal, g_min); +} + +TEST(JsonRestNewTest, UpdateNamespacePropertiesResponseNoMissing) { + UpdateNamespacePropertiesResponse resp{ + .updated = {"u1"}, .removed = {"r1"}, .missing = {}}; + + nlohmann::json golden = R"({ + "updated": ["u1"], + "removed": ["r1"] + })"_json; + + TestJsonConversion(resp, golden); +} + +TEST(JsonRestNewTest, ListTablesResponseWithPageToken) { + ListTablesResponse r{.next_page_token = "token456", + .identifiers = {CreateTableIdentifier({"db1"}, "t1"), + CreateTableIdentifier({"db2", "s"}, "t2")}}; + + nlohmann::json golden = R"({ + "next-page-token": "token456", + "identifiers": [ + {"namespace":["db1"], "name":"t1"}, + {"namespace":["db2","s"], "name":"t2"} + ] + })"_json; + + TestJsonConversion(r, golden); + TestJsonRoundTrip(r); +} + +TEST(JsonRestNewTest, ListTablesResponseWithoutPageToken) { + ListTablesResponse r{.next_page_token = "", + .identifiers = {CreateTableIdentifier({"db1"}, "t1")}}; + + nlohmann::json golden = R"({ + "identifiers": [ + {"namespace":["db1"], "name":"t1"} + ] + })"_json; + + TestJsonConversion(r, golden); +} + +TEST(JsonRestNewTest, ListTablesResponseEmpty) { + ListTablesResponse r{.next_page_token = "", .identifiers = {}}; + + nlohmann::json golden = R"({ + "identifiers": [] + })"_json; + + TestJsonConversion(r, golden); +} + +TEST(JsonRestNewTest, ListTablesResponseIgnoreUnknown) { + nlohmann::json minimal = R"({"identifiers":[{"namespace":["db"],"name":"t"}]})"_json; + TestUnknownFieldIgnored(minimal); +} + +TEST(JsonRestNewBoundaryTest, EmptyCollections) { + CreateNamespaceRequest request{ + .namespace_ = Namespace{.levels = {"ns"}}, + .properties = std::unordered_map{}}; + + auto json = ToJson(request); + EXPECT_FALSE(json.contains("properties")); + + auto parsed = CreateNamespaceRequestFromJson(json); + ASSERT_TRUE(parsed.has_value()); + EXPECT_TRUE(parsed->properties.empty()); +} + +TEST(JsonRestNewBoundaryTest, InvalidJsonStructure) { + nlohmann::json invalid = nlohmann::json::array(); + + auto result = ListNamespacesResponseFromJson(invalid); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + + invalid = R"({ + "namespaces": "should be array" + })"_json; + + result = ListNamespacesResponseFromJson(invalid); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); +} + +TEST(JsonRestNewBoundaryTest, DeepNestedNamespace) { + std::vector deep_namespace; + for (int i = 0; i < 100; ++i) { + deep_namespace.push_back("level" + std::to_string(i)); + } + + CreateNamespaceRequest req{.namespace_ = Namespace{.levels = deep_namespace}, + .properties = {}}; + + TestJsonRoundTrip(req); +} + +} // namespace iceberg::rest diff --git a/src/iceberg/util/json_util_internal.h b/src/iceberg/util/json_util_internal.h index 81d17b04b..6205ad18c 100644 --- a/src/iceberg/util/json_util_internal.h +++ b/src/iceberg/util/json_util_internal.h @@ -44,6 +44,15 @@ inline std::string SafeDumpJson(const nlohmann::json& json) { nlohmann::detail::error_handler_t::ignore); } +template +Result GetTypedJsonValue(const nlohmann::json& json) { + try { + return json.get(); + } catch (const std::exception& ex) { + return JsonParseError("Failed to parse {}: {}", SafeDumpJson(json), ex.what()); + } +} + template Result GetJsonValueImpl(const nlohmann::json& json, std::string_view key) { try {