diff --git a/be/src/storage/flat_json_config.h b/be/src/storage/flat_json_config.h index 63d03668e31e17..99db73ac69ebb5 100644 --- a/be/src/storage/flat_json_config.h +++ b/be/src/storage/flat_json_config.h @@ -17,12 +17,21 @@ #include #include +#include +#include +#include +#include #include "gen_cpp/AgentService_types.h" namespace starrocks { class FlatJsonConfig { public: + // Default max force-path columns (per JSON column) when not specified by the caller. + static constexpr int DEFAULT_COLUMN_PATHS_MAX = 200; + + using ColumnPathsMap = std::unordered_map>; + // Constructor FlatJsonConfig(); @@ -31,7 +40,8 @@ class FlatJsonConfig { : _flat_json_enable(enable), _flat_json_null_factor(nullFactor), _flat_json_sparsity_factor(sparsityFactor), - _flat_json_max_column_max(maxColumnMax) {} + _flat_json_max_column_max(maxColumnMax), + _flat_json_column_paths_max(DEFAULT_COLUMN_PATHS_MAX) {} // Getters and Setters bool is_flat_json_enabled() const { return _flat_json_enable; } @@ -46,27 +56,96 @@ class FlatJsonConfig { int get_flat_json_max_column_max() const { return _flat_json_max_column_max; } void set_flat_json_max_column_max(int max) { _flat_json_max_column_max = max; } + // Per-JSON-column force-flatten paths (dot-separated, no leading "$."). + const ColumnPathsMap& get_column_paths_map() const { return _flat_json_column_paths; } + + // Returns the path set for the given JSON column, or nullptr if the column has no forced paths. + const std::unordered_set* get_column_paths_for(const std::string& column_name) const { + auto it = _flat_json_column_paths.find(column_name); + return it == _flat_json_column_paths.end() ? nullptr : &it->second; + } + + void set_column_paths_map(ColumnPathsMap paths) { _flat_json_column_paths = std::move(paths); } + + void set_column_paths(const std::string& column_name, const std::vector& paths) { + std::unordered_set s; + for (const auto& p : paths) { + s.insert(p); + } + _flat_json_column_paths[column_name] = std::move(s); + } + + int get_column_paths_max() const { return _flat_json_column_paths_max; } + void set_column_paths_max(int max) { _flat_json_column_paths_max = max; } + void to_pb(FlatJsonConfigPB* binlog_config_pb) { binlog_config_pb->set_flat_json_enable(_flat_json_enable); binlog_config_pb->set_flat_json_null_factor(_flat_json_null_factor); binlog_config_pb->set_flat_json_sparsity_factor(_flat_json_sparsity_factor); binlog_config_pb->set_flat_json_max_column_max(_flat_json_max_column_max); + binlog_config_pb->clear_flat_json_column_paths(); + for (const auto& [col, paths] : _flat_json_column_paths) { + auto* entry = binlog_config_pb->add_flat_json_column_paths(); + entry->set_column_name(col); + for (const auto& p : paths) { + entry->add_paths(p); + } + } + binlog_config_pb->set_flat_json_column_paths_max(_flat_json_column_paths_max); } // Update function using another FlatJsonConfig void update(const FlatJsonConfig& config) { - update(config.is_flat_json_enabled(), config.get_flat_json_null_factor(), - config.get_flat_json_sparsity_factor(), config.get_flat_json_max_column_max()); + _flat_json_enable = config.is_flat_json_enabled(); + _flat_json_null_factor = config.get_flat_json_null_factor(); + _flat_json_sparsity_factor = config.get_flat_json_sparsity_factor(); + _flat_json_max_column_max = config.get_flat_json_max_column_max(); + _flat_json_column_paths = config.get_column_paths_map(); + _flat_json_column_paths_max = config.get_column_paths_max(); } void update(const TFlatJsonConfig& config) { - update(config.flat_json_enable, config.flat_json_null_factor, config.flat_json_sparsity_factor, - config.flat_json_column_max); + _flat_json_enable = config.flat_json_enable; + _flat_json_null_factor = config.flat_json_null_factor; + _flat_json_sparsity_factor = config.flat_json_sparsity_factor; + _flat_json_max_column_max = config.flat_json_column_max; + _flat_json_column_paths.clear(); + if (config.__isset.flat_json_column_paths) { + for (const auto& [col, paths] : config.flat_json_column_paths) { + std::unordered_set s(paths.begin(), paths.end()); + _flat_json_column_paths.emplace(col, std::move(s)); + } + } + if (config.__isset.flat_json_column_paths_max && config.flat_json_column_paths_max > 0) { + _flat_json_column_paths_max = static_cast(config.flat_json_column_paths_max); + } } void update(const FlatJsonConfigPB& flat_json_config_pb) { - update(flat_json_config_pb.flat_json_enable(), flat_json_config_pb.flat_json_null_factor(), - flat_json_config_pb.flat_json_sparsity_factor(), flat_json_config_pb.flat_json_max_column_max()); + _flat_json_enable = flat_json_config_pb.flat_json_enable(); + _flat_json_null_factor = flat_json_config_pb.flat_json_null_factor(); + _flat_json_sparsity_factor = flat_json_config_pb.flat_json_sparsity_factor(); + _flat_json_max_column_max = flat_json_config_pb.flat_json_max_column_max(); + _flat_json_column_paths.clear(); + for (const auto& entry : flat_json_config_pb.flat_json_column_paths()) { + std::unordered_set s; + for (const auto& p : entry.paths()) { + s.insert(p); + } + _flat_json_column_paths.emplace(entry.column_name(), std::move(s)); + } + if (flat_json_config_pb.has_flat_json_column_paths_max() && + flat_json_config_pb.flat_json_column_paths_max() > 0) { + _flat_json_column_paths_max = static_cast(flat_json_config_pb.flat_json_column_paths_max()); + } + } + + // Update function using four parameters (kept for backward compatibility with original API) + void update(bool enable, double nullFactor, double sparsityFactor, int maxColumnMax) { + _flat_json_enable = enable; + _flat_json_null_factor = nullFactor; + _flat_json_sparsity_factor = sparsityFactor; + _flat_json_max_column_max = maxColumnMax; } // Copy Assignment @@ -76,25 +155,35 @@ class FlatJsonConfig { _flat_json_null_factor = other._flat_json_null_factor; _flat_json_sparsity_factor = other._flat_json_sparsity_factor; _flat_json_max_column_max = other._flat_json_max_column_max; + _flat_json_column_paths = other._flat_json_column_paths; + _flat_json_column_paths_max = other._flat_json_column_paths_max; } return *this; } - // Update function using four parameters - void update(bool enable, double nullFactor, double sparsityFactor, int maxColumnMax) { - _flat_json_enable = enable; - _flat_json_null_factor = nullFactor; - _flat_json_sparsity_factor = sparsityFactor; - _flat_json_max_column_max = maxColumnMax; - } - std::string to_string() const { std::ostringstream oss; oss << "FlatJsonConfig{"; oss << "flat_json_enable=" << (_flat_json_enable ? "true" : "false") << ", "; oss << "flat_json_null_factor=" << _flat_json_null_factor << ", "; oss << "flat_json_sparsity_factor=" << _flat_json_sparsity_factor << ", "; - oss << "flat_json_max_column_max=" << _flat_json_max_column_max; + oss << "flat_json_max_column_max=" << _flat_json_max_column_max << ", "; + oss << "flat_json_column_paths={"; + bool first_col = true; + for (const auto& [col, paths] : _flat_json_column_paths) { + if (!first_col) oss << ","; + oss << col << ":["; + bool first_p = true; + for (const auto& p : paths) { + if (!first_p) oss << ","; + oss << p; + first_p = false; + } + oss << "]"; + first_col = false; + } + oss << "}, "; + oss << "flat_json_column_paths_max=" << _flat_json_column_paths_max; oss << "}"; return oss.str(); } @@ -104,5 +193,8 @@ class FlatJsonConfig { double _flat_json_null_factor = 0; double _flat_json_sparsity_factor = 0; int _flat_json_max_column_max = 0; + // Per-JSON-column force-flatten paths: column_name -> set of dot-separated paths (no leading "$."). + ColumnPathsMap _flat_json_column_paths; + int _flat_json_column_paths_max = DEFAULT_COLUMN_PATHS_MAX; }; } // namespace starrocks diff --git a/be/src/storage/lake/tablet_reader.cpp b/be/src/storage/lake/tablet_reader.cpp index 393065c3fdc260..b81d51f4908be1 100644 --- a/be/src/storage/lake/tablet_reader.cpp +++ b/be/src/storage/lake/tablet_reader.cpp @@ -269,7 +269,7 @@ Status TabletReader::init_compaction_column_paths(const TabletReaderParams& read metadata && metadata->has_flat_json_config()) { auto flat_json_config = std::make_shared(); flat_json_config->update(metadata->flat_json_config()); - deriver.init_flat_json_config(flat_json_config.get()); + deriver.init_flat_json_config(flat_json_config.get(), col_name); } deriver.derived(readers); diff --git a/be/src/storage/rowset/json_column_compactor.cpp b/be/src/storage/rowset/json_column_compactor.cpp index 35abb0d5a923dd..6edfa4aafba5d6 100644 --- a/be/src/storage/rowset/json_column_compactor.cpp +++ b/be/src/storage/rowset/json_column_compactor.cpp @@ -46,7 +46,7 @@ Status FlatJsonColumnCompactor::_compact_columns(MutableColumns& json_datas) { vc.emplace_back(js.get()); } deriver.set_generate_filter(true); - deriver.init_flat_json_config(_flat_json_config); + deriver.init_flat_json_config(_flat_json_config, _column_name); deriver.derived(vc); diff --git a/be/src/storage/rowset/json_column_writer.cpp b/be/src/storage/rowset/json_column_writer.cpp index 80dad2d968c201..4d2309a3cadff9 100644 --- a/be/src/storage/rowset/json_column_writer.cpp +++ b/be/src/storage/rowset/json_column_writer.cpp @@ -87,7 +87,7 @@ Status FlatJsonColumnWriter::append(const Column& column) { Status FlatJsonColumnWriter::_flat_column(MutableColumns& json_datas) { // all json datas must full json JsonPathDeriver deriver; - deriver.init_flat_json_config(_flat_json_config); + deriver.init_flat_json_config(_flat_json_config, _column_name); deriver.set_generate_filter(true); std::vector vc; diff --git a/be/src/storage/tablet_reader.cpp b/be/src/storage/tablet_reader.cpp index 56053df95b3a43..ef9869d0ce6a6a 100644 --- a/be/src/storage/tablet_reader.cpp +++ b/be/src/storage/tablet_reader.cpp @@ -187,7 +187,7 @@ Status TabletReader::_init_compaction_column_paths(const TabletReaderParams& rea // must all be flat json type JsonPathDeriver deriver; auto flat_json_config = _tablet->flat_json_config(); - deriver.init_flat_json_config(flat_json_config.get()); + deriver.init_flat_json_config(flat_json_config.get(), col_name); deriver.derived(readers); auto paths = deriver.flat_paths(); auto types = deriver.flat_types(); diff --git a/be/src/util/json_flattener.cpp b/be/src/util/json_flattener.cpp index 8bff1dff24c83c..ffbe8c86f9f3d8 100644 --- a/be/src/util/json_flattener.cpp +++ b/be/src/util/json_flattener.cpp @@ -391,11 +391,19 @@ JsonPathDeriver::JsonPathDeriver(const std::vector& paths, const st } } -void JsonPathDeriver::init_flat_json_config(const FlatJsonConfig* flat_json_config) { +void JsonPathDeriver::init_flat_json_config(const FlatJsonConfig* flat_json_config, const std::string& column_name) { if (flat_json_config != nullptr) { _max_json_null_factor = flat_json_config->get_flat_json_null_factor(); _min_json_sparsity_factory = flat_json_config->get_flat_json_sparsity_factor(); _max_column = flat_json_config->get_flat_json_max_column_max(); + _column_paths.clear(); + if (!column_name.empty()) { + // Load only the paths targeted at this specific JSON column. + if (const auto* paths = flat_json_config->get_column_paths_for(column_name); paths != nullptr) { + _column_paths = *paths; + } + } + _column_paths_max = flat_json_config->get_column_paths_max(); } else { _max_json_null_factor = config::json_flat_null_factor; _min_json_sparsity_factory = config::json_flat_sparsity_factor; @@ -420,6 +428,10 @@ void JsonPathDeriver::derived(const std::vector& json_datas) { _total_rows = res.value(); _path_root = std::make_shared(); + // Pre-mark force paths so _clean_sparsity_path never prunes them. + for (const auto& fp : _column_paths) { + _mark_force_path(fp, _path_root.get()); + } // init path by flat JSON _derived_on_flat_json(json_datas); @@ -565,6 +577,21 @@ void JsonPathDeriver::_derived(const Column* col, size_t mark_row) { } } +// Walk the tree along `path` (dot-separated), creating nodes as needed, and +// mark every node on the path with force=true so they survive early pruning. +void JsonPathDeriver::_mark_force_path(const std::string_view& path, JsonFlatPath* node) { + node->force = true; + if (path.empty()) { + return; + } + auto [key, next] = JsonFlatPath::split_path(path); + auto [iter, inserted] = node->children.try_emplace(key); + if (inserted) { + iter->second = std::make_unique(); + } + _mark_force_path(next, iter->second.get()); +} + void JsonPathDeriver::_clean_sparsity_path(const std::string_view& name, JsonFlatPath* node, size_t check_hits_min) { for (auto& [key, child] : node->children) { _clean_sparsity_path(key, child.get(), check_hits_min); @@ -572,7 +599,8 @@ void JsonPathDeriver::_clean_sparsity_path(const std::string_view& name, JsonFla auto iter = node->children.begin(); while (iter != node->children.end()) { auto child = iter->second.get(); - if (child->hits < check_hits_min) { + // Never prune a node that belongs to a force path. + if (child->hits < check_hits_min && !child->force) { if (_generate_filter) { _remain_keys.insert(iter->first); } @@ -685,7 +713,17 @@ uint32_t JsonPathDeriver::_dfs_finalize(JsonFlatPath* node, const std::string& a bool is_base_type = node->base_type_count >= node->hits - (node->hits * config::json_flat_complex_type_factor); bool type_check = config::enable_json_flat_complex_type || is_base_type; - if (type_check && node->multi_times <= 0 && node->hits >= _total_rows * _min_json_sparsity_factory) { + + // Force paths bypass the sparsity threshold entirely. + // We still skip if multi_times > 0 (ambiguous: same key appears twice in + // one object) or hits == 0 (path never seen in any row). + if (node->force && node->hits > 0 && node->multi_times <= 0) { + hit_leaf->emplace_back(node, absolute_path); + node->type = flat_json::JSON_BITS_TO_LOGICAL_TYPE.at(node->json_type); + node->remain = false; + return 1; + } else if (!node->force && type_check && node->multi_times <= 0 && + node->hits >= _total_rows * _min_json_sparsity_factory) { hit_leaf->emplace_back(node, absolute_path); node->type = flat_json::JSON_BITS_TO_LOGICAL_TYPE.at(node->json_type); node->remain = false; @@ -726,21 +764,51 @@ void JsonPathDeriver::_finalize() { std::vector> hit_leaf; _dfs_finalize(_path_root.get(), "", &hit_leaf); - // sort by name, just for stable order - std::sort(hit_leaf.begin(), hit_leaf.end(), - [&](const auto& a, const auto& b) { return a.first->hits > b.first->hits; }); - size_t limit = _max_column > 0 ? _max_column : std::numeric_limits::max(); - for (size_t i = limit; i < hit_leaf.size(); i++) { - if (!hit_leaf[i].first->remain && hit_leaf[i].first->hits >= _total_rows) { + // Separate forced leaves from normal leaves so each has its own quota. + std::vector> forced_leaves; + std::vector> normal_leaves; + for (auto& item : hit_leaf) { + if (item.first->force) { + forced_leaves.push_back(std::move(item)); + } else { + normal_leaves.push_back(std::move(item)); + } + } + + // Apply _column_paths_max quota to force leaves (sort by hits desc for determinism). + std::sort(forced_leaves.begin(), forced_leaves.end(), + [](const auto& a, const auto& b) { return a.first->hits > b.first->hits; }); + size_t force_limit = (_column_paths_max > 0) + ? static_cast(_column_paths_max) + : std::numeric_limits::max(); + for (size_t i = force_limit; i < forced_leaves.size(); i++) { + forced_leaves[i].first->remain = true; + _has_remain = true; + } + if (forced_leaves.size() > force_limit) { + forced_leaves.resize(force_limit); + } + + // Apply _max_column quota to normal leaves (existing behaviour). + std::sort(normal_leaves.begin(), normal_leaves.end(), + [](const auto& a, const auto& b) { return a.first->hits > b.first->hits; }); + size_t limit = _max_column > 0 ? static_cast(_max_column) : std::numeric_limits::max(); + for (size_t i = limit; i < normal_leaves.size(); i++) { + if (!normal_leaves[i].first->remain && normal_leaves[i].first->hits >= _total_rows) { limit++; continue; } - hit_leaf[i].first->remain = true; + normal_leaves[i].first->remain = true; } - if (hit_leaf.size() > limit) { + if (normal_leaves.size() > limit) { _has_remain |= true; - hit_leaf.resize(limit); + normal_leaves.resize(limit); } + + // Merge back and sort by path name for stable column order. + hit_leaf.clear(); + hit_leaf.insert(hit_leaf.end(), forced_leaves.begin(), forced_leaves.end()); + hit_leaf.insert(hit_leaf.end(), normal_leaves.begin(), normal_leaves.end()); std::sort(hit_leaf.begin(), hit_leaf.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); for (auto& [node, path] : hit_leaf) { node->index = _paths.size(); diff --git a/be/src/util/json_flattener.h b/be/src/util/json_flattener.h index 494fa8521ac565..38bad0b8cf9ba7 100644 --- a/be/src/util/json_flattener.h +++ b/be/src/util/json_flattener.h @@ -68,6 +68,7 @@ class JsonFlatPath { int index = -1; // flat paths array index, only use for leaf, to find column LogicalType type = LogicalType::TYPE_JSON; bool remain = false; + bool force = false; // true: must flatten regardless of sparsity (column_paths) OP op = OP_INCLUDE; // merge flat json use, to mark the path is need FlatJsonHashMap> children; @@ -122,7 +123,10 @@ class JsonPathDeriver { public: JsonPathDeriver(); JsonPathDeriver(const std::vector& paths, const std::vector& types, bool has_remain); - void init_flat_json_config(const FlatJsonConfig* flat_json_config); + // column_name: identifier of the JSON column being derived. Force-flatten paths + // configured for this specific column are applied; paths targeted at other columns + // are ignored. Pass empty string to disable per-column force-flatten. + void init_flat_json_config(const FlatJsonConfig* flat_json_config, const std::string& column_name = ""); ~JsonPathDeriver() = default; @@ -161,6 +165,10 @@ class JsonPathDeriver { // clean sparsity path, to save memory void _clean_sparsity_path(const std::string_view& name, JsonFlatPath* root, size_t check_hits_min); + // Pre-mark all nodes along `path` (dot-separated) with force=true so that + // _clean_sparsity_path and _dfs_finalize never discard them. + void _mark_force_path(const std::string_view& path, JsonFlatPath* node); + private: bool _has_remain = false; std::vector _paths; @@ -170,6 +178,10 @@ class JsonPathDeriver { double _max_json_null_factor = 0; int _max_column = 0; + // column_paths: dot-separated paths that bypass the sparsity check. + std::unordered_set _column_paths; + int _column_paths_max = FlatJsonConfig::DEFAULT_COLUMN_PATHS_MAX; + size_t _total_rows; std::shared_ptr _path_root; diff --git a/be/test/util/json_flattener_test.cpp b/be/test/util/json_flattener_test.cpp index 9db83c60c44eb8..3579e5ef743256 100644 --- a/be/test/util/json_flattener_test.cpp +++ b/be/test/util/json_flattener_test.cpp @@ -15,6 +15,7 @@ #include "util/json_flattener.h" #include +#include "storage/flat_json_config.h" #include #include #include @@ -646,4 +647,171 @@ INSTANTIATE_TEST_SUITE_P(JsonBoolExtractionCases, JsonBoolExtractionTest, std::make_tuple(R"({"bool_field": "1"})", true), std::make_tuple(R"({"bool_field": "0"})", false))); +// column_paths (force-flatten) tests +class JsonPathDeriverForcePathTest : public testing::Test { +public: + // Column identifier used by all tests below; matches cfg.set_column_paths(kJsonColName, ...). + static constexpr const char* kJsonColName = "events"; + + void SetUp() override { + config::enable_json_flat_complex_type = true; + config::json_flat_sparsity_factor = 0.9; + } + void TearDown() override { + config::enable_json_flat_complex_type = false; + config::json_flat_sparsity_factor = 0.3; + } + + JsonPathDeriver derive(const std::vector& jsons, const FlatJsonConfig& cfg, + const std::string& column_name = kJsonColName) { + auto col = JsonColumn::create(); + for (const auto& s : jsons) { + ASSIGN_OR_ABORT(auto v, JsonValue::parse(s)); + col->append(&v); + } + std::vector columns{col.get()}; + JsonPathDeriver jf; + jf.init_flat_json_config(&cfg, column_name); + jf.derived(columns); + return jf; + } +}; + +// FORCE depth 1: sparse k2 (1/3) forced → column +TEST_F(JsonPathDeriverForcePathTest, force_1level_sparse_field) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths(kJsonColName, {"k2"}); + + auto jf = derive({ + R"({"k1": 1, "k2": 42})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2"), paths.end()); +} + +// FORCE depth 2: sparse k2.j1 (1/3) forced → column, dense k2.j2 → normal column +TEST_F(JsonPathDeriverForcePathTest, force_2level_sparse_nested) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths(kJsonColName, {"k2.j1"}); + + auto jf = derive({ + R"({"k1": 1, "k2": {"j1": 10, "j2": 100}})", + R"({"k1": 2, "k2": {"j2": 200}})", + R"({"k1": 3, "k2": {"j2": 300}})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j2"), paths.end()); +} + +// FORCE depth 3: sparse k2.j1.p1 (1/3) forced → column, dense k2.j1.p2 → normal column +TEST_F(JsonPathDeriverForcePathTest, force_3level_sparse_nested) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths(kJsonColName, {"k2.j1.p1"}); + + auto jf = derive({ + R"({"k1": 1, "k2": {"j1": {"p1": 99, "p2": 1}}})", + R"({"k1": 2, "k2": {"j1": {"p2": 2}}})", + R"({"k1": 3, "k2": {"j1": {"p2": 3}}})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1.p1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1.p2"), paths.end()); +} + +// FORCE not in data: force path with hits=0 must not become column +TEST_F(JsonPathDeriverForcePathTest, force_path_not_in_data_is_excluded) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths(kJsonColName, {"k2.j1"}); + + auto jf = derive({ + R"({"k1": 1})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_EQ(std::find(paths.begin(), paths.end(), "k2.j1"), paths.end()); +} + +// FORCE mixed: sparse k2.j1 forced alongside dense k1, k2.j2 +TEST_F(JsonPathDeriverForcePathTest, force_mixed_sparse_and_dense) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths(kJsonColName, {"k2.j1"}); + + auto jf = derive({ + R"({"k1": 1, "k2": {"j1": 10, "j2": 100}})", + R"({"k1": 2, "k2": {"j2": 200}})", + R"({"k1": 3, "k2": {"j2": 300}})", + R"({"k1": 4, "k2": {"j2": 400}})", + R"({"k1": 5, "k2": {"j2": 500}})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j2"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1"), paths.end()); +} + +// FORCE quota: column_paths_max=2 caps 3 force paths → excess goes to remain +TEST_F(JsonPathDeriverForcePathTest, force_paths_max_quota) { + FlatJsonConfig cfg(true, 0.3, 0.0, 100); + cfg.set_column_paths(kJsonColName, {"k1", "k2", "k3"}); + cfg.set_column_paths_max(2); + + auto jf = derive({ + R"({"k1": 1, "k2": 2, "k3": 3})", + R"({"k1": 4, "k2": 5, "k3": 6})", + R"({"k1": 7, "k2": 8, "k3": 9})", + }, cfg); + + auto paths = jf.flat_paths(); + size_t force_count = 0; + for (const auto& p : {"k1", "k2", "k3"}) { + if (std::find(paths.begin(), paths.end(), p) != paths.end()) force_count++; + } + EXPECT_LE(force_count, 2u); + EXPECT_TRUE(jf.has_remain_json()); +} + +// Per-column scope: paths configured for a different JSON column must NOT leak +// into this column. With column "other" forced and column "events" derived, +// sparse "k2" in "events" should fall back to sparsity rules (pruned here). +TEST_F(JsonPathDeriverForcePathTest, per_column_scope_isolation) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths("other", {"k2"}); // forced for a different JSON column + + auto jf = derive({ + R"({"k1": 1, "k2": 42})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg, "events"); + + auto paths = jf.flat_paths(); + // k1 is dense → normal column; k2 is sparse (1/3) and NOT force-scoped for "events" + EXPECT_NE(std::find(paths.begin(), paths.end(), "k1"), paths.end()); + EXPECT_EQ(std::find(paths.begin(), paths.end(), "k2"), paths.end()); +} + +// Empty column_name disables per-column force even if the map has entries. +TEST_F(JsonPathDeriverForcePathTest, empty_column_name_disables_force) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths("events", {"k2"}); + + auto jf = derive({ + R"({"k1": 1, "k2": 42})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg, /*column_name=*/""); + + auto paths = jf.flat_paths(); + EXPECT_EQ(std::find(paths.begin(), paths.end(), "k2"), paths.end()); +} + } // namespace starrocks diff --git a/docs/en/using_starrocks/Flat_json_column_paths.md b/docs/en/using_starrocks/Flat_json_column_paths.md new file mode 100644 index 00000000000000..1d295242e7453a --- /dev/null +++ b/docs/en/using_starrocks/Flat_json_column_paths.md @@ -0,0 +1,314 @@ +--- +displayed_sidebar: docs +sidebar_position: 111 +--- + +# Flat JSON Force-Flatten Paths (`column_paths`) + +Flat JSON automatically extracts JSON fields that appear densely across rows (see +[Flat JSON](./Flat_json.md)). The sparsity heuristic works well for typical +payloads, but some fields are important for queries even when they appear in +only a small fraction of rows. `flat_json.column_paths` lets you explicitly +force-flatten a list of JSON paths for a specific JSON column, bypassing the +sparsity check. + +This page describes the property syntax, how to set and update it, and how to +verify that the paths you specified were actually materialized as sub-columns. + +## When to use + +Use `column_paths` when a field is: + +- **Queried frequently** but absent from most rows (below the sparsity threshold). +- **Used in predicates or joins** that benefit from columnar access. +- **Part of a low-cardinality dictionary** path that would otherwise be skipped. + +Do NOT use it to force-flatten dense fields — those are picked up automatically +by the sparsity heuristic and consume the `flat_json.column.max` quota. + +## Syntax + +`column_paths` is scoped **per JSON column**. The property key encodes the JSON +column name: + +``` +flat_json.column_paths. = ", , ..." +flat_json.column_paths..add = ", " (ALTER only) +flat_json.column_paths..remove = "" (ALTER only) +flat_json.column_paths_max = (per-column cap) +``` + +- `` is the identifier of a JSON column in the table schema. + If the column does not exist or is not of JSON type, the DDL fails during analysis. +- Paths are comma-separated. Each path may start with `$.` (optional; stripped + at parse time). Nested paths use `.` as separator: `$.user.country`. +- The reserved suffixes `.add` and `.remove` are only valid inside + `ALTER TABLE ... SET (...)` — not at `CREATE TABLE` time. +- `flat_json.column_paths_max` caps the number of force-flattened columns **per + JSON column**, independent of `flat_json.column.max` (which caps + sparsity-derived columns). Default is `200`. + +## Prerequisites + +Flat JSON must be enabled on the table before any `column_paths` key is +accepted: + +```sql +ALTER TABLE t SET ("flat_json.enable" = "true"); +``` + +Attempting to set `column_paths` when Flat JSON is disabled raises a semantic +error. + +## Configure at `CREATE TABLE` time + +```sql +CREATE TABLE user_events ( + id BIGINT, + ts DATETIME, + events JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.null.factor" = "0.3", + "flat_json.sparsity.factor" = "0.5", + "flat_json.column.max" = "50", + -- Force-flatten events.browser and events.utm_source regardless of sparsity. + "flat_json.column_paths.events" = "$.browser, $.utm_source", + -- Per-column cap on force paths (optional; default is 200). + "flat_json.column_paths_max" = "20" +); +``` + +Multiple JSON columns each get their own entry: + +```sql +CREATE TABLE logs ( + id BIGINT, + request JSON, + response JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.column_paths.request" = "$.method, $.path", + "flat_json.column_paths.response" = "$.status, $.latency_ms" +); +``` + +## Update via `ALTER TABLE` + +Three operations are supported per JSON column. They can be combined in one +`ALTER` statement. + +### Full replace + +Overwrites the complete path list for a column. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events" = "$.browser, $.country, $.utm_source" +); +``` + +Setting the value to an empty string clears all force paths for that column +(the column falls back to pure sparsity-based flattening): + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths.events" = ""); +``` + +### Incremental add + +Appends paths not already in the list. Existing paths are preserved. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.add" = "$.device_id, $.session_id" +); +``` + +### Incremental remove + +Removes matching paths. Paths not in the current list are silently ignored. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.remove" = "$.utm_source" +); +``` + +### Global per-column cap + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths_max" = "50"); +``` + +Setting it to `0` resets to the server default (`200`). + +### Combined example + +```sql +ALTER TABLE logs SET ( + "flat_json.column_paths.request.add" = "$.user_agent", + "flat_json.column_paths.response.remove" = "$.latency_ms", + "flat_json.column_paths_max" = "100" +); +``` + +## Interaction with sparsity + +| Path state | Behavior | +|-----------------------------|-------------------------------------------------------------------| +| Listed in `column_paths` | Forced to a sub-column regardless of sparsity (if it has `hits > 0`). | +| Listed but never appears | Excluded (the engine does not create empty sub-columns). | +| Not listed, above sparsity | Auto-flattened via normal Flat JSON heuristic. | +| Not listed, below sparsity | Stays inside the remainder JSON blob. | + +The two quotas are independent: +- `flat_json.column.max` caps **auto-detected** sparse-flattened columns. +- `flat_json.column_paths_max` caps **forced** columns. + +## Verification: step-by-step + +Follow these steps to confirm that a configured path was actually materialized +as a sub-column. + +### Step 1. Enable Flat JSON and configure paths + +```sql +CREATE TABLE user_events ( + id BIGINT, ts DATETIME, events JSON +) DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.sparsity.factor" = "0.9", + "flat_json.column_paths.events" = "$.browser" +); +``` + +`sparsity.factor = 0.9` is set high to ensure that ordinary sparsity would NOT +flatten `$.browser`. This way, if `browser` shows up as a sub-column, it is +proof that `column_paths` took effect. + +### Step 2. Load data where the forced path is sparse + +```sql +INSERT INTO user_events VALUES + (1, now(), parse_json('{"session":"a","browser":"Chrome"}')), + (2, now(), parse_json('{"session":"b"}')), + (3, now(), parse_json('{"session":"c"}')), + (4, now(), parse_json('{"session":"d"}')), + (5, now(), parse_json('{"session":"e"}')); +``` + +`browser` appears in 1/5 rows (sparsity 20%), well below the 90% threshold. +Without `column_paths` it would NOT be flattened. + +### Step 3. Inspect materialized sub-columns + +Use `flat_json_meta` on the `[_META_]` pseudo-table: + +```sql +SELECT flat_json_meta(events) FROM user_events[_META_]; +``` + +Expected output (keys in any order): + +``` ++-------------------------------------------------------+ +| flat_json_meta(events) | ++-------------------------------------------------------+ +| ["session(VARCHAR)", "browser(VARCHAR)"] | ++-------------------------------------------------------+ +``` + +Seeing `browser(VARCHAR)` in the list confirms the force-flatten happened. + +If the table has multiple JSON columns, query each one individually: + +```sql +SELECT flat_json_meta(request) FROM logs[_META_]; +SELECT flat_json_meta(response) FROM logs[_META_]; +``` + +### Step 4. Verify query path usage via Query Profile + +Run a query that touches the forced path: + +```sql +SELECT get_json_string(events, '$.browser') +FROM user_events +WHERE get_json_string(events, '$.browser') = 'Chrome'; +``` + +Then inspect the profile (enable with `SET enable_profile = true;` then +`SHOW PROFILELIST;` and `ANALYZE PROFILE FROM ''`). Look for: + +- `PushdownAccessPaths`: non-zero — the planner pushed the path access down to storage. +- `AccessPathHits`: non-zero — the sub-column was read directly instead of going through JSON. +- `/events: ` under `AccessPathHits` — your JSON column appears as a hit entry. + +If `AccessPathHits` is zero but `AccessPathUnhits` is non-zero, the path was +pushed down but the sub-column did not exist for that rowset (typical right +after enabling the feature — new data is flattened but pre-existing segments +are not). Trigger compaction or reload the data to re-flatten historical rows. + +### Step 5. Force compaction for existing data (optional) + +New data is flattened at load time. For data that was already in the table +before you set `column_paths`, you can trigger compaction to re-flatten: + +```sql +ALTER TABLE user_events COMPACT; +``` + +Watch the BE log for lines like: + +``` +Compaction flat json column: nulls(TINYINT),browser(VARCHAR),session(VARCHAR) +``` + +Repeat Step 3 to confirm `browser` is now present. + +## Troubleshooting + +| Symptom | Cause | Fix | +|------------------------------------------------------------|--------------------------------------------------------------------------|----------------------------------------------------------------------------| +| `Property 'flat_json.column_paths.foo' references unknown or non-JSON column 'foo'` | Column `foo` doesn't exist or isn't `JSON`. | Use the correct JSON column name; check `DESC `. | +| `flat JSON configuration must be set after enabling flat JSON.` | Tried to set `column_paths` on a table with `flat_json.enable = false`. | `ALTER TABLE t SET ("flat_json.enable" = "true");` first. | +| `flat_json_meta` does not list a configured path | The path does not appear in ANY row yet (`hits = 0`). | Load data containing the path, or wait until such rows arrive. | +| `AccessPathHits = 0` but sub-column exists | Query reads old rowsets predating the config change. | Run `ALTER TABLE t COMPACT;` to re-flatten. | +| Configured paths exceed the quota | `flat_json.column_paths_max` reached for that JSON column. | Raise the cap, or prune the list with `.remove`. | +| Sync appears wrong on follower FE after removing paths | Older builds had an emit-only-if-non-empty bug. | This page's design emits the full state on every write (fixed). | + +## Metadata and cluster-wide consistency + +`column_paths` configuration is persisted via the FE EditLog +(`OP_MODIFY_FLAT_JSON_CONFIG`). Every FE in the cluster replays the same log, +so the leader and followers converge to the same configuration. The config is +also pushed to every BE as part of the tablet meta and applied on the next +flush/compaction. + +When debugging divergence, check: + +```sql +-- View current effective config +SHOW CREATE TABLE user_events; +``` + +The `PROPERTIES` block will list every `flat_json.column_paths.` entry +that is currently active. Compare this output between FE replicas; it must +match. + +## Related + +- [Flat JSON overview](./Flat_json.md) +- [CREATE TABLE reference](../sql-reference/sql-statements/table_bucket_part_index/CREATE_TABLE.md) +- [ALTER TABLE reference](../sql-reference/sql-statements/table_bucket_part_index/ALTER_TABLE.md) +- BE config: `json_flat_null_factor`, `json_flat_sparsity_factor`, + `json_flat_column_max`, `enable_compaction_flat_json` diff --git a/docs/zh/using_starrocks/Flat_json_column_paths.md b/docs/zh/using_starrocks/Flat_json_column_paths.md new file mode 100644 index 00000000000000..720f99bf76738d --- /dev/null +++ b/docs/zh/using_starrocks/Flat_json_column_paths.md @@ -0,0 +1,305 @@ +--- +displayed_sidebar: docs +sidebar_position: 111 +--- + +# Flat JSON 강제 평탄화 경로 (`column_paths`) + +Flat JSON은 여러 행에 걸쳐 빈번하게 등장하는 JSON 필드를 자동으로 별도 컬럼으로 +추출합니다 ([Flat JSON](./Flat_json.md) 참조). 희소성(sparsity) 휴리스틱은 대부분의 +페이로드에 잘 동작하지만, 일부 필드는 소수의 행에만 등장해도 쿼리 성능에 중요할 수 +있습니다. `flat_json.column_paths`는 **특정 JSON 컬럼**에 대해 지정한 경로들을 희소성 +체크를 무시하고 **강제로 평탄화**하도록 설정하는 기능입니다. + +이 문서는 해당 프로퍼티의 문법, 설정/변경 방법, 그리고 **지정한 경로가 실제로 +서브컬럼으로 materialize 되었는지 검증하는 단계별 방법**을 다룹니다. + +## 언제 사용하나 + +다음과 같은 필드에 사용하세요: + +- **자주 쿼리**되지만 **대부분의 행에는 없는** 필드 (희소성 임계값 미만). +- **조건절 또는 조인**에 쓰여서 컬럼 액세스로 이득을 보는 필드. +- 저-카디널리티 dictionary 최적화 대상 필드인데 희소성 때문에 스킵될 가능성이 있는 경우. + +**사용하지 말아야 할 경우:** 대부분의 행에 등장하는 dense 필드. 이런 필드는 기존 +휴리스틱이 자동으로 평탄화하며, `flat_json.column.max` 쿼터를 소비합니다. + +## 문법 + +`column_paths`는 **JSON 컬럼별**로 독립적으로 지정됩니다. 프로퍼티 키에 JSON 컬럼명을 +박아 스코프를 구분합니다: + +``` +flat_json.column_paths. = "<경로1>, <경로2>, ..." +flat_json.column_paths..add = "<경로A>, <경로B>" (ALTER 전용) +flat_json.column_paths..remove = "<경로C>" (ALTER 전용) +flat_json.column_paths_max = <정수> (컬럼별 상한) +``` + +- ``은 테이블 스키마에 존재하는 JSON 타입 컬럼이어야 합니다. 존재하지 않거나 + JSON 타입이 아니면 DDL analyze 단계에서 실패합니다. +- 경로는 쉼표로 구분합니다. 각 경로는 `$.` 접두사로 시작할 수 있으며(선택, 파싱 시 제거), + 중첩 경로는 `.`으로 구분합니다: `$.user.country`. +- 예약 접미사 `.add`, `.remove`는 `ALTER TABLE ... SET (...)`에서만 유효합니다 + (`CREATE TABLE`에서는 사용할 수 없음). +- `flat_json.column_paths_max`는 **JSON 컬럼당** 강제 평탄화 컬럼의 상한입니다 + (`flat_json.column.max`는 희소성 기반 컬럼 상한과 별개). 기본값은 `200`. + +## 전제 조건 + +`column_paths`를 설정하려면 먼저 Flat JSON이 활성화되어 있어야 합니다: + +```sql +ALTER TABLE t SET ("flat_json.enable" = "true"); +``` + +비활성 상태에서 `column_paths` 관련 키를 설정하면 semantic 에러가 발생합니다. + +## CREATE TABLE 시점에 설정 + +```sql +CREATE TABLE user_events ( + id BIGINT, + ts DATETIME, + events JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.null.factor" = "0.3", + "flat_json.sparsity.factor" = "0.5", + "flat_json.column.max" = "50", + -- events.browser, events.utm_source을 희소성 무시하고 강제 평탄화 + "flat_json.column_paths.events" = "$.browser, $.utm_source", + -- 강제 경로의 컬럼별 상한 (선택; 기본 200) + "flat_json.column_paths_max" = "20" +); +``` + +여러 JSON 컬럼이 있으면 각각 별도 키를 사용합니다: + +```sql +CREATE TABLE logs ( + id BIGINT, + request JSON, + response JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.column_paths.request" = "$.method, $.path", + "flat_json.column_paths.response" = "$.status, $.latency_ms" +); +``` + +## ALTER TABLE로 변경 + +JSON 컬럼별로 3가지 연산을 지원합니다. 한 `ALTER` 문에서 조합할 수 있습니다. + +### 전체 교체 + +해당 컬럼의 경로 리스트를 완전히 새 값으로 덮어씁니다. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events" = "$.browser, $.country, $.utm_source" +); +``` + +빈 문자열을 주면 해당 컬럼의 강제 경로를 모두 제거합니다 (이후 그 컬럼은 순수하게 +희소성 기반으로만 평탄화됩니다): + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths.events" = ""); +``` + +### 증분 추가 + +기존 리스트에 없는 경로만 뒤에 추가합니다. 기존 경로는 보존됩니다. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.add" = "$.device_id, $.session_id" +); +``` + +### 증분 제거 + +매칭되는 경로를 제거합니다. 현재 리스트에 없는 경로는 조용히 무시됩니다. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.remove" = "$.utm_source" +); +``` + +### 컬럼별 상한 조정 + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths_max" = "50"); +``` + +`0`으로 설정하면 서버 기본값(`200`)으로 복원됩니다. + +### 조합 예시 + +```sql +ALTER TABLE logs SET ( + "flat_json.column_paths.request.add" = "$.user_agent", + "flat_json.column_paths.response.remove" = "$.latency_ms", + "flat_json.column_paths_max" = "100" +); +``` + +## 희소성과의 상호작용 + +| 경로 상태 | 동작 | +|-----------------------------------------|-------------------------------------------------------------------| +| `column_paths`에 포함 | 희소성 무시하고 서브컬럼으로 만듦 (단, `hits > 0`이어야 함). | +| 포함됐지만 데이터에 전혀 등장하지 않음 | 제외 (빈 서브컬럼은 생성되지 않음). | +| 미포함, 희소성 임계치 이상 | 일반 Flat JSON 휴리스틱으로 자동 평탄화. | +| 미포함, 희소성 임계치 미만 | 잔여(remainder) JSON blob에 유지. | + +두 쿼터는 독립적으로 동작합니다: +- `flat_json.column.max`: **자동 감지된** 희소성 기반 컬럼의 상한. +- `flat_json.column_paths_max`: **강제 지정된** 컬럼의 상한. + +## 검증: 단계별 가이드 + +지정한 경로가 실제로 서브컬럼으로 materialize 되었는지 다음 단계로 확인합니다. + +### Step 1. Flat JSON 활성화 + 경로 지정 + +```sql +CREATE TABLE user_events ( + id BIGINT, ts DATETIME, events JSON +) DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.sparsity.factor" = "0.9", + "flat_json.column_paths.events" = "$.browser" +); +``` + +`sparsity.factor = 0.9`로 높게 설정해서, 일반 희소성 로직으로는 `$.browser`가 절대 +평탄화되지 않게 합니다. 이렇게 하면 `browser`가 서브컬럼으로 나타난다는 것은 곧 +`column_paths`가 동작했다는 증거가 됩니다. + +### Step 2. 해당 경로가 희소하게 나타나는 데이터 적재 + +```sql +INSERT INTO user_events VALUES + (1, now(), parse_json('{"session":"a","browser":"Chrome"}')), + (2, now(), parse_json('{"session":"b"}')), + (3, now(), parse_json('{"session":"c"}')), + (4, now(), parse_json('{"session":"d"}')), + (5, now(), parse_json('{"session":"e"}')); +``` + +`browser`는 1/5 (20%)에만 등장하므로 90% 임계치에 크게 못 미칩니다. 즉 +`column_paths`가 없었다면 평탄화되지 **않을** 데이터입니다. + +### Step 3. Materialize 된 서브컬럼 확인 + +`[_META_]` 가상 테이블에 `flat_json_meta` 함수를 사용합니다: + +```sql +SELECT flat_json_meta(events) FROM user_events[_META_]; +``` + +예상 결과 (키 순서는 상관없음): + +``` ++-------------------------------------------------------+ +| flat_json_meta(events) | ++-------------------------------------------------------+ +| ["session(VARCHAR)", "browser(VARCHAR)"] | ++-------------------------------------------------------+ +``` + +**목록에 `browser(VARCHAR)`가 보이면 강제 평탄화가 적용된 것입니다.** + +테이블에 JSON 컬럼이 여러 개라면 각각 조회: + +```sql +SELECT flat_json_meta(request) FROM logs[_META_]; +SELECT flat_json_meta(response) FROM logs[_META_]; +``` + +### Step 4. Query Profile로 서브컬럼 사용 확인 + +강제 경로를 건드리는 쿼리 실행: + +```sql +SELECT get_json_string(events, '$.browser') +FROM user_events +WHERE get_json_string(events, '$.browser') = 'Chrome'; +``` + +프로파일 확인 (`SET enable_profile = true;` → `SHOW PROFILELIST;` → +`ANALYZE PROFILE FROM ''`). 다음 지표를 보세요: + +- `PushdownAccessPaths`: 0보다 크면 플래너가 경로 접근을 스토리지로 push down한 것. +- `AccessPathHits`: 0보다 크면 서브컬럼을 직접 읽어 JSON 파싱 없이 값 접근 성공. +- `/events: ` 항목이 `AccessPathHits` 아래에 보임 — 해당 JSON 컬럼이 hit 되었음. + +`AccessPathHits = 0` 인데 `AccessPathUnhits > 0` 이면, 플래너는 push down했지만 해당 +rowset에 서브컬럼이 없는 상태입니다 (기능 활성화 직후에 흔함 — 신규 데이터는 +평탄화되지만 기존 세그먼트는 그대로). 컴팩션을 트리거하거나 데이터를 다시 적재해서 +재평탄화하세요. + +### Step 5. 기존 데이터에 대한 강제 컴팩션 (선택) + +신규 데이터는 로드 시점에 평탄화됩니다. `column_paths` 설정 이전의 데이터에 대해서는 +컴팩션을 트리거해서 재평탄화할 수 있습니다: + +```sql +ALTER TABLE user_events COMPACT; +``` + +BE 로그에서 다음과 같은 줄을 확인: + +``` +Compaction flat json column: nulls(TINYINT),browser(VARCHAR),session(VARCHAR) +``` + +그 후 Step 3을 반복해서 `browser`가 나타나는지 확인합니다. + +## 트러블슈팅 + +| 증상 | 원인 | 조치 | +|---------------------------------------------------------------|------------------------------------------------------------------------|----------------------------------------------------------------------------| +| `Property 'flat_json.column_paths.foo' references unknown or non-JSON column 'foo'` | `foo` 컬럼이 없거나 JSON 타입이 아님. | 올바른 JSON 컬럼명 사용. `DESC
`로 확인. | +| `flat JSON configuration must be set after enabling flat JSON.` | Flat JSON이 비활성 상태에서 `column_paths` 설정 시도. | 먼저 `ALTER TABLE t SET ("flat_json.enable" = "true");`. | +| `flat_json_meta`에 설정한 경로가 안 보임 | 아직 해당 경로가 어느 행에도 등장하지 않음 (`hits = 0`). | 해당 경로를 포함한 데이터를 적재하거나 등장할 때까지 대기. | +| 서브컬럼은 존재하지만 `AccessPathHits = 0` | 쿼리가 설정 변경 이전의 기존 rowset을 읽고 있음. | `ALTER TABLE t COMPACT;`로 재평탄화. | +| 지정한 경로 수가 쿼터 초과 | 해당 JSON 컬럼에 대해 `flat_json.column_paths_max`에 도달. | 상한을 올리거나 `.remove`로 경로를 줄임. | +| Remove 후 follower FE의 상태가 이상함 | 초기 버전의 emit-only-if-non-empty 버그. | 본 설계에서는 항상 전체 상태를 emit하여 수정됨. | + +## 메타데이터와 클러스터 전체 일관성 + +`column_paths` 설정은 FE EditLog(`OP_MODIFY_FLAT_JSON_CONFIG`)로 영속화됩니다. 모든 +FE가 같은 로그를 replay하므로 리더와 팔로워가 동일한 설정으로 수렴합니다. 설정은 +tablet meta의 일부로 모든 BE에도 전파되며, 다음 flush/compaction 시점에 반영됩니다. + +디버깅 시 확인 방법: + +```sql +-- 현재 유효한 설정 확인 +SHOW CREATE TABLE user_events; +``` + +`PROPERTIES` 블록에 현재 활성화된 `flat_json.column_paths.` 항목들이 표시됩니다. +FE replica들 간에 이 출력을 비교해서 동일한지 확인하세요. + +## 관련 문서 + +- [Flat JSON 개요](./Flat_json.md) +- [CREATE TABLE 레퍼런스](../sql-reference/sql-statements/table_bucket_part_index/CREATE_TABLE.md) +- [ALTER TABLE 레퍼런스](../sql-reference/sql-statements/table_bucket_part_index/ALTER_TABLE.md) +- BE 파라미터: `json_flat_null_factor`, `json_flat_sparsity_factor`, + `json_flat_column_max`, `enable_compaction_flat_json` diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java index c1822ef9a264b3..9b3f9d9f32f8bf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java @@ -521,7 +521,9 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties)) { boolean isSuccess = schemaChangeHandler.updateFlatJsonConfigMeta(db, table.getId(), properties, TTabletMetaType.FLAT_JSON_CONFIG); if (!isSuccess) { diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java index bc5429307d4196..01ae29443b37f4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java @@ -2516,6 +2516,27 @@ public void updateTableMeta(Database db, String tableName, Map p } } + // Extracts the JSON column name from a key of the form + // flat_json.column_paths. (returns col_name) + // flat_json.column_paths..add (returns col_name) + // flat_json.column_paths..remove (returns col_name) + // Returns empty string if the key does not match. Used by ALTER to detect stale + // per-column properties that should be erased before leader->follower replication. + private static String extractColumnNameForFlatJsonKey(String key) { + if (!key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + return ""; + } + String suffix = key.substring(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + return suffix.substring(0, suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + } + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + return suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + } + return suffix; + } + public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map properties, TTabletMetaType metaType) { FlatJsonConfig newFlatJsonConfig; @@ -2550,10 +2571,12 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map = "..." + java.util.Map> replaceMap = + PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + for (java.util.Map.Entry> entry : replaceMap.entrySet()) { + java.util.List existing = newFlatJsonConfig.getColumnPaths(entry.getKey()); + if (!entry.getValue().equals(existing)) { + newFlatJsonConfig.setColumnPaths(entry.getKey(), entry.getValue()); + hasChanged = true; + } + } + // Per-column incremental add: flat_json.column_paths..add = "..." + java.util.Map> addMap = PropertyAnalyzer.analyzeFlatJsonColumnPathsOps( + properties, PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD); + for (java.util.Map.Entry> entry : addMap.entrySet()) { + java.util.List current = new java.util.ArrayList<>( + newFlatJsonConfig.getColumnPaths(entry.getKey())); + boolean localChanged = false; + for (String path : entry.getValue()) { + if (!current.contains(path)) { + current.add(path); + localChanged = true; + } + } + if (localChanged) { + newFlatJsonConfig.setColumnPaths(entry.getKey(), current); + hasChanged = true; + } + } + // Per-column incremental remove: flat_json.column_paths..remove = "..." + java.util.Map> removeMap = PropertyAnalyzer.analyzeFlatJsonColumnPathsOps( + properties, PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE); + for (java.util.Map.Entry> entry : removeMap.entrySet()) { + java.util.List current = new java.util.ArrayList<>( + newFlatJsonConfig.getColumnPaths(entry.getKey())); + if (current.removeAll(entry.getValue())) { + newFlatJsonConfig.setColumnPaths(entry.getKey(), current); + hasChanged = true; + } + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + int max = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + if (max >= 0 && max != newFlatJsonConfig.getFlatJsonColumnPathsMax()) { + newFlatJsonConfig.setFlatJsonColumnPathsMax(max); + hasChanged = true; + } + } if (!hasChanged) { LOG.info("table {} flat json config is same as the previous config, so nothing need to do", olapTable.getName()); return true; @@ -2582,6 +2651,16 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map + k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX) && + !newFlatJsonConfig.getFlatJsonColumnPaths().containsKey( + extractColumnNameForFlatJsonKey(k))); GlobalStateMgr.getCurrentState().getLocalMetastore().modifyFlatJsonMeta(db, olapTable, newFlatJsonConfig); } catch (Exception e) { isModifiedSuccess = false; diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java index 68c952bdbaf350..fd03fd3d07e853 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java @@ -20,8 +20,13 @@ import com.starrocks.common.util.PropertyAnalyzer; import com.starrocks.thrift.TFlatJsonConfig; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.TreeMap; public class FlatJsonConfig implements Writable { @SerializedName("flatJsonEnable") @@ -36,11 +41,23 @@ public class FlatJsonConfig implements Writable { @SerializedName("flatJsonColumnMax") private int flatJsonColumnMax; + // Per-JSON-column force-flatten paths: -> list of dot-separated paths + // (no leading "$."). Empty/absent entry means the column has no forced paths. + @SerializedName("flatJsonColumnPaths") + private Map> flatJsonColumnPaths; + + // Upper bound on force-flatten columns per JSON column (independent of flatJsonColumnMax). + // 0 means "use BE default". + @SerializedName("flatJsonColumnPathsMax") + private int flatJsonColumnPathsMax; + public FlatJsonConfig(boolean enabled, double nullFactor, double sparsityFactor, int columnMax) { this.flatJsonEnable = enabled; this.flatJsonNullFactor = nullFactor; this.flatJsonSparsityFactor = sparsityFactor; this.flatJsonColumnMax = columnMax; + this.flatJsonColumnPaths = new LinkedHashMap<>(); + this.flatJsonColumnPathsMax = 0; } public FlatJsonConfig(FlatJsonConfig config) { @@ -48,6 +65,8 @@ public FlatJsonConfig(FlatJsonConfig config) { this.flatJsonNullFactor = config.getFlatJsonNullFactor(); this.flatJsonSparsityFactor = config.getFlatJsonSparsityFactor(); this.flatJsonColumnMax = config.getFlatJsonColumnMax(); + this.flatJsonColumnPaths = deepCopyPaths(config.getFlatJsonColumnPaths()); + this.flatJsonColumnPathsMax = config.getFlatJsonColumnPathsMax(); } public FlatJsonConfig() { @@ -72,6 +91,16 @@ public void buildFromProperties(Map properties) { flatJsonColumnMax = Integer.parseInt(properties.get( PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)); } + // Replace (not merge) per-column paths from the properties map. This is the EditLog + // replay path on follower FEs: leader emits the full state in toProperties(), so this + // must overwrite any stale in-memory entries. Callers that want incremental ops + // (.add/.remove) resolve them BEFORE serializing to properties. + Map> perCol = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + flatJsonColumnPaths = new LinkedHashMap<>(perCol); + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + int max = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + flatJsonColumnPathsMax = Math.max(max, 0); + } } public boolean getFlatJsonEnable() { @@ -106,16 +135,69 @@ public void setFlatJsonColumnMax(int flatJsonColumnMax) { this.flatJsonColumnMax = flatJsonColumnMax; } + public Map> getFlatJsonColumnPaths() { + return flatJsonColumnPaths == null ? Collections.emptyMap() : flatJsonColumnPaths; + } + + public void setFlatJsonColumnPaths(Map> paths) { + this.flatJsonColumnPaths = paths == null ? new LinkedHashMap<>() : new LinkedHashMap<>(paths); + } + + // Returns the path list for a single JSON column (empty list if absent). + public List getColumnPaths(String columnName) { + Map> map = getFlatJsonColumnPaths(); + List list = map.get(columnName); + return list == null ? Collections.emptyList() : list; + } + + // Replaces the path list for a single JSON column. Passing an empty/null list REMOVES + // the entry so the column falls back to pure sparsity-based flattening. + public void setColumnPaths(String columnName, List paths) { + if (flatJsonColumnPaths == null) { + flatJsonColumnPaths = new LinkedHashMap<>(); + } + if (paths == null || paths.isEmpty()) { + flatJsonColumnPaths.remove(columnName); + } else { + flatJsonColumnPaths.put(columnName, new ArrayList<>(paths)); + } + } + + public int getFlatJsonColumnPathsMax() { + return flatJsonColumnPathsMax; + } + + public void setFlatJsonColumnPathsMax(int max) { + this.flatJsonColumnPathsMax = max; + } + + // Serializes this config back into a flat properties map for EditLog persistence. + // + // CRITICAL: the EditLog replay path on follower FEs uses + // TableProperty.modifyTableProperties(map) which does putAll() (merge, not replace) into + // the existing properties map. If we conditionally omit keys for empty/zero values, + // followers would retain stale values after a user removes all paths or resets _max to 0, + // diverging from the leader's state. To keep leader and followers in sync we: + // 1. Always emit the global _max key (so 0 reliably resets it). + // 2. Emit one key per surviving column. Keys that previously existed but are now + // absent must be cleared on the follower; SchemaChangeHandler's ALTER path + // explicitly erases stale keys before calling toProperties() (see there for details). public Map toProperties() { Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE, String.valueOf(flatJsonEnable)); - - // Only include other flat JSON properties if flat JSON is enabled if (flatJsonEnable) { properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR, String.valueOf(flatJsonNullFactor)); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR, String.valueOf(flatJsonSparsityFactor)); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, String.valueOf(flatJsonColumnMax)); + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX, + String.valueOf(flatJsonColumnPathsMax)); + if (flatJsonColumnPaths != null) { + for (Map.Entry> entry : flatJsonColumnPaths.entrySet()) { + String key = PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX + entry.getKey(); + properties.put(key, String.join(",", entry.getValue())); + } + } } return properties; } @@ -126,18 +208,35 @@ public TFlatJsonConfig toTFlatJsonConfig() { tFlatJsonConfig.setFlat_json_null_factor(flatJsonNullFactor); tFlatJsonConfig.setFlat_json_sparsity_factor(flatJsonSparsityFactor); tFlatJsonConfig.setFlat_json_column_max(flatJsonColumnMax); + Map> paths = getFlatJsonColumnPaths(); + if (!paths.isEmpty()) { + tFlatJsonConfig.setFlat_json_column_paths(new TreeMap<>(paths)); + } + if (flatJsonColumnPathsMax > 0) { + tFlatJsonConfig.setFlat_json_column_paths_max(flatJsonColumnPathsMax); + } return tFlatJsonConfig; } - - + private static Map> deepCopyPaths(Map> src) { + Map> dst = new LinkedHashMap<>(); + if (src != null) { + for (Map.Entry> e : src.entrySet()) { + dst.put(e.getKey(), new ArrayList<>(e.getValue())); + } + } + return dst; + } @Override public String toString() { return String.format("{ flat_json_enable : %b,\n " + "flat_json_null_factor : %f,\n " + "flat_json_sparsity_factor : %f,\n" + - "flat_json_column_max : %d }", flatJsonEnable, flatJsonNullFactor, flatJsonSparsityFactor, - flatJsonColumnMax); + "flat_json_column_max : %d,\n" + + "flat_json_column_paths : %s,\n" + + "flat_json_column_paths_max : %d }", + flatJsonEnable, flatJsonNullFactor, flatJsonSparsityFactor, flatJsonColumnMax, + getFlatJsonColumnPaths(), flatJsonColumnPathsMax); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index e3b4fef847c041..9c158491842324 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -3154,6 +3154,22 @@ public Map getUniqueProperties() { if (!Strings.isNullOrEmpty(flatJsonColumnMax)) { properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, flatJsonColumnMax); } + + // flat json per-column force paths: forward any key under the prefix verbatim + for (Map.Entry entry : tableProperties.entrySet()) { + String key = entry.getKey(); + if (key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX) + && !Strings.isNullOrEmpty(entry.getValue())) { + properties.put(key, entry.getValue()); + } + } + + // flat json force paths global max + String flatJsonColumnPathsMax = + tableProperties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); + if (!Strings.isNullOrEmpty(flatJsonColumnPathsMax)) { + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX, flatJsonColumnPathsMax); + } } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java index 5abb34065d9a7c..492f1782f28846 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java @@ -558,9 +558,11 @@ public TableProperty buildFlatJsonConfig() { if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties)) { boolean enableFlatJson = PropertyAnalyzer.analyzeFlatJsonEnabled(properties); - + // In gsonPostProcess, we should be tolerant of existing properties even when flat_json.enable is false. // The validation should be done at ALTER TABLE time, not during deserialization/copy. // If flat_json.enable is false, ignore other flat JSON properties and use default values. @@ -573,6 +575,17 @@ public TableProperty buildFlatJsonConfig() { PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, Config.flat_json_column_max); flatJsonConfig = new FlatJsonConfig(enableFlatJson, flatJsonNullFactor, flatJsonSparsityFactory, flatJsonColumnMax); + // Per-column force paths. analyzeFlatJsonColumnPaths only picks up full-replace + // keys (flat_json.column_paths.); .add/.remove are incremental ops handled + // by SchemaChangeHandler before reaching this path. + Map> perCol = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + if (!perCol.isEmpty()) { + flatJsonConfig.setFlatJsonColumnPaths(perCol); + } + int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + if (columnPathsMax >= 0) { + flatJsonConfig.setFlatJsonColumnPathsMax(columnPathsMax); + } } catch (AnalysisException e) { throw new RuntimeException("Failed to analyze flat JSON properties: " + e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index b3b853d1adef89..ed9c9afb513f6f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -177,6 +177,22 @@ public class PropertyAnalyzer { public static final String PROPERTIES_FLAT_JSON_COLUMN_MAX = "flat_json.column.max"; + // Per-JSON-column force-flatten paths are expressed as keys of the form: + // flat_json.column_paths. = "$.path1, $.path2" (full replace) + // flat_json.column_paths..add = "$.path3" (incremental add) + // flat_json.column_paths..remove = "$.path1" (incremental remove) + // Reserved suffixes: "add", "remove" (cannot be used as json column names for the .add/.remove ops). + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX = "flat_json.column_paths."; + + // Operation suffixes for incremental updates (ALTER TABLE SET only). + public static final String FLAT_JSON_COLUMN_PATHS_OP_ADD = "add"; + public static final String FLAT_JSON_COLUMN_PATHS_OP_REMOVE = "remove"; + + // Global upper bound on force-flatten columns per JSON column. Distinct from flat_json.column.max + // which caps auto-detected sparse-derived columns. Uses underscore (not dot) to avoid + // collision with a user-supplied JSON column literally named "max". + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX = "flat_json.column_paths_max"; + public static final String PROPERTIES_STORAGE_TYPE_COLUMN = "column"; public static final String PROPERTIES_STORAGE_TYPE_COLUMN_WITH_ROW = "column_with_row"; @@ -613,6 +629,125 @@ public static boolean analyzeFlatJsonEnabled(Map properties) { return flatJsonEnabled; } + // Returns true if the properties map contains ANY flat_json.column_paths.* key (replace/add/remove) + // or the global flat_json.column_paths_max key. + public static boolean hasFlatJsonColumnPathsProperty(Map properties) { + if (properties == null) { + return false; + } + if (properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + return true; + } + for (String key : properties.keySet()) { + if (key.startsWith(PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + return true; + } + } + return false; + } + + // Parses a comma-separated list of JSON paths. Strips the optional "$." prefix so the + // returned strings are in internal dot-separated form ("a.b.c"). Empty tokens are skipped. + private static java.util.List parseFlatJsonPathList(String raw) { + if (raw == null) { + return java.util.Collections.emptyList(); + } + raw = raw.trim(); + if (raw.isEmpty()) { + return java.util.Collections.emptyList(); + } + java.util.List result = new java.util.ArrayList<>(); + for (String part : raw.split(",")) { + String path = part.trim(); + if (path.startsWith("$.")) { + path = path.substring(2); + } + if (!path.isEmpty()) { + result.add(path); + } + } + return java.util.Collections.unmodifiableList(result); + } + + // Splits a property key "flat_json.column_paths.[.]" into (columnName, op). + // op is "" for bare keys (full replace), or "add"/"remove" for incremental ops. + // Returns null if the key is not a flat_json.column_paths.* key. + private static String[] splitColumnPathsKey(String key) { + if (!key.startsWith(PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + return null; + } + String suffix = key.substring(PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + if (suffix.isEmpty()) { + return null; + } + // Check if the key ends with ".add" or ".remove" (operation suffix). + if (suffix.endsWith("." + FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + String col = suffix.substring(0, suffix.length() - FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + if (!col.isEmpty()) { + return new String[] {col, FLAT_JSON_COLUMN_PATHS_OP_ADD}; + } + } else if (suffix.endsWith("." + FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + String col = suffix.substring(0, suffix.length() - FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + if (!col.isEmpty()) { + return new String[] {col, FLAT_JSON_COLUMN_PATHS_OP_REMOVE}; + } + } + return new String[] {suffix, ""}; + } + + // Extracts per-column full-replace entries: "flat_json.column_paths." -> List. + // Keys with .add/.remove suffix are NOT included here (use analyzeFlatJsonColumnPathsOps). + public static java.util.Map> analyzeFlatJsonColumnPaths( + Map properties) { + java.util.Map> result = new java.util.HashMap<>(); + if (properties == null) { + return result; + } + for (Map.Entry entry : properties.entrySet()) { + String[] parts = splitColumnPathsKey(entry.getKey()); + if (parts == null || !parts[1].isEmpty()) { + continue; + } + result.put(parts[0], parseFlatJsonPathList(entry.getValue())); + } + return result; + } + + // Extracts per-column incremental op entries. + // op = "add" -> keys of the form "flat_json.column_paths..add" + // op = "remove" -> keys of the form "flat_json.column_paths..remove" + public static java.util.Map> analyzeFlatJsonColumnPathsOps( + Map properties, String op) { + java.util.Map> result = new java.util.HashMap<>(); + if (properties == null) { + return result; + } + for (Map.Entry entry : properties.entrySet()) { + String[] parts = splitColumnPathsKey(entry.getKey()); + if (parts == null || !op.equals(parts[1])) { + continue; + } + result.put(parts[0], parseFlatJsonPathList(entry.getValue())); + } + return result; + } + + public static int analyzeFlatJsonColumnPathsMax(Map properties) { + if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + return -1; + } + int max; + try { + max = Integer.parseInt(properties.get(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)); + } catch (NumberFormatException e) { + throw new SemanticException(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX + ": " + e.getMessage()); + } + if (max < 0) { + throw new SemanticException("Illegal " + PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX + ": " + max); + } + return max; + } + public static boolean analyzeEnableLoadProfile(Map properties) { boolean enableLoadProfile = false; if (properties != null && properties.containsKey(PROPERTIES_ENABLE_LOAD_PROFILE)) { diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 5e5b8c3f6950d1..e8c7cd5649add5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -4332,6 +4332,7 @@ public void modifyFlatJsonMeta(Database db, OlapTable table, FlatJsonConfig flat flatJsonConfig.toProperties() ); GlobalStateMgr.getCurrentState().getEditLog().logModifyFlatJsonConfig(info, wal -> { + table.getTableProperty().modifyTableProperties(flatJsonConfig.toProperties()); table.setFlatJsonConfig(flatJsonConfig); }); } @@ -4594,6 +4595,10 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation olapTable.setHasDelete(); } else { TableProperty tableProperty = olapTable.getTableProperty(); + if (opCode == OperationType.OP_MODIFY_FLAT_JSON_CONFIG) { + tableProperty.getProperties().keySet() + .removeIf(k -> k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)); + } tableProperty.modifyTableProperties(properties); tableProperty.buildProperty(opCode); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java index dffe4084e5a0ce..9b15bcec3e5de2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java @@ -915,7 +915,9 @@ private void processFlatJsonConfig(Map properties, OlapTable tab // Check if other flat JSON properties are set when flat_json.enable is false if (!enableFlatJson && (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX))) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties))) { throw new DdlException("flat JSON configuration must be set after enabling flat JSON."); } @@ -925,9 +927,42 @@ private void processFlatJsonConfig(Map properties, OlapTable tab PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR, Config.flat_json_sparsity_factory); int flatJsonColumnMax = PropertyAnalyzer.analyzeIntProp(properties, PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, Config.flat_json_column_max); + // Per-column force-flatten paths (only full-replace keys are valid at CREATE time; + // .add/.remove are ALTER-only ops). + java.util.Map> perCol = + PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + + // Validate that every column named in perCol is a JSON-typed column in the schema. + if (!perCol.isEmpty()) { + java.util.Set jsonColumnNames = new java.util.HashSet<>(); + for (Column col : table.getBaseSchema()) { + if (col.getType() != null && col.getType().isJsonType()) { + jsonColumnNames.add(col.getName()); + } + } + for (String colName : perCol.keySet()) { + if (!jsonColumnNames.contains(colName)) { + throw new DdlException( + "flat_json.column_paths references unknown or non-JSON column: '" + colName + "'"); + } + } + } + + // Remove the per-column keys from the raw properties map so downstream handlers + // don't re-process them, but let them be re-emitted later via FlatJsonConfig.toProperties(). + properties.keySet().removeIf(k -> + k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)); + properties.remove(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); FlatJsonConfig flatJsonConfig = new FlatJsonConfig(enableFlatJson, flatJsonNullFactor, flatJsonSparsityFactory, flatJsonColumnMax); + if (!perCol.isEmpty()) { + flatJsonConfig.setFlatJsonColumnPaths(perCol); + } + if (columnPathsMax >= 0) { + flatJsonConfig.setFlatJsonColumnPathsMax(columnPathsMax); + } table.setFlatJsonConfig(flatJsonConfig); LOG.info("create table {} set flat json config: {}", tableName, flatJsonConfig.toString()); @@ -940,6 +975,8 @@ private boolean hasFlatJsonProperties(Map properties) { return properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX); + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java index 9e2bc65a6ee676..0c34ebda891eac 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java @@ -204,6 +204,48 @@ public Void visitTableRenameClause(TableRenameClause clause, ConnectContext cont return null; } + // Validates flat_json.column_paths.* (and column_paths_max) keys in the given properties map + // against the table's JSON columns. Throws SemanticException on any violation. + // Called from both the flat_json.enable branch and the dedicated column_paths branch so that + // bundling enable + column_paths in a single ALTER TABLE does not bypass validation. + private void validateFlatJsonColumnPathsProperties(Map properties, OlapTable olapTable) { + java.util.Set jsonColumnNames = new java.util.HashSet<>(); + for (Column col : olapTable.getBaseSchema()) { + if (col.getType() != null && col.getType().isJsonType()) { + jsonColumnNames.add(col.getName()); + } + } + for (String key : properties.keySet()) { + if (!key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + continue; + } + String suffix = key.substring(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + String columnName; + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + columnName = suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + } else if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + columnName = suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + } else { + columnName = suffix; + } + if (columnName.isEmpty()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Invalid property key '" + key + "': missing JSON column name"); + } + if (!jsonColumnNames.contains(columnName)) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Property '" + key + "' references unknown or non-JSON column '" + columnName + "'"); + } + PropertyAnalyzer.analyzeFlatJsonColumnPaths( + java.util.Collections.singletonMap(key, properties.get(key))); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + } + } + @Override public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, ConnectContext context) { Map properties = clause.getProperties(); @@ -217,7 +259,8 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, if (properties.size() != 1 && !(TableProperty.isSamePrefixProperties(properties, TableProperty.DYNAMIC_PARTITION_PROPERTY_PREFIX) - || TableProperty.isSamePrefixProperties(properties, TableProperty.BINLOG_PROPERTY_PREFIX))) { + || TableProperty.isSamePrefixProperties(properties, TableProperty.BINLOG_PROPERTY_PREFIX) + || TableProperty.isSamePrefixProperties(properties, TableProperty.FLAT_JSON_PROPERTY_PREFIX))) { ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, "Can only set one table property at a time"); } @@ -430,8 +473,21 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, e.getMessage()); } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE)) { - // Allow setting flat_json.enable to true or false + // Validate the enable flag itself. PropertyAnalyzer.analyzeFlatJsonEnabled(properties); + // When column_paths properties are bundled in the same ALTER, validate them here. + // The else-if chain would otherwise skip column_paths validation entirely when + // flat_json.enable is also present. + if (table instanceof OlapTable && + (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties))) { + if (!"true".equalsIgnoreCase(properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE))) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "flat_json.column_paths can only be set when " + + PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE + " is true"); + } + validateFlatJsonColumnPathsProperties(properties, (OlapTable) table); + } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)) { @@ -451,6 +507,16 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, " haven't been enabled"); } } + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties)) { + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + if (olapTable.getFlatJsonConfig() == null || !olapTable.getFlatJsonConfig().getFlatJsonEnable()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Property " + PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE + " haven't been enabled"); + } + validateFlatJsonColumnPathsProperties(properties, olapTable); + } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY)) { if (!properties.get(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY).equalsIgnoreCase("default") && !properties.get(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY).equalsIgnoreCase("real_time")) { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 2f162692674178..da207df02eb4db 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -280,11 +280,22 @@ message BinlogConfigPB { optional int64 binlog_max_size = 4; } +// Per-JSON-column force-flatten paths. column_name is the JSON column's +// identifier in the table schema; paths are dot-separated (no leading "$."). +message FlatJsonColumnPathsEntryPB { + optional string column_name = 1; + repeated string paths = 2; +} + message FlatJsonConfigPB { optional bool flat_json_enable = 1; optional double flat_json_null_factor = 2; optional double flat_json_sparsity_factor = 3; optional int64 flat_json_max_column_max = 4; + // Per-JSON-column paths that must always be flattened, regardless of sparsity. + repeated FlatJsonColumnPathsEntryPB flat_json_column_paths = 5; + // Upper bound on force-flattened columns per JSON column (0 => server default). + optional int64 flat_json_column_paths_max = 6; } message TabletMetaPB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index c7467a6104637e..35b1fbf67ff89e 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -90,6 +90,9 @@ struct TFlatJsonConfig { 2: optional double flat_json_null_factor; 3: optional double flat_json_sparsity_factor; 4: optional i64 flat_json_column_max; + // Per-JSON-column force-flatten paths: column_name -> list of dot-separated paths (no leading "$."). + 5: optional map> flat_json_column_paths; + 6: optional i64 flat_json_column_paths_max; } // If you want to add types,