From 2fe802456280c5d0e12a59927bd9c769dfc2859e Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Fri, 17 Apr 2026 17:51:58 +0900 Subject: [PATCH 1/9] [Feature] Add flat_json.column_paths to control which JSON paths are always columnized Co-Authored-By: Claude Sonnet 4.6 --- be/src/storage/flat_json_config.h | 86 +++++++++++++++---- be/src/util/json_flattener.cpp | 84 +++++++++++++++--- be/src/util/json_flattener.h | 9 ++ .../com/starrocks/alter/AlterJobExecutor.java | 6 +- .../starrocks/alter/SchemaChangeHandler.java | 52 ++++++++++- .../com/starrocks/catalog/FlatJsonConfig.java | 67 +++++++++++++-- .../java/com/starrocks/catalog/OlapTable.java | 13 +++ .../com/starrocks/catalog/TableProperty.java | 14 ++- .../common/util/PropertyAnalyzer.java | 62 +++++++++++++ .../starrocks/server/OlapTableFactory.java | 18 +++- .../analyzer/AlterTableClauseAnalyzer.java | 29 +++++++ gensrc/proto/olap_file.proto | 5 ++ gensrc/thrift/AgentService.thrift | 8 ++ 13 files changed, 414 insertions(+), 39 deletions(-) diff --git a/be/src/storage/flat_json_config.h b/be/src/storage/flat_json_config.h index 63d03668e31e17..e6e952f80c86d9 100644 --- a/be/src/storage/flat_json_config.h +++ b/be/src/storage/flat_json_config.h @@ -17,12 +17,18 @@ #include #include +#include +#include +#include #include "gen_cpp/AgentService_types.h" namespace starrocks { class FlatJsonConfig { public: + // Default max force-path columns when not specified by the caller. + static constexpr int DEFAULT_COLUMN_PATHS_MAX = 200; + // Constructor FlatJsonConfig(); @@ -31,7 +37,8 @@ class FlatJsonConfig { : _flat_json_enable(enable), _flat_json_null_factor(nullFactor), _flat_json_sparsity_factor(sparsityFactor), - _flat_json_max_column_max(maxColumnMax) {} + _flat_json_max_column_max(maxColumnMax), + _flat_json_column_paths_max(DEFAULT_COLUMN_PATHS_MAX) {} // Getters and Setters bool is_flat_json_enabled() const { return _flat_json_enable; } @@ -46,27 +53,68 @@ class FlatJsonConfig { int get_flat_json_max_column_max() const { return _flat_json_max_column_max; } void set_flat_json_max_column_max(int max) { _flat_json_max_column_max = max; } + // Paths that must always be flattened, regardless of sparsity. + // Stored in internal dot-separated format without a leading "$.". + const std::unordered_set& get_column_paths() const { return _flat_json_column_paths; } + void set_column_paths(const std::vector& paths) { + _flat_json_column_paths.clear(); + for (const auto& p : paths) { + _flat_json_column_paths.insert(p); + } + } + + int get_column_paths_max() const { return _flat_json_column_paths_max; } + void set_column_paths_max(int max) { _flat_json_column_paths_max = max; } + void to_pb(FlatJsonConfigPB* binlog_config_pb) { binlog_config_pb->set_flat_json_enable(_flat_json_enable); binlog_config_pb->set_flat_json_null_factor(_flat_json_null_factor); binlog_config_pb->set_flat_json_sparsity_factor(_flat_json_sparsity_factor); binlog_config_pb->set_flat_json_max_column_max(_flat_json_max_column_max); + binlog_config_pb->clear_flat_json_column_paths(); + for (const auto& p : _flat_json_column_paths) { + binlog_config_pb->add_flat_json_column_paths(p); + } + binlog_config_pb->set_flat_json_column_paths_max(_flat_json_column_paths_max); } // Update function using another FlatJsonConfig void update(const FlatJsonConfig& config) { - update(config.is_flat_json_enabled(), config.get_flat_json_null_factor(), - config.get_flat_json_sparsity_factor(), config.get_flat_json_max_column_max()); + _flat_json_enable = config.is_flat_json_enabled(); + _flat_json_null_factor = config.get_flat_json_null_factor(); + _flat_json_sparsity_factor = config.get_flat_json_sparsity_factor(); + _flat_json_max_column_max = config.get_flat_json_max_column_max(); + _flat_json_column_paths = config.get_column_paths(); + _flat_json_column_paths_max = config.get_column_paths_max(); } void update(const TFlatJsonConfig& config) { - update(config.flat_json_enable, config.flat_json_null_factor, config.flat_json_sparsity_factor, - config.flat_json_column_max); + _flat_json_enable = config.flat_json_enable; + _flat_json_null_factor = config.flat_json_null_factor; + _flat_json_sparsity_factor = config.flat_json_sparsity_factor; + _flat_json_max_column_max = config.flat_json_column_max; + if (config.__isset.flat_json_column_paths) { + set_column_paths(std::vector(config.flat_json_column_paths.begin(), + config.flat_json_column_paths.end())); + } + if (config.__isset.flat_json_column_paths_max && config.flat_json_column_paths_max > 0) { + _flat_json_column_paths_max = static_cast(config.flat_json_column_paths_max); + } } void update(const FlatJsonConfigPB& flat_json_config_pb) { - update(flat_json_config_pb.flat_json_enable(), flat_json_config_pb.flat_json_null_factor(), - flat_json_config_pb.flat_json_sparsity_factor(), flat_json_config_pb.flat_json_max_column_max()); + _flat_json_enable = flat_json_config_pb.flat_json_enable(); + _flat_json_null_factor = flat_json_config_pb.flat_json_null_factor(); + _flat_json_sparsity_factor = flat_json_config_pb.flat_json_sparsity_factor(); + _flat_json_max_column_max = flat_json_config_pb.flat_json_max_column_max(); + _flat_json_column_paths.clear(); + for (const auto& p : flat_json_config_pb.flat_json_column_paths()) { + _flat_json_column_paths.insert(p); + } + if (flat_json_config_pb.has_flat_json_column_paths_max() && + flat_json_config_pb.flat_json_column_paths_max() > 0) { + _flat_json_column_paths_max = static_cast(flat_json_config_pb.flat_json_column_paths_max()); + } } // Copy Assignment @@ -76,25 +124,28 @@ class FlatJsonConfig { _flat_json_null_factor = other._flat_json_null_factor; _flat_json_sparsity_factor = other._flat_json_sparsity_factor; _flat_json_max_column_max = other._flat_json_max_column_max; + _flat_json_column_paths = other._flat_json_column_paths; + _flat_json_column_paths_max = other._flat_json_column_paths_max; } return *this; } - // Update function using four parameters - void update(bool enable, double nullFactor, double sparsityFactor, int maxColumnMax) { - _flat_json_enable = enable; - _flat_json_null_factor = nullFactor; - _flat_json_sparsity_factor = sparsityFactor; - _flat_json_max_column_max = maxColumnMax; - } - std::string to_string() const { std::ostringstream oss; oss << "FlatJsonConfig{"; oss << "flat_json_enable=" << (_flat_json_enable ? "true" : "false") << ", "; oss << "flat_json_null_factor=" << _flat_json_null_factor << ", "; oss << "flat_json_sparsity_factor=" << _flat_json_sparsity_factor << ", "; - oss << "flat_json_max_column_max=" << _flat_json_max_column_max; + oss << "flat_json_max_column_max=" << _flat_json_max_column_max << ", "; + oss << "flat_json_column_paths=["; + bool first = true; + for (const auto& p : _flat_json_column_paths) { + if (!first) oss << ","; + oss << p; + first = false; + } + oss << "], "; + oss << "flat_json_column_paths_max=" << _flat_json_column_paths_max; oss << "}"; return oss.str(); } @@ -104,5 +155,8 @@ class FlatJsonConfig { double _flat_json_null_factor = 0; double _flat_json_sparsity_factor = 0; int _flat_json_max_column_max = 0; + // Force-flatten paths (dot-separated, no leading "$."). + std::unordered_set _flat_json_column_paths; + int _flat_json_column_paths_max = DEFAULT_COLUMN_PATHS_MAX; }; } // namespace starrocks diff --git a/be/src/util/json_flattener.cpp b/be/src/util/json_flattener.cpp index 8bff1dff24c83c..59b36b366f1b43 100644 --- a/be/src/util/json_flattener.cpp +++ b/be/src/util/json_flattener.cpp @@ -396,6 +396,8 @@ void JsonPathDeriver::init_flat_json_config(const FlatJsonConfig* flat_json_conf _max_json_null_factor = flat_json_config->get_flat_json_null_factor(); _min_json_sparsity_factory = flat_json_config->get_flat_json_sparsity_factor(); _max_column = flat_json_config->get_flat_json_max_column_max(); + _column_paths = flat_json_config->get_column_paths(); + _column_paths_max = flat_json_config->get_column_paths_max(); } else { _max_json_null_factor = config::json_flat_null_factor; _min_json_sparsity_factory = config::json_flat_sparsity_factor; @@ -420,6 +422,10 @@ void JsonPathDeriver::derived(const std::vector& json_datas) { _total_rows = res.value(); _path_root = std::make_shared(); + // Pre-mark force paths so _clean_sparsity_path never prunes them. + for (const auto& fp : _column_paths) { + _mark_force_path(fp, _path_root.get()); + } // init path by flat JSON _derived_on_flat_json(json_datas); @@ -565,6 +571,21 @@ void JsonPathDeriver::_derived(const Column* col, size_t mark_row) { } } +// Walk the tree along `path` (dot-separated), creating nodes as needed, and +// mark every node on the path with force=true so they survive early pruning. +void JsonPathDeriver::_mark_force_path(const std::string_view& path, JsonFlatPath* node) { + node->force = true; + if (path.empty()) { + return; + } + auto [key, next] = JsonFlatPath::split_path(path); + auto [iter, inserted] = node->children.try_emplace(key); + if (inserted) { + iter->second = std::make_unique(); + } + _mark_force_path(next, iter->second.get()); +} + void JsonPathDeriver::_clean_sparsity_path(const std::string_view& name, JsonFlatPath* node, size_t check_hits_min) { for (auto& [key, child] : node->children) { _clean_sparsity_path(key, child.get(), check_hits_min); @@ -572,7 +593,8 @@ void JsonPathDeriver::_clean_sparsity_path(const std::string_view& name, JsonFla auto iter = node->children.begin(); while (iter != node->children.end()) { auto child = iter->second.get(); - if (child->hits < check_hits_min) { + // Never prune a node that belongs to a force path. + if (child->hits < check_hits_min && !child->force) { if (_generate_filter) { _remain_keys.insert(iter->first); } @@ -685,7 +707,17 @@ uint32_t JsonPathDeriver::_dfs_finalize(JsonFlatPath* node, const std::string& a bool is_base_type = node->base_type_count >= node->hits - (node->hits * config::json_flat_complex_type_factor); bool type_check = config::enable_json_flat_complex_type || is_base_type; - if (type_check && node->multi_times <= 0 && node->hits >= _total_rows * _min_json_sparsity_factory) { + + // Force paths bypass the sparsity threshold entirely. + // We still skip if multi_times > 0 (ambiguous: same key appears twice in + // one object) or hits == 0 (path never seen in any row). + if (node->force && node->hits > 0 && node->multi_times <= 0) { + hit_leaf->emplace_back(node, absolute_path); + node->type = flat_json::JSON_BITS_TO_LOGICAL_TYPE.at(node->json_type); + node->remain = false; + return 1; + } else if (!node->force && type_check && node->multi_times <= 0 && + node->hits >= _total_rows * _min_json_sparsity_factory) { hit_leaf->emplace_back(node, absolute_path); node->type = flat_json::JSON_BITS_TO_LOGICAL_TYPE.at(node->json_type); node->remain = false; @@ -726,21 +758,51 @@ void JsonPathDeriver::_finalize() { std::vector> hit_leaf; _dfs_finalize(_path_root.get(), "", &hit_leaf); - // sort by name, just for stable order - std::sort(hit_leaf.begin(), hit_leaf.end(), - [&](const auto& a, const auto& b) { return a.first->hits > b.first->hits; }); - size_t limit = _max_column > 0 ? _max_column : std::numeric_limits::max(); - for (size_t i = limit; i < hit_leaf.size(); i++) { - if (!hit_leaf[i].first->remain && hit_leaf[i].first->hits >= _total_rows) { + // Separate forced leaves from normal leaves so each has its own quota. + std::vector> forced_leaves; + std::vector> normal_leaves; + for (auto& item : hit_leaf) { + if (item.first->force) { + forced_leaves.push_back(std::move(item)); + } else { + normal_leaves.push_back(std::move(item)); + } + } + + // Apply _column_paths_max quota to force leaves (sort by hits desc for determinism). + std::sort(forced_leaves.begin(), forced_leaves.end(), + [](const auto& a, const auto& b) { return a.first->hits > b.first->hits; }); + size_t force_limit = (_column_paths_max > 0) + ? static_cast(_column_paths_max) + : std::numeric_limits::max(); + for (size_t i = force_limit; i < forced_leaves.size(); i++) { + forced_leaves[i].first->remain = true; + _has_remain = true; + } + if (forced_leaves.size() > force_limit) { + forced_leaves.resize(force_limit); + } + + // Apply _max_column quota to normal leaves (existing behaviour). + std::sort(normal_leaves.begin(), normal_leaves.end(), + [](const auto& a, const auto& b) { return a.first->hits > b.first->hits; }); + size_t limit = _max_column > 0 ? static_cast(_max_column) : std::numeric_limits::max(); + for (size_t i = limit; i < normal_leaves.size(); i++) { + if (!normal_leaves[i].first->remain && normal_leaves[i].first->hits >= _total_rows) { limit++; continue; } - hit_leaf[i].first->remain = true; + normal_leaves[i].first->remain = true; } - if (hit_leaf.size() > limit) { + if (normal_leaves.size() > limit) { _has_remain |= true; - hit_leaf.resize(limit); + normal_leaves.resize(limit); } + + // Merge back and sort by path name for stable column order. + hit_leaf.clear(); + hit_leaf.insert(hit_leaf.end(), forced_leaves.begin(), forced_leaves.end()); + hit_leaf.insert(hit_leaf.end(), normal_leaves.begin(), normal_leaves.end()); std::sort(hit_leaf.begin(), hit_leaf.end(), [](const auto& a, const auto& b) { return a.second < b.second; }); for (auto& [node, path] : hit_leaf) { node->index = _paths.size(); diff --git a/be/src/util/json_flattener.h b/be/src/util/json_flattener.h index 494fa8521ac565..1a81be9d7991f7 100644 --- a/be/src/util/json_flattener.h +++ b/be/src/util/json_flattener.h @@ -68,6 +68,7 @@ class JsonFlatPath { int index = -1; // flat paths array index, only use for leaf, to find column LogicalType type = LogicalType::TYPE_JSON; bool remain = false; + bool force = false; // true: must flatten regardless of sparsity (column_paths) OP op = OP_INCLUDE; // merge flat json use, to mark the path is need FlatJsonHashMap> children; @@ -161,6 +162,10 @@ class JsonPathDeriver { // clean sparsity path, to save memory void _clean_sparsity_path(const std::string_view& name, JsonFlatPath* root, size_t check_hits_min); + // Pre-mark all nodes along `path` (dot-separated) with force=true so that + // _clean_sparsity_path and _dfs_finalize never discard them. + void _mark_force_path(const std::string_view& path, JsonFlatPath* node); + private: bool _has_remain = false; std::vector _paths; @@ -170,6 +175,10 @@ class JsonPathDeriver { double _max_json_null_factor = 0; int _max_column = 0; + // column_paths: dot-separated paths that bypass the sparsity check. + std::unordered_set _column_paths; + int _column_paths_max = FlatJsonConfig::DEFAULT_COLUMN_PATHS_MAX; + size_t _total_rows; std::shared_ptr _path_root; diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java index c1822ef9a264b3..5a558b9b66a6ef 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java @@ -521,7 +521,11 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { boolean isSuccess = schemaChangeHandler.updateFlatJsonConfigMeta(db, table.getId(), properties, TTabletMetaType.FLAT_JSON_CONFIG); if (!isSuccess) { diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java index bc5429307d4196..5577a382ab0439 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java @@ -2550,10 +2550,14 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map columnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + if (!columnPaths.equals(newFlatJsonConfig.getFlatJsonColumnPaths())) { + newFlatJsonConfig.setFlatJsonColumnPaths(columnPaths); + hasChanged = true; + } + } + // Incremental add: append paths not already in the list + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD)) { + java.util.List toAdd = PropertyAnalyzer.analyzeFlatJsonColumnPaths( + java.util.Collections.singletonMap( + PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, + properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD))); + java.util.List current = new java.util.ArrayList<>(newFlatJsonConfig.getFlatJsonColumnPaths()); + for (String path : toAdd) { + if (!current.contains(path)) { + current.add(path); + hasChanged = true; + } + } + if (hasChanged) { + newFlatJsonConfig.setFlatJsonColumnPaths(current); + } + } + // Incremental remove: delete matching paths from the list + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE)) { + java.util.List toRemove = PropertyAnalyzer.analyzeFlatJsonColumnPaths( + java.util.Collections.singletonMap( + PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, + properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE))); + java.util.List current = new java.util.ArrayList<>(newFlatJsonConfig.getFlatJsonColumnPaths()); + if (current.removeAll(toRemove)) { + newFlatJsonConfig.setFlatJsonColumnPaths(current); + hasChanged = true; + } + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + int max = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + if (max >= 0 && max != newFlatJsonConfig.getFlatJsonColumnPathsMax()) { + newFlatJsonConfig.setFlatJsonColumnPathsMax(max); + hasChanged = true; + } + } if (!hasChanged) { LOG.info("table {} flat json config is same as the previous config, so nothing need to do", olapTable.getName()); return true; diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java index 68c952bdbaf350..84d11e2f0fe69a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java @@ -20,7 +20,10 @@ import com.starrocks.common.util.PropertyAnalyzer; import com.starrocks.thrift.TFlatJsonConfig; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; public class FlatJsonConfig implements Writable { @@ -36,11 +39,23 @@ public class FlatJsonConfig implements Writable { @SerializedName("flatJsonColumnMax") private int flatJsonColumnMax; + // Paths that must always be flattened, regardless of sparsity. + // Stored in the internal dot-separated format (no leading "$."). + @SerializedName("flatJsonColumnPaths") + private List flatJsonColumnPaths; + + // Upper bound on the number of force-path columns (independent of flatJsonColumnMax). + // 0 means "use system default" (Config.flat_json_column_paths_max). + @SerializedName("flatJsonColumnPathsMax") + private int flatJsonColumnPathsMax; + public FlatJsonConfig(boolean enabled, double nullFactor, double sparsityFactor, int columnMax) { this.flatJsonEnable = enabled; this.flatJsonNullFactor = nullFactor; this.flatJsonSparsityFactor = sparsityFactor; this.flatJsonColumnMax = columnMax; + this.flatJsonColumnPaths = Collections.emptyList(); + this.flatJsonColumnPathsMax = 0; } public FlatJsonConfig(FlatJsonConfig config) { @@ -48,6 +63,8 @@ public FlatJsonConfig(FlatJsonConfig config) { this.flatJsonNullFactor = config.getFlatJsonNullFactor(); this.flatJsonSparsityFactor = config.getFlatJsonSparsityFactor(); this.flatJsonColumnMax = config.getFlatJsonColumnMax(); + this.flatJsonColumnPaths = new ArrayList<>(config.getFlatJsonColumnPaths()); + this.flatJsonColumnPathsMax = config.getFlatJsonColumnPathsMax(); } public FlatJsonConfig() { @@ -72,6 +89,15 @@ public void buildFromProperties(Map properties) { flatJsonColumnMax = Integer.parseInt(properties.get( PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)); } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { + flatJsonColumnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + int max = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + if (max >= 0) { + flatJsonColumnPathsMax = max; + } + } } public boolean getFlatJsonEnable() { @@ -106,6 +132,22 @@ public void setFlatJsonColumnMax(int flatJsonColumnMax) { this.flatJsonColumnMax = flatJsonColumnMax; } + public List getFlatJsonColumnPaths() { + return flatJsonColumnPaths == null ? Collections.emptyList() : flatJsonColumnPaths; + } + + public void setFlatJsonColumnPaths(List paths) { + this.flatJsonColumnPaths = paths == null ? Collections.emptyList() : paths; + } + + public int getFlatJsonColumnPathsMax() { + return flatJsonColumnPathsMax; + } + + public void setFlatJsonColumnPathsMax(int max) { + this.flatJsonColumnPathsMax = max; + } + public Map toProperties() { Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE, String.valueOf(flatJsonEnable)); @@ -116,6 +158,14 @@ public Map toProperties() { properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR, String.valueOf(flatJsonSparsityFactor)); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, String.valueOf(flatJsonColumnMax)); + List fps = getFlatJsonColumnPaths(); + if (!fps.isEmpty()) { + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, String.join(",", fps)); + } + if (flatJsonColumnPathsMax > 0) { + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX, + String.valueOf(flatJsonColumnPathsMax)); + } } return properties; } @@ -126,18 +176,25 @@ public TFlatJsonConfig toTFlatJsonConfig() { tFlatJsonConfig.setFlat_json_null_factor(flatJsonNullFactor); tFlatJsonConfig.setFlat_json_sparsity_factor(flatJsonSparsityFactor); tFlatJsonConfig.setFlat_json_column_max(flatJsonColumnMax); + List fps = getFlatJsonColumnPaths(); + if (!fps.isEmpty()) { + tFlatJsonConfig.setFlat_json_column_paths(fps); + } + if (flatJsonColumnPathsMax > 0) { + tFlatJsonConfig.setFlat_json_column_paths_max(flatJsonColumnPathsMax); + } return tFlatJsonConfig; } - - - @Override public String toString() { return String.format("{ flat_json_enable : %b,\n " + "flat_json_null_factor : %f,\n " + "flat_json_sparsity_factor : %f,\n" + - "flat_json_column_max : %d }", flatJsonEnable, flatJsonNullFactor, flatJsonSparsityFactor, - flatJsonColumnMax); + "flat_json_column_max : %d,\n" + + "flat_json_column_paths : %s,\n" + + "flat_json_column_paths_max : %d }", + flatJsonEnable, flatJsonNullFactor, flatJsonSparsityFactor, flatJsonColumnMax, + getFlatJsonColumnPaths(), flatJsonColumnPathsMax); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index e3b4fef847c041..a316692ad7edce 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -3154,6 +3154,19 @@ public Map getUniqueProperties() { if (!Strings.isNullOrEmpty(flatJsonColumnMax)) { properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, flatJsonColumnMax); } + + // flat json force paths + String flatJsonColumnPaths = tableProperties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS); + if (!Strings.isNullOrEmpty(flatJsonColumnPaths)) { + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, flatJsonColumnPaths); + } + + // flat json force paths max + String flatJsonColumnPathsMax = + tableProperties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); + if (!Strings.isNullOrEmpty(flatJsonColumnPathsMax)) { + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX, flatJsonColumnPathsMax); + } } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java index 5abb34065d9a7c..d17609127d3e72 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java @@ -558,9 +558,11 @@ public TableProperty buildFlatJsonConfig() { if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { boolean enableFlatJson = PropertyAnalyzer.analyzeFlatJsonEnabled(properties); - + // In gsonPostProcess, we should be tolerant of existing properties even when flat_json.enable is false. // The validation should be done at ALTER TABLE time, not during deserialization/copy. // If flat_json.enable is false, ignore other flat JSON properties and use default values. @@ -573,6 +575,14 @@ public TableProperty buildFlatJsonConfig() { PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, Config.flat_json_column_max); flatJsonConfig = new FlatJsonConfig(enableFlatJson, flatJsonNullFactor, flatJsonSparsityFactory, flatJsonColumnMax); + List columnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + if (!columnPaths.isEmpty()) { + flatJsonConfig.setFlatJsonColumnPaths(columnPaths); + } + int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + if (columnPathsMax >= 0) { + flatJsonConfig.setFlatJsonColumnPathsMax(columnPathsMax); + } } catch (AnalysisException e) { throw new RuntimeException("Failed to analyze flat JSON properties: " + e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index b3b853d1adef89..565916e06aea87 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -177,6 +177,17 @@ public class PropertyAnalyzer { public static final String PROPERTIES_FLAT_JSON_COLUMN_MAX = "flat_json.column.max"; + // Comma-separated list of JSON paths that must always be flattened, + // regardless of sparsity (e.g. "page_stms_1,area_id" or "$.page_stms_1,$.area_id"). + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS = "flat_json.column_paths"; + + // Upper bound on the number of force-path columns (independent of flat_json.column.max). + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX = "flat_json.column_paths.max"; + + // Incremental add/remove operations for column_paths (ALTER TABLE SET only). + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD = "flat_json.column_paths.add"; + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE = "flat_json.column_paths.remove"; + public static final String PROPERTIES_STORAGE_TYPE_COLUMN = "column"; public static final String PROPERTIES_STORAGE_TYPE_COLUMN_WITH_ROW = "column_with_row"; @@ -613,6 +624,57 @@ public static boolean analyzeFlatJsonEnabled(Map properties) { return flatJsonEnabled; } + /** + * Parse and validate the {@code flat_json.column_paths} table property. + * + *

Each entry is trimmed and a leading {@code $.} prefix is stripped so that + * users may supply either {@code "page_stms_1"} or {@code "$.page_stms_1"}. + * An empty string after trimming is silently ignored. + * + * @return an immutable, order-preserving list of normalised path strings + */ + public static java.util.List analyzeFlatJsonColumnPaths(Map properties) { + if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { + return java.util.Collections.emptyList(); + } + String raw = properties.get(PROPERTIES_FLAT_JSON_COLUMN_PATHS).trim(); + if (raw.isEmpty()) { + return java.util.Collections.emptyList(); + } + java.util.List result = new java.util.ArrayList<>(); + for (String part : raw.split(",")) { + String path = part.trim(); + if (path.startsWith("$.")) { + path = path.substring(2); + } + if (!path.isEmpty()) { + result.add(path); + } + } + return java.util.Collections.unmodifiableList(result); + } + + /** + * Parse and validate the {@code flat_json.column_paths.max} table property. + * + * @return the configured maximum, or {@code -1} if the property is absent + */ + public static int analyzeFlatJsonColumnPathsMax(Map properties) { + if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + return -1; + } + int max; + try { + max = Integer.parseInt(properties.get(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)); + } catch (NumberFormatException e) { + throw new SemanticException("flat_json.column_paths.max: " + e.getMessage()); + } + if (max < 0) { + throw new SemanticException("Illegal flat_json.column_paths.max: " + max); + } + return max; + } + public static boolean analyzeEnableLoadProfile(Map properties) { boolean enableLoadProfile = false; if (properties != null && properties.containsKey(PROPERTIES_ENABLE_LOAD_PROFILE)) { diff --git a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java index dffe4084e5a0ce..df4de283fe2738 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java @@ -915,7 +915,9 @@ private void processFlatJsonConfig(Map properties, OlapTable tab // Check if other flat JSON properties are set when flat_json.enable is false if (!enableFlatJson && (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX))) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX))) { throw new DdlException("flat JSON configuration must be set after enabling flat JSON."); } @@ -925,9 +927,19 @@ private void processFlatJsonConfig(Map properties, OlapTable tab PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR, Config.flat_json_sparsity_factory); int flatJsonColumnMax = PropertyAnalyzer.analyzeIntProp(properties, PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, Config.flat_json_column_max); + java.util.List columnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + properties.remove(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS); + int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + properties.remove(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); FlatJsonConfig flatJsonConfig = new FlatJsonConfig(enableFlatJson, flatJsonNullFactor, flatJsonSparsityFactory, flatJsonColumnMax); + if (!columnPaths.isEmpty()) { + flatJsonConfig.setFlatJsonColumnPaths(columnPaths); + } + if (columnPathsMax >= 0) { + flatJsonConfig.setFlatJsonColumnPathsMax(columnPathsMax); + } table.setFlatJsonConfig(flatJsonConfig); LOG.info("create table {} set flat json config: {}", tableName, flatJsonConfig.toString()); @@ -940,6 +952,8 @@ private boolean hasFlatJsonProperties(Map properties) { return properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX); + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java index 9e2bc65a6ee676..ca9efa1739f85b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java @@ -451,6 +451,35 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, " haven't been enabled"); } } + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE) || + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + if (olapTable.getFlatJsonConfig() == null || !olapTable.getFlatJsonConfig().getFlatJsonEnable()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Property " + PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE + " haven't been enabled"); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { + PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD)) { + PropertyAnalyzer.analyzeFlatJsonColumnPaths( + java.util.Collections.singletonMap( + PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, + properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD))); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE)) { + PropertyAnalyzer.analyzeFlatJsonColumnPaths( + java.util.Collections.singletonMap( + PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, + properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE))); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + } + } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY)) { if (!properties.get(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY).equalsIgnoreCase("default") && !properties.get(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY).equalsIgnoreCase("real_time")) { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 2f162692674178..3fbacab105f08e 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -285,6 +285,11 @@ message FlatJsonConfigPB { optional double flat_json_null_factor = 2; optional double flat_json_sparsity_factor = 3; optional int64 flat_json_max_column_max = 4; + // Paths that must be flattened regardless of sparsity (dot-separated, + // no leading "$."). Counted against flat_json_column_paths_max, not + // flat_json_max_column_max. + repeated string flat_json_column_paths = 5; + optional int64 flat_json_column_paths_max = 6; } message TabletMetaPB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index c7467a6104637e..26edb340029c91 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -90,6 +90,14 @@ struct TFlatJsonConfig { 2: optional double flat_json_null_factor; 3: optional double flat_json_sparsity_factor; 4: optional i64 flat_json_column_max; + // Paths listed here are always flattened into typed columns regardless of + // sparsity. Use the dot-separated internal format (e.g. "page_stms_1", + // "a.b.c") or the JSON-path form with a leading "$." prefix which is + // stripped automatically (e.g. "$.page_stms_1"). + 5: optional list flat_json_column_paths; + // Maximum number of force-path columns. Counted separately from + // flat_json_column_max so that forced paths do not consume the normal quota. + 6: optional i64 flat_json_column_paths_max; } // If you want to add types, From 4926946c40d7a05a13faa4548963a88dd1f749a0 Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Fri, 17 Apr 2026 18:39:55 +0900 Subject: [PATCH 2/9] Reviewed configuration's key names and comments for each items --- .../common/util/PropertyAnalyzer.java | 18 ++---------------- gensrc/proto/olap_file.proto | 3 --- gensrc/thrift/AgentService.thrift | 6 ------ 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index 565916e06aea87..e6c8c3b161ffb0 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -178,10 +178,10 @@ public class PropertyAnalyzer { public static final String PROPERTIES_FLAT_JSON_COLUMN_MAX = "flat_json.column.max"; // Comma-separated list of JSON paths that must always be flattened, - // regardless of sparsity (e.g. "page_stms_1,area_id" or "$.page_stms_1,$.area_id"). + // regardless of sparsity (e.g. "event_ts,event_id" or "$.event_ts,$.event_id"). public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS = "flat_json.column_paths"; - // Upper bound on the number of force-path columns (independent of flat_json.column.max). + // Upper bound on the number of user-specified columns (independent of flat_json.column.max). public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX = "flat_json.column_paths.max"; // Incremental add/remove operations for column_paths (ALTER TABLE SET only). @@ -624,15 +624,6 @@ public static boolean analyzeFlatJsonEnabled(Map properties) { return flatJsonEnabled; } - /** - * Parse and validate the {@code flat_json.column_paths} table property. - * - *

Each entry is trimmed and a leading {@code $.} prefix is stripped so that - * users may supply either {@code "page_stms_1"} or {@code "$.page_stms_1"}. - * An empty string after trimming is silently ignored. - * - * @return an immutable, order-preserving list of normalised path strings - */ public static java.util.List analyzeFlatJsonColumnPaths(Map properties) { if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { return java.util.Collections.emptyList(); @@ -654,11 +645,6 @@ public static java.util.List analyzeFlatJsonColumnPaths(Map properties) { if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { return -1; diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 3fbacab105f08e..7ec2bac4058b09 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -285,9 +285,6 @@ message FlatJsonConfigPB { optional double flat_json_null_factor = 2; optional double flat_json_sparsity_factor = 3; optional int64 flat_json_max_column_max = 4; - // Paths that must be flattened regardless of sparsity (dot-separated, - // no leading "$."). Counted against flat_json_column_paths_max, not - // flat_json_max_column_max. repeated string flat_json_column_paths = 5; optional int64 flat_json_column_paths_max = 6; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 26edb340029c91..4a3d959921bdc0 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -90,13 +90,7 @@ struct TFlatJsonConfig { 2: optional double flat_json_null_factor; 3: optional double flat_json_sparsity_factor; 4: optional i64 flat_json_column_max; - // Paths listed here are always flattened into typed columns regardless of - // sparsity. Use the dot-separated internal format (e.g. "page_stms_1", - // "a.b.c") or the JSON-path form with a leading "$." prefix which is - // stripped automatically (e.g. "$.page_stms_1"). 5: optional list flat_json_column_paths; - // Maximum number of force-path columns. Counted separately from - // flat_json_column_max so that forced paths do not consume the normal quota. 6: optional i64 flat_json_column_paths_max; } From a1c1443b04300af774c4dfa99191905ee54b7dcb Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Mon, 20 Apr 2026 13:57:56 +0900 Subject: [PATCH 3/9] Add JsonPathDeriverForcePathTest for column_paths force-columnize behavior --- be/test/util/json_flattener_test.cpp | 130 +++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/be/test/util/json_flattener_test.cpp b/be/test/util/json_flattener_test.cpp index 9db83c60c44eb8..67dd05c97974a9 100644 --- a/be/test/util/json_flattener_test.cpp +++ b/be/test/util/json_flattener_test.cpp @@ -15,6 +15,7 @@ #include "util/json_flattener.h" #include +#include "storage/flat_json_config.h" #include #include #include @@ -646,4 +647,133 @@ INSTANTIATE_TEST_SUITE_P(JsonBoolExtractionCases, JsonBoolExtractionTest, std::make_tuple(R"({"bool_field": "1"})", true), std::make_tuple(R"({"bool_field": "0"})", false))); +// column_paths (force-flatten) tests +class JsonPathDeriverForcePathTest : public testing::Test { +public: + void SetUp() override { + config::enable_json_flat_complex_type = true; + config::json_flat_sparsity_factor = 0.9; + } + void TearDown() override { + config::enable_json_flat_complex_type = false; + config::json_flat_sparsity_factor = 0.3; + } + + JsonPathDeriver derive(const std::vector& jsons, const FlatJsonConfig& cfg) { + auto col = JsonColumn::create(); + for (const auto& s : jsons) { + ASSIGN_OR_ABORT(auto v, JsonValue::parse(s)); + col->append(&v); + } + std::vector columns{col.get()}; + JsonPathDeriver jf; + jf.init_flat_json_config(&cfg); + jf.derived(columns); + return jf; + } +}; + +// FORCE depth 1: sparse k2 (1/3) forced → column +TEST_F(JsonPathDeriverForcePathTest, force_1level_sparse_field) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths({"k2"}); + + auto jf = derive({ + R"({"k1": 1, "k2": 42})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2"), paths.end()); +} + +// FORCE depth 2: sparse k2.j1 (1/3) forced → column, dense k2.j2 → normal column +TEST_F(JsonPathDeriverForcePathTest, force_2level_sparse_nested) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths({"k2.j1"}); + + auto jf = derive({ + R"({"k1": 1, "k2": {"j1": 10, "j2": 100}})", + R"({"k1": 2, "k2": {"j2": 200}})", + R"({"k1": 3, "k2": {"j2": 300}})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j2"), paths.end()); +} + +// FORCE depth 3: sparse k2.j1.p1 (1/3) forced → column, dense k2.j1.p2 → normal column +TEST_F(JsonPathDeriverForcePathTest, force_3level_sparse_nested) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths({"k2.j1.p1"}); + + auto jf = derive({ + R"({"k1": 1, "k2": {"j1": {"p1": 99, "p2": 1}}})", + R"({"k1": 2, "k2": {"j1": {"p2": 2}}})", + R"({"k1": 3, "k2": {"j1": {"p2": 3}}})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1.p1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1.p2"), paths.end()); +} + +// FORCE not in data: force path with hits=0 must not become column +TEST_F(JsonPathDeriverForcePathTest, force_path_not_in_data_is_excluded) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths({"k2.j1"}); + + auto jf = derive({ + R"({"k1": 1})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_EQ(std::find(paths.begin(), paths.end(), "k2.j1"), paths.end()); +} + +// FORCE mixed: sparse k2.j1 forced alongside dense k1, k2.j2 +TEST_F(JsonPathDeriverForcePathTest, force_mixed_sparse_and_dense) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths({"k2.j1"}); + + auto jf = derive({ + R"({"k1": 1, "k2": {"j1": 10, "j2": 100}})", + R"({"k1": 2, "k2": {"j2": 200}})", + R"({"k1": 3, "k2": {"j2": 300}})", + R"({"k1": 4, "k2": {"j2": 400}})", + R"({"k1": 5, "k2": {"j2": 500}})", + }, cfg); + + auto paths = jf.flat_paths(); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k1"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j2"), paths.end()); + EXPECT_NE(std::find(paths.begin(), paths.end(), "k2.j1"), paths.end()); +} + +// FORCE quota: column_paths_max=2 caps 3 force paths → excess goes to remain +TEST_F(JsonPathDeriverForcePathTest, force_paths_max_quota) { + FlatJsonConfig cfg(true, 0.3, 0.0, 100); + cfg.set_column_paths({"k1", "k2", "k3"}); + cfg.set_column_paths_max(2); + + auto jf = derive({ + R"({"k1": 1, "k2": 2, "k3": 3})", + R"({"k1": 4, "k2": 5, "k3": 6})", + R"({"k1": 7, "k2": 8, "k3": 9})", + }, cfg); + + auto paths = jf.flat_paths(); + size_t force_count = 0; + for (const auto& p : {"k1", "k2", "k3"}) { + if (std::find(paths.begin(), paths.end(), p) != paths.end()) force_count++; + } + EXPECT_LE(force_count, 2u); + EXPECT_TRUE(jf.has_remain_json()); +} + } // namespace starrocks From 5cf48715ddb831ad50b1041b2b7d3741053f51d6 Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Wed, 22 Apr 2026 00:54:28 +0900 Subject: [PATCH 4/9] [Refactor] Scope flat_json.column_paths per JSON column and fix FE EditLog sync --- be/src/storage/flat_json_config.h | 76 +++-- be/src/storage/lake/tablet_reader.cpp | 2 +- .../storage/rowset/json_column_compactor.cpp | 2 +- be/src/storage/rowset/json_column_writer.cpp | 2 +- be/src/storage/tablet_reader.cpp | 2 +- be/src/util/json_flattener.cpp | 10 +- be/src/util/json_flattener.h | 5 +- be/test/util/json_flattener_test.cpp | 54 ++- .../using_starrocks/Flat_json_column_paths.md | 314 ++++++++++++++++++ .../using_starrocks/Flat_json_column_paths.md | 305 +++++++++++++++++ .../com/starrocks/alter/AlterJobExecutor.java | 6 +- .../starrocks/alter/SchemaChangeHandler.java | 89 +++-- .../com/starrocks/catalog/FlatJsonConfig.java | 102 ++++-- .../java/com/starrocks/catalog/OlapTable.java | 13 +- .../com/starrocks/catalog/TableProperty.java | 13 +- .../common/util/PropertyAnalyzer.java | 117 ++++++- .../starrocks/server/OlapTableFactory.java | 23 +- .../analyzer/AlterTableClauseAnalyzer.java | 52 ++- gensrc/proto/olap_file.proto | 11 +- gensrc/thrift/AgentService.thrift | 3 +- 20 files changed, 1049 insertions(+), 152 deletions(-) create mode 100644 docs/en/using_starrocks/Flat_json_column_paths.md create mode 100644 docs/zh/using_starrocks/Flat_json_column_paths.md diff --git a/be/src/storage/flat_json_config.h b/be/src/storage/flat_json_config.h index e6e952f80c86d9..8f886fc8fe213c 100644 --- a/be/src/storage/flat_json_config.h +++ b/be/src/storage/flat_json_config.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -26,9 +27,11 @@ namespace starrocks { class FlatJsonConfig { public: - // Default max force-path columns when not specified by the caller. + // Default max force-path columns (per JSON column) when not specified by the caller. static constexpr int DEFAULT_COLUMN_PATHS_MAX = 200; + using ColumnPathsMap = std::unordered_map>; + // Constructor FlatJsonConfig(); @@ -53,14 +56,23 @@ class FlatJsonConfig { int get_flat_json_max_column_max() const { return _flat_json_max_column_max; } void set_flat_json_max_column_max(int max) { _flat_json_max_column_max = max; } - // Paths that must always be flattened, regardless of sparsity. - // Stored in internal dot-separated format without a leading "$.". - const std::unordered_set& get_column_paths() const { return _flat_json_column_paths; } - void set_column_paths(const std::vector& paths) { - _flat_json_column_paths.clear(); + // Per-JSON-column force-flatten paths (dot-separated, no leading "$."). + const ColumnPathsMap& get_column_paths_map() const { return _flat_json_column_paths; } + + // Returns the path set for the given JSON column, or nullptr if the column has no forced paths. + const std::unordered_set* get_column_paths_for(const std::string& column_name) const { + auto it = _flat_json_column_paths.find(column_name); + return it == _flat_json_column_paths.end() ? nullptr : &it->second; + } + + void set_column_paths_map(ColumnPathsMap paths) { _flat_json_column_paths = std::move(paths); } + + void set_column_paths(const std::string& column_name, const std::vector& paths) { + std::unordered_set s; for (const auto& p : paths) { - _flat_json_column_paths.insert(p); + s.insert(p); } + _flat_json_column_paths[column_name] = std::move(s); } int get_column_paths_max() const { return _flat_json_column_paths_max; } @@ -72,8 +84,12 @@ class FlatJsonConfig { binlog_config_pb->set_flat_json_sparsity_factor(_flat_json_sparsity_factor); binlog_config_pb->set_flat_json_max_column_max(_flat_json_max_column_max); binlog_config_pb->clear_flat_json_column_paths(); - for (const auto& p : _flat_json_column_paths) { - binlog_config_pb->add_flat_json_column_paths(p); + for (const auto& [col, paths] : _flat_json_column_paths) { + auto* entry = binlog_config_pb->add_flat_json_column_paths(); + entry->set_column_name(col); + for (const auto& p : paths) { + entry->add_paths(p); + } } binlog_config_pb->set_flat_json_column_paths_max(_flat_json_column_paths_max); } @@ -84,7 +100,7 @@ class FlatJsonConfig { _flat_json_null_factor = config.get_flat_json_null_factor(); _flat_json_sparsity_factor = config.get_flat_json_sparsity_factor(); _flat_json_max_column_max = config.get_flat_json_max_column_max(); - _flat_json_column_paths = config.get_column_paths(); + _flat_json_column_paths = config.get_column_paths_map(); _flat_json_column_paths_max = config.get_column_paths_max(); } @@ -93,9 +109,12 @@ class FlatJsonConfig { _flat_json_null_factor = config.flat_json_null_factor; _flat_json_sparsity_factor = config.flat_json_sparsity_factor; _flat_json_max_column_max = config.flat_json_column_max; + _flat_json_column_paths.clear(); if (config.__isset.flat_json_column_paths) { - set_column_paths(std::vector(config.flat_json_column_paths.begin(), - config.flat_json_column_paths.end())); + for (const auto& [col, paths] : config.flat_json_column_paths) { + std::unordered_set s(paths.begin(), paths.end()); + _flat_json_column_paths.emplace(col, std::move(s)); + } } if (config.__isset.flat_json_column_paths_max && config.flat_json_column_paths_max > 0) { _flat_json_column_paths_max = static_cast(config.flat_json_column_paths_max); @@ -108,8 +127,12 @@ class FlatJsonConfig { _flat_json_sparsity_factor = flat_json_config_pb.flat_json_sparsity_factor(); _flat_json_max_column_max = flat_json_config_pb.flat_json_max_column_max(); _flat_json_column_paths.clear(); - for (const auto& p : flat_json_config_pb.flat_json_column_paths()) { - _flat_json_column_paths.insert(p); + for (const auto& entry : flat_json_config_pb.flat_json_column_paths()) { + std::unordered_set s; + for (const auto& p : entry.paths()) { + s.insert(p); + } + _flat_json_column_paths.emplace(entry.column_name(), std::move(s)); } if (flat_json_config_pb.has_flat_json_column_paths_max() && flat_json_config_pb.flat_json_column_paths_max() > 0) { @@ -137,14 +160,21 @@ class FlatJsonConfig { oss << "flat_json_null_factor=" << _flat_json_null_factor << ", "; oss << "flat_json_sparsity_factor=" << _flat_json_sparsity_factor << ", "; oss << "flat_json_max_column_max=" << _flat_json_max_column_max << ", "; - oss << "flat_json_column_paths=["; - bool first = true; - for (const auto& p : _flat_json_column_paths) { - if (!first) oss << ","; - oss << p; - first = false; + oss << "flat_json_column_paths={"; + bool first_col = true; + for (const auto& [col, paths] : _flat_json_column_paths) { + if (!first_col) oss << ","; + oss << col << ":["; + bool first_p = true; + for (const auto& p : paths) { + if (!first_p) oss << ","; + oss << p; + first_p = false; + } + oss << "]"; + first_col = false; } - oss << "], "; + oss << "}, "; oss << "flat_json_column_paths_max=" << _flat_json_column_paths_max; oss << "}"; return oss.str(); @@ -155,8 +185,8 @@ class FlatJsonConfig { double _flat_json_null_factor = 0; double _flat_json_sparsity_factor = 0; int _flat_json_max_column_max = 0; - // Force-flatten paths (dot-separated, no leading "$."). - std::unordered_set _flat_json_column_paths; + // Per-JSON-column force-flatten paths: column_name -> set of dot-separated paths (no leading "$."). + ColumnPathsMap _flat_json_column_paths; int _flat_json_column_paths_max = DEFAULT_COLUMN_PATHS_MAX; }; } // namespace starrocks diff --git a/be/src/storage/lake/tablet_reader.cpp b/be/src/storage/lake/tablet_reader.cpp index 393065c3fdc260..b81d51f4908be1 100644 --- a/be/src/storage/lake/tablet_reader.cpp +++ b/be/src/storage/lake/tablet_reader.cpp @@ -269,7 +269,7 @@ Status TabletReader::init_compaction_column_paths(const TabletReaderParams& read metadata && metadata->has_flat_json_config()) { auto flat_json_config = std::make_shared(); flat_json_config->update(metadata->flat_json_config()); - deriver.init_flat_json_config(flat_json_config.get()); + deriver.init_flat_json_config(flat_json_config.get(), col_name); } deriver.derived(readers); diff --git a/be/src/storage/rowset/json_column_compactor.cpp b/be/src/storage/rowset/json_column_compactor.cpp index 35abb0d5a923dd..6edfa4aafba5d6 100644 --- a/be/src/storage/rowset/json_column_compactor.cpp +++ b/be/src/storage/rowset/json_column_compactor.cpp @@ -46,7 +46,7 @@ Status FlatJsonColumnCompactor::_compact_columns(MutableColumns& json_datas) { vc.emplace_back(js.get()); } deriver.set_generate_filter(true); - deriver.init_flat_json_config(_flat_json_config); + deriver.init_flat_json_config(_flat_json_config, _column_name); deriver.derived(vc); diff --git a/be/src/storage/rowset/json_column_writer.cpp b/be/src/storage/rowset/json_column_writer.cpp index 80dad2d968c201..4d2309a3cadff9 100644 --- a/be/src/storage/rowset/json_column_writer.cpp +++ b/be/src/storage/rowset/json_column_writer.cpp @@ -87,7 +87,7 @@ Status FlatJsonColumnWriter::append(const Column& column) { Status FlatJsonColumnWriter::_flat_column(MutableColumns& json_datas) { // all json datas must full json JsonPathDeriver deriver; - deriver.init_flat_json_config(_flat_json_config); + deriver.init_flat_json_config(_flat_json_config, _column_name); deriver.set_generate_filter(true); std::vector vc; diff --git a/be/src/storage/tablet_reader.cpp b/be/src/storage/tablet_reader.cpp index 56053df95b3a43..ef9869d0ce6a6a 100644 --- a/be/src/storage/tablet_reader.cpp +++ b/be/src/storage/tablet_reader.cpp @@ -187,7 +187,7 @@ Status TabletReader::_init_compaction_column_paths(const TabletReaderParams& rea // must all be flat json type JsonPathDeriver deriver; auto flat_json_config = _tablet->flat_json_config(); - deriver.init_flat_json_config(flat_json_config.get()); + deriver.init_flat_json_config(flat_json_config.get(), col_name); deriver.derived(readers); auto paths = deriver.flat_paths(); auto types = deriver.flat_types(); diff --git a/be/src/util/json_flattener.cpp b/be/src/util/json_flattener.cpp index 59b36b366f1b43..ffbe8c86f9f3d8 100644 --- a/be/src/util/json_flattener.cpp +++ b/be/src/util/json_flattener.cpp @@ -391,12 +391,18 @@ JsonPathDeriver::JsonPathDeriver(const std::vector& paths, const st } } -void JsonPathDeriver::init_flat_json_config(const FlatJsonConfig* flat_json_config) { +void JsonPathDeriver::init_flat_json_config(const FlatJsonConfig* flat_json_config, const std::string& column_name) { if (flat_json_config != nullptr) { _max_json_null_factor = flat_json_config->get_flat_json_null_factor(); _min_json_sparsity_factory = flat_json_config->get_flat_json_sparsity_factor(); _max_column = flat_json_config->get_flat_json_max_column_max(); - _column_paths = flat_json_config->get_column_paths(); + _column_paths.clear(); + if (!column_name.empty()) { + // Load only the paths targeted at this specific JSON column. + if (const auto* paths = flat_json_config->get_column_paths_for(column_name); paths != nullptr) { + _column_paths = *paths; + } + } _column_paths_max = flat_json_config->get_column_paths_max(); } else { _max_json_null_factor = config::json_flat_null_factor; diff --git a/be/src/util/json_flattener.h b/be/src/util/json_flattener.h index 1a81be9d7991f7..38bad0b8cf9ba7 100644 --- a/be/src/util/json_flattener.h +++ b/be/src/util/json_flattener.h @@ -123,7 +123,10 @@ class JsonPathDeriver { public: JsonPathDeriver(); JsonPathDeriver(const std::vector& paths, const std::vector& types, bool has_remain); - void init_flat_json_config(const FlatJsonConfig* flat_json_config); + // column_name: identifier of the JSON column being derived. Force-flatten paths + // configured for this specific column are applied; paths targeted at other columns + // are ignored. Pass empty string to disable per-column force-flatten. + void init_flat_json_config(const FlatJsonConfig* flat_json_config, const std::string& column_name = ""); ~JsonPathDeriver() = default; diff --git a/be/test/util/json_flattener_test.cpp b/be/test/util/json_flattener_test.cpp index 67dd05c97974a9..3579e5ef743256 100644 --- a/be/test/util/json_flattener_test.cpp +++ b/be/test/util/json_flattener_test.cpp @@ -650,6 +650,9 @@ INSTANTIATE_TEST_SUITE_P(JsonBoolExtractionCases, JsonBoolExtractionTest, // column_paths (force-flatten) tests class JsonPathDeriverForcePathTest : public testing::Test { public: + // Column identifier used by all tests below; matches cfg.set_column_paths(kJsonColName, ...). + static constexpr const char* kJsonColName = "events"; + void SetUp() override { config::enable_json_flat_complex_type = true; config::json_flat_sparsity_factor = 0.9; @@ -659,7 +662,8 @@ class JsonPathDeriverForcePathTest : public testing::Test { config::json_flat_sparsity_factor = 0.3; } - JsonPathDeriver derive(const std::vector& jsons, const FlatJsonConfig& cfg) { + JsonPathDeriver derive(const std::vector& jsons, const FlatJsonConfig& cfg, + const std::string& column_name = kJsonColName) { auto col = JsonColumn::create(); for (const auto& s : jsons) { ASSIGN_OR_ABORT(auto v, JsonValue::parse(s)); @@ -667,7 +671,7 @@ class JsonPathDeriverForcePathTest : public testing::Test { } std::vector columns{col.get()}; JsonPathDeriver jf; - jf.init_flat_json_config(&cfg); + jf.init_flat_json_config(&cfg, column_name); jf.derived(columns); return jf; } @@ -676,7 +680,7 @@ class JsonPathDeriverForcePathTest : public testing::Test { // FORCE depth 1: sparse k2 (1/3) forced → column TEST_F(JsonPathDeriverForcePathTest, force_1level_sparse_field) { FlatJsonConfig cfg(true, 0.3, 0.9, 100); - cfg.set_column_paths({"k2"}); + cfg.set_column_paths(kJsonColName, {"k2"}); auto jf = derive({ R"({"k1": 1, "k2": 42})", @@ -692,7 +696,7 @@ TEST_F(JsonPathDeriverForcePathTest, force_1level_sparse_field) { // FORCE depth 2: sparse k2.j1 (1/3) forced → column, dense k2.j2 → normal column TEST_F(JsonPathDeriverForcePathTest, force_2level_sparse_nested) { FlatJsonConfig cfg(true, 0.3, 0.9, 100); - cfg.set_column_paths({"k2.j1"}); + cfg.set_column_paths(kJsonColName, {"k2.j1"}); auto jf = derive({ R"({"k1": 1, "k2": {"j1": 10, "j2": 100}})", @@ -708,7 +712,7 @@ TEST_F(JsonPathDeriverForcePathTest, force_2level_sparse_nested) { // FORCE depth 3: sparse k2.j1.p1 (1/3) forced → column, dense k2.j1.p2 → normal column TEST_F(JsonPathDeriverForcePathTest, force_3level_sparse_nested) { FlatJsonConfig cfg(true, 0.3, 0.9, 100); - cfg.set_column_paths({"k2.j1.p1"}); + cfg.set_column_paths(kJsonColName, {"k2.j1.p1"}); auto jf = derive({ R"({"k1": 1, "k2": {"j1": {"p1": 99, "p2": 1}}})", @@ -724,7 +728,7 @@ TEST_F(JsonPathDeriverForcePathTest, force_3level_sparse_nested) { // FORCE not in data: force path with hits=0 must not become column TEST_F(JsonPathDeriverForcePathTest, force_path_not_in_data_is_excluded) { FlatJsonConfig cfg(true, 0.3, 0.9, 100); - cfg.set_column_paths({"k2.j1"}); + cfg.set_column_paths(kJsonColName, {"k2.j1"}); auto jf = derive({ R"({"k1": 1})", @@ -739,7 +743,7 @@ TEST_F(JsonPathDeriverForcePathTest, force_path_not_in_data_is_excluded) { // FORCE mixed: sparse k2.j1 forced alongside dense k1, k2.j2 TEST_F(JsonPathDeriverForcePathTest, force_mixed_sparse_and_dense) { FlatJsonConfig cfg(true, 0.3, 0.9, 100); - cfg.set_column_paths({"k2.j1"}); + cfg.set_column_paths(kJsonColName, {"k2.j1"}); auto jf = derive({ R"({"k1": 1, "k2": {"j1": 10, "j2": 100}})", @@ -758,7 +762,7 @@ TEST_F(JsonPathDeriverForcePathTest, force_mixed_sparse_and_dense) { // FORCE quota: column_paths_max=2 caps 3 force paths → excess goes to remain TEST_F(JsonPathDeriverForcePathTest, force_paths_max_quota) { FlatJsonConfig cfg(true, 0.3, 0.0, 100); - cfg.set_column_paths({"k1", "k2", "k3"}); + cfg.set_column_paths(kJsonColName, {"k1", "k2", "k3"}); cfg.set_column_paths_max(2); auto jf = derive({ @@ -776,4 +780,38 @@ TEST_F(JsonPathDeriverForcePathTest, force_paths_max_quota) { EXPECT_TRUE(jf.has_remain_json()); } +// Per-column scope: paths configured for a different JSON column must NOT leak +// into this column. With column "other" forced and column "events" derived, +// sparse "k2" in "events" should fall back to sparsity rules (pruned here). +TEST_F(JsonPathDeriverForcePathTest, per_column_scope_isolation) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths("other", {"k2"}); // forced for a different JSON column + + auto jf = derive({ + R"({"k1": 1, "k2": 42})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg, "events"); + + auto paths = jf.flat_paths(); + // k1 is dense → normal column; k2 is sparse (1/3) and NOT force-scoped for "events" + EXPECT_NE(std::find(paths.begin(), paths.end(), "k1"), paths.end()); + EXPECT_EQ(std::find(paths.begin(), paths.end(), "k2"), paths.end()); +} + +// Empty column_name disables per-column force even if the map has entries. +TEST_F(JsonPathDeriverForcePathTest, empty_column_name_disables_force) { + FlatJsonConfig cfg(true, 0.3, 0.9, 100); + cfg.set_column_paths("events", {"k2"}); + + auto jf = derive({ + R"({"k1": 1, "k2": 42})", + R"({"k1": 2})", + R"({"k1": 3})", + }, cfg, /*column_name=*/""); + + auto paths = jf.flat_paths(); + EXPECT_EQ(std::find(paths.begin(), paths.end(), "k2"), paths.end()); +} + } // namespace starrocks diff --git a/docs/en/using_starrocks/Flat_json_column_paths.md b/docs/en/using_starrocks/Flat_json_column_paths.md new file mode 100644 index 00000000000000..1d295242e7453a --- /dev/null +++ b/docs/en/using_starrocks/Flat_json_column_paths.md @@ -0,0 +1,314 @@ +--- +displayed_sidebar: docs +sidebar_position: 111 +--- + +# Flat JSON Force-Flatten Paths (`column_paths`) + +Flat JSON automatically extracts JSON fields that appear densely across rows (see +[Flat JSON](./Flat_json.md)). The sparsity heuristic works well for typical +payloads, but some fields are important for queries even when they appear in +only a small fraction of rows. `flat_json.column_paths` lets you explicitly +force-flatten a list of JSON paths for a specific JSON column, bypassing the +sparsity check. + +This page describes the property syntax, how to set and update it, and how to +verify that the paths you specified were actually materialized as sub-columns. + +## When to use + +Use `column_paths` when a field is: + +- **Queried frequently** but absent from most rows (below the sparsity threshold). +- **Used in predicates or joins** that benefit from columnar access. +- **Part of a low-cardinality dictionary** path that would otherwise be skipped. + +Do NOT use it to force-flatten dense fields — those are picked up automatically +by the sparsity heuristic and consume the `flat_json.column.max` quota. + +## Syntax + +`column_paths` is scoped **per JSON column**. The property key encodes the JSON +column name: + +``` +flat_json.column_paths. = ", , ..." +flat_json.column_paths..add = ", " (ALTER only) +flat_json.column_paths..remove = "" (ALTER only) +flat_json.column_paths_max = (per-column cap) +``` + +- `` is the identifier of a JSON column in the table schema. + If the column does not exist or is not of JSON type, the DDL fails during analysis. +- Paths are comma-separated. Each path may start with `$.` (optional; stripped + at parse time). Nested paths use `.` as separator: `$.user.country`. +- The reserved suffixes `.add` and `.remove` are only valid inside + `ALTER TABLE ... SET (...)` — not at `CREATE TABLE` time. +- `flat_json.column_paths_max` caps the number of force-flattened columns **per + JSON column**, independent of `flat_json.column.max` (which caps + sparsity-derived columns). Default is `200`. + +## Prerequisites + +Flat JSON must be enabled on the table before any `column_paths` key is +accepted: + +```sql +ALTER TABLE t SET ("flat_json.enable" = "true"); +``` + +Attempting to set `column_paths` when Flat JSON is disabled raises a semantic +error. + +## Configure at `CREATE TABLE` time + +```sql +CREATE TABLE user_events ( + id BIGINT, + ts DATETIME, + events JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.null.factor" = "0.3", + "flat_json.sparsity.factor" = "0.5", + "flat_json.column.max" = "50", + -- Force-flatten events.browser and events.utm_source regardless of sparsity. + "flat_json.column_paths.events" = "$.browser, $.utm_source", + -- Per-column cap on force paths (optional; default is 200). + "flat_json.column_paths_max" = "20" +); +``` + +Multiple JSON columns each get their own entry: + +```sql +CREATE TABLE logs ( + id BIGINT, + request JSON, + response JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.column_paths.request" = "$.method, $.path", + "flat_json.column_paths.response" = "$.status, $.latency_ms" +); +``` + +## Update via `ALTER TABLE` + +Three operations are supported per JSON column. They can be combined in one +`ALTER` statement. + +### Full replace + +Overwrites the complete path list for a column. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events" = "$.browser, $.country, $.utm_source" +); +``` + +Setting the value to an empty string clears all force paths for that column +(the column falls back to pure sparsity-based flattening): + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths.events" = ""); +``` + +### Incremental add + +Appends paths not already in the list. Existing paths are preserved. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.add" = "$.device_id, $.session_id" +); +``` + +### Incremental remove + +Removes matching paths. Paths not in the current list are silently ignored. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.remove" = "$.utm_source" +); +``` + +### Global per-column cap + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths_max" = "50"); +``` + +Setting it to `0` resets to the server default (`200`). + +### Combined example + +```sql +ALTER TABLE logs SET ( + "flat_json.column_paths.request.add" = "$.user_agent", + "flat_json.column_paths.response.remove" = "$.latency_ms", + "flat_json.column_paths_max" = "100" +); +``` + +## Interaction with sparsity + +| Path state | Behavior | +|-----------------------------|-------------------------------------------------------------------| +| Listed in `column_paths` | Forced to a sub-column regardless of sparsity (if it has `hits > 0`). | +| Listed but never appears | Excluded (the engine does not create empty sub-columns). | +| Not listed, above sparsity | Auto-flattened via normal Flat JSON heuristic. | +| Not listed, below sparsity | Stays inside the remainder JSON blob. | + +The two quotas are independent: +- `flat_json.column.max` caps **auto-detected** sparse-flattened columns. +- `flat_json.column_paths_max` caps **forced** columns. + +## Verification: step-by-step + +Follow these steps to confirm that a configured path was actually materialized +as a sub-column. + +### Step 1. Enable Flat JSON and configure paths + +```sql +CREATE TABLE user_events ( + id BIGINT, ts DATETIME, events JSON +) DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.sparsity.factor" = "0.9", + "flat_json.column_paths.events" = "$.browser" +); +``` + +`sparsity.factor = 0.9` is set high to ensure that ordinary sparsity would NOT +flatten `$.browser`. This way, if `browser` shows up as a sub-column, it is +proof that `column_paths` took effect. + +### Step 2. Load data where the forced path is sparse + +```sql +INSERT INTO user_events VALUES + (1, now(), parse_json('{"session":"a","browser":"Chrome"}')), + (2, now(), parse_json('{"session":"b"}')), + (3, now(), parse_json('{"session":"c"}')), + (4, now(), parse_json('{"session":"d"}')), + (5, now(), parse_json('{"session":"e"}')); +``` + +`browser` appears in 1/5 rows (sparsity 20%), well below the 90% threshold. +Without `column_paths` it would NOT be flattened. + +### Step 3. Inspect materialized sub-columns + +Use `flat_json_meta` on the `[_META_]` pseudo-table: + +```sql +SELECT flat_json_meta(events) FROM user_events[_META_]; +``` + +Expected output (keys in any order): + +``` ++-------------------------------------------------------+ +| flat_json_meta(events) | ++-------------------------------------------------------+ +| ["session(VARCHAR)", "browser(VARCHAR)"] | ++-------------------------------------------------------+ +``` + +Seeing `browser(VARCHAR)` in the list confirms the force-flatten happened. + +If the table has multiple JSON columns, query each one individually: + +```sql +SELECT flat_json_meta(request) FROM logs[_META_]; +SELECT flat_json_meta(response) FROM logs[_META_]; +``` + +### Step 4. Verify query path usage via Query Profile + +Run a query that touches the forced path: + +```sql +SELECT get_json_string(events, '$.browser') +FROM user_events +WHERE get_json_string(events, '$.browser') = 'Chrome'; +``` + +Then inspect the profile (enable with `SET enable_profile = true;` then +`SHOW PROFILELIST;` and `ANALYZE PROFILE FROM ''`). Look for: + +- `PushdownAccessPaths`: non-zero — the planner pushed the path access down to storage. +- `AccessPathHits`: non-zero — the sub-column was read directly instead of going through JSON. +- `/events: ` under `AccessPathHits` — your JSON column appears as a hit entry. + +If `AccessPathHits` is zero but `AccessPathUnhits` is non-zero, the path was +pushed down but the sub-column did not exist for that rowset (typical right +after enabling the feature — new data is flattened but pre-existing segments +are not). Trigger compaction or reload the data to re-flatten historical rows. + +### Step 5. Force compaction for existing data (optional) + +New data is flattened at load time. For data that was already in the table +before you set `column_paths`, you can trigger compaction to re-flatten: + +```sql +ALTER TABLE user_events COMPACT; +``` + +Watch the BE log for lines like: + +``` +Compaction flat json column: nulls(TINYINT),browser(VARCHAR),session(VARCHAR) +``` + +Repeat Step 3 to confirm `browser` is now present. + +## Troubleshooting + +| Symptom | Cause | Fix | +|------------------------------------------------------------|--------------------------------------------------------------------------|----------------------------------------------------------------------------| +| `Property 'flat_json.column_paths.foo' references unknown or non-JSON column 'foo'` | Column `foo` doesn't exist or isn't `JSON`. | Use the correct JSON column name; check `DESC `. | +| `flat JSON configuration must be set after enabling flat JSON.` | Tried to set `column_paths` on a table with `flat_json.enable = false`. | `ALTER TABLE t SET ("flat_json.enable" = "true");` first. | +| `flat_json_meta` does not list a configured path | The path does not appear in ANY row yet (`hits = 0`). | Load data containing the path, or wait until such rows arrive. | +| `AccessPathHits = 0` but sub-column exists | Query reads old rowsets predating the config change. | Run `ALTER TABLE t COMPACT;` to re-flatten. | +| Configured paths exceed the quota | `flat_json.column_paths_max` reached for that JSON column. | Raise the cap, or prune the list with `.remove`. | +| Sync appears wrong on follower FE after removing paths | Older builds had an emit-only-if-non-empty bug. | This page's design emits the full state on every write (fixed). | + +## Metadata and cluster-wide consistency + +`column_paths` configuration is persisted via the FE EditLog +(`OP_MODIFY_FLAT_JSON_CONFIG`). Every FE in the cluster replays the same log, +so the leader and followers converge to the same configuration. The config is +also pushed to every BE as part of the tablet meta and applied on the next +flush/compaction. + +When debugging divergence, check: + +```sql +-- View current effective config +SHOW CREATE TABLE user_events; +``` + +The `PROPERTIES` block will list every `flat_json.column_paths.` entry +that is currently active. Compare this output between FE replicas; it must +match. + +## Related + +- [Flat JSON overview](./Flat_json.md) +- [CREATE TABLE reference](../sql-reference/sql-statements/table_bucket_part_index/CREATE_TABLE.md) +- [ALTER TABLE reference](../sql-reference/sql-statements/table_bucket_part_index/ALTER_TABLE.md) +- BE config: `json_flat_null_factor`, `json_flat_sparsity_factor`, + `json_flat_column_max`, `enable_compaction_flat_json` diff --git a/docs/zh/using_starrocks/Flat_json_column_paths.md b/docs/zh/using_starrocks/Flat_json_column_paths.md new file mode 100644 index 00000000000000..720f99bf76738d --- /dev/null +++ b/docs/zh/using_starrocks/Flat_json_column_paths.md @@ -0,0 +1,305 @@ +--- +displayed_sidebar: docs +sidebar_position: 111 +--- + +# Flat JSON 강제 평탄화 경로 (`column_paths`) + +Flat JSON은 여러 행에 걸쳐 빈번하게 등장하는 JSON 필드를 자동으로 별도 컬럼으로 +추출합니다 ([Flat JSON](./Flat_json.md) 참조). 희소성(sparsity) 휴리스틱은 대부분의 +페이로드에 잘 동작하지만, 일부 필드는 소수의 행에만 등장해도 쿼리 성능에 중요할 수 +있습니다. `flat_json.column_paths`는 **특정 JSON 컬럼**에 대해 지정한 경로들을 희소성 +체크를 무시하고 **강제로 평탄화**하도록 설정하는 기능입니다. + +이 문서는 해당 프로퍼티의 문법, 설정/변경 방법, 그리고 **지정한 경로가 실제로 +서브컬럼으로 materialize 되었는지 검증하는 단계별 방법**을 다룹니다. + +## 언제 사용하나 + +다음과 같은 필드에 사용하세요: + +- **자주 쿼리**되지만 **대부분의 행에는 없는** 필드 (희소성 임계값 미만). +- **조건절 또는 조인**에 쓰여서 컬럼 액세스로 이득을 보는 필드. +- 저-카디널리티 dictionary 최적화 대상 필드인데 희소성 때문에 스킵될 가능성이 있는 경우. + +**사용하지 말아야 할 경우:** 대부분의 행에 등장하는 dense 필드. 이런 필드는 기존 +휴리스틱이 자동으로 평탄화하며, `flat_json.column.max` 쿼터를 소비합니다. + +## 문법 + +`column_paths`는 **JSON 컬럼별**로 독립적으로 지정됩니다. 프로퍼티 키에 JSON 컬럼명을 +박아 스코프를 구분합니다: + +``` +flat_json.column_paths. = "<경로1>, <경로2>, ..." +flat_json.column_paths..add = "<경로A>, <경로B>" (ALTER 전용) +flat_json.column_paths..remove = "<경로C>" (ALTER 전용) +flat_json.column_paths_max = <정수> (컬럼별 상한) +``` + +- ``은 테이블 스키마에 존재하는 JSON 타입 컬럼이어야 합니다. 존재하지 않거나 + JSON 타입이 아니면 DDL analyze 단계에서 실패합니다. +- 경로는 쉼표로 구분합니다. 각 경로는 `$.` 접두사로 시작할 수 있으며(선택, 파싱 시 제거), + 중첩 경로는 `.`으로 구분합니다: `$.user.country`. +- 예약 접미사 `.add`, `.remove`는 `ALTER TABLE ... SET (...)`에서만 유효합니다 + (`CREATE TABLE`에서는 사용할 수 없음). +- `flat_json.column_paths_max`는 **JSON 컬럼당** 강제 평탄화 컬럼의 상한입니다 + (`flat_json.column.max`는 희소성 기반 컬럼 상한과 별개). 기본값은 `200`. + +## 전제 조건 + +`column_paths`를 설정하려면 먼저 Flat JSON이 활성화되어 있어야 합니다: + +```sql +ALTER TABLE t SET ("flat_json.enable" = "true"); +``` + +비활성 상태에서 `column_paths` 관련 키를 설정하면 semantic 에러가 발생합니다. + +## CREATE TABLE 시점에 설정 + +```sql +CREATE TABLE user_events ( + id BIGINT, + ts DATETIME, + events JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.null.factor" = "0.3", + "flat_json.sparsity.factor" = "0.5", + "flat_json.column.max" = "50", + -- events.browser, events.utm_source을 희소성 무시하고 강제 평탄화 + "flat_json.column_paths.events" = "$.browser, $.utm_source", + -- 강제 경로의 컬럼별 상한 (선택; 기본 200) + "flat_json.column_paths_max" = "20" +); +``` + +여러 JSON 컬럼이 있으면 각각 별도 키를 사용합니다: + +```sql +CREATE TABLE logs ( + id BIGINT, + request JSON, + response JSON +) +DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.column_paths.request" = "$.method, $.path", + "flat_json.column_paths.response" = "$.status, $.latency_ms" +); +``` + +## ALTER TABLE로 변경 + +JSON 컬럼별로 3가지 연산을 지원합니다. 한 `ALTER` 문에서 조합할 수 있습니다. + +### 전체 교체 + +해당 컬럼의 경로 리스트를 완전히 새 값으로 덮어씁니다. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events" = "$.browser, $.country, $.utm_source" +); +``` + +빈 문자열을 주면 해당 컬럼의 강제 경로를 모두 제거합니다 (이후 그 컬럼은 순수하게 +희소성 기반으로만 평탄화됩니다): + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths.events" = ""); +``` + +### 증분 추가 + +기존 리스트에 없는 경로만 뒤에 추가합니다. 기존 경로는 보존됩니다. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.add" = "$.device_id, $.session_id" +); +``` + +### 증분 제거 + +매칭되는 경로를 제거합니다. 현재 리스트에 없는 경로는 조용히 무시됩니다. + +```sql +ALTER TABLE user_events SET ( + "flat_json.column_paths.events.remove" = "$.utm_source" +); +``` + +### 컬럼별 상한 조정 + +```sql +ALTER TABLE user_events SET ("flat_json.column_paths_max" = "50"); +``` + +`0`으로 설정하면 서버 기본값(`200`)으로 복원됩니다. + +### 조합 예시 + +```sql +ALTER TABLE logs SET ( + "flat_json.column_paths.request.add" = "$.user_agent", + "flat_json.column_paths.response.remove" = "$.latency_ms", + "flat_json.column_paths_max" = "100" +); +``` + +## 희소성과의 상호작용 + +| 경로 상태 | 동작 | +|-----------------------------------------|-------------------------------------------------------------------| +| `column_paths`에 포함 | 희소성 무시하고 서브컬럼으로 만듦 (단, `hits > 0`이어야 함). | +| 포함됐지만 데이터에 전혀 등장하지 않음 | 제외 (빈 서브컬럼은 생성되지 않음). | +| 미포함, 희소성 임계치 이상 | 일반 Flat JSON 휴리스틱으로 자동 평탄화. | +| 미포함, 희소성 임계치 미만 | 잔여(remainder) JSON blob에 유지. | + +두 쿼터는 독립적으로 동작합니다: +- `flat_json.column.max`: **자동 감지된** 희소성 기반 컬럼의 상한. +- `flat_json.column_paths_max`: **강제 지정된** 컬럼의 상한. + +## 검증: 단계별 가이드 + +지정한 경로가 실제로 서브컬럼으로 materialize 되었는지 다음 단계로 확인합니다. + +### Step 1. Flat JSON 활성화 + 경로 지정 + +```sql +CREATE TABLE user_events ( + id BIGINT, ts DATETIME, events JSON +) DUPLICATE KEY (id) +DISTRIBUTED BY HASH(id) BUCKETS 4 +PROPERTIES ( + "flat_json.enable" = "true", + "flat_json.sparsity.factor" = "0.9", + "flat_json.column_paths.events" = "$.browser" +); +``` + +`sparsity.factor = 0.9`로 높게 설정해서, 일반 희소성 로직으로는 `$.browser`가 절대 +평탄화되지 않게 합니다. 이렇게 하면 `browser`가 서브컬럼으로 나타난다는 것은 곧 +`column_paths`가 동작했다는 증거가 됩니다. + +### Step 2. 해당 경로가 희소하게 나타나는 데이터 적재 + +```sql +INSERT INTO user_events VALUES + (1, now(), parse_json('{"session":"a","browser":"Chrome"}')), + (2, now(), parse_json('{"session":"b"}')), + (3, now(), parse_json('{"session":"c"}')), + (4, now(), parse_json('{"session":"d"}')), + (5, now(), parse_json('{"session":"e"}')); +``` + +`browser`는 1/5 (20%)에만 등장하므로 90% 임계치에 크게 못 미칩니다. 즉 +`column_paths`가 없었다면 평탄화되지 **않을** 데이터입니다. + +### Step 3. Materialize 된 서브컬럼 확인 + +`[_META_]` 가상 테이블에 `flat_json_meta` 함수를 사용합니다: + +```sql +SELECT flat_json_meta(events) FROM user_events[_META_]; +``` + +예상 결과 (키 순서는 상관없음): + +``` ++-------------------------------------------------------+ +| flat_json_meta(events) | ++-------------------------------------------------------+ +| ["session(VARCHAR)", "browser(VARCHAR)"] | ++-------------------------------------------------------+ +``` + +**목록에 `browser(VARCHAR)`가 보이면 강제 평탄화가 적용된 것입니다.** + +테이블에 JSON 컬럼이 여러 개라면 각각 조회: + +```sql +SELECT flat_json_meta(request) FROM logs[_META_]; +SELECT flat_json_meta(response) FROM logs[_META_]; +``` + +### Step 4. Query Profile로 서브컬럼 사용 확인 + +강제 경로를 건드리는 쿼리 실행: + +```sql +SELECT get_json_string(events, '$.browser') +FROM user_events +WHERE get_json_string(events, '$.browser') = 'Chrome'; +``` + +프로파일 확인 (`SET enable_profile = true;` → `SHOW PROFILELIST;` → +`ANALYZE PROFILE FROM ''`). 다음 지표를 보세요: + +- `PushdownAccessPaths`: 0보다 크면 플래너가 경로 접근을 스토리지로 push down한 것. +- `AccessPathHits`: 0보다 크면 서브컬럼을 직접 읽어 JSON 파싱 없이 값 접근 성공. +- `/events: ` 항목이 `AccessPathHits` 아래에 보임 — 해당 JSON 컬럼이 hit 되었음. + +`AccessPathHits = 0` 인데 `AccessPathUnhits > 0` 이면, 플래너는 push down했지만 해당 +rowset에 서브컬럼이 없는 상태입니다 (기능 활성화 직후에 흔함 — 신규 데이터는 +평탄화되지만 기존 세그먼트는 그대로). 컴팩션을 트리거하거나 데이터를 다시 적재해서 +재평탄화하세요. + +### Step 5. 기존 데이터에 대한 강제 컴팩션 (선택) + +신규 데이터는 로드 시점에 평탄화됩니다. `column_paths` 설정 이전의 데이터에 대해서는 +컴팩션을 트리거해서 재평탄화할 수 있습니다: + +```sql +ALTER TABLE user_events COMPACT; +``` + +BE 로그에서 다음과 같은 줄을 확인: + +``` +Compaction flat json column: nulls(TINYINT),browser(VARCHAR),session(VARCHAR) +``` + +그 후 Step 3을 반복해서 `browser`가 나타나는지 확인합니다. + +## 트러블슈팅 + +| 증상 | 원인 | 조치 | +|---------------------------------------------------------------|------------------------------------------------------------------------|----------------------------------------------------------------------------| +| `Property 'flat_json.column_paths.foo' references unknown or non-JSON column 'foo'` | `foo` 컬럼이 없거나 JSON 타입이 아님. | 올바른 JSON 컬럼명 사용. `DESC
`로 확인. | +| `flat JSON configuration must be set after enabling flat JSON.` | Flat JSON이 비활성 상태에서 `column_paths` 설정 시도. | 먼저 `ALTER TABLE t SET ("flat_json.enable" = "true");`. | +| `flat_json_meta`에 설정한 경로가 안 보임 | 아직 해당 경로가 어느 행에도 등장하지 않음 (`hits = 0`). | 해당 경로를 포함한 데이터를 적재하거나 등장할 때까지 대기. | +| 서브컬럼은 존재하지만 `AccessPathHits = 0` | 쿼리가 설정 변경 이전의 기존 rowset을 읽고 있음. | `ALTER TABLE t COMPACT;`로 재평탄화. | +| 지정한 경로 수가 쿼터 초과 | 해당 JSON 컬럼에 대해 `flat_json.column_paths_max`에 도달. | 상한을 올리거나 `.remove`로 경로를 줄임. | +| Remove 후 follower FE의 상태가 이상함 | 초기 버전의 emit-only-if-non-empty 버그. | 본 설계에서는 항상 전체 상태를 emit하여 수정됨. | + +## 메타데이터와 클러스터 전체 일관성 + +`column_paths` 설정은 FE EditLog(`OP_MODIFY_FLAT_JSON_CONFIG`)로 영속화됩니다. 모든 +FE가 같은 로그를 replay하므로 리더와 팔로워가 동일한 설정으로 수렴합니다. 설정은 +tablet meta의 일부로 모든 BE에도 전파되며, 다음 flush/compaction 시점에 반영됩니다. + +디버깅 시 확인 방법: + +```sql +-- 현재 유효한 설정 확인 +SHOW CREATE TABLE user_events; +``` + +`PROPERTIES` 블록에 현재 활성화된 `flat_json.column_paths.` 항목들이 표시됩니다. +FE replica들 간에 이 출력을 비교해서 동일한지 확인하세요. + +## 관련 문서 + +- [Flat JSON 개요](./Flat_json.md) +- [CREATE TABLE 레퍼런스](../sql-reference/sql-statements/table_bucket_part_index/CREATE_TABLE.md) +- [ALTER TABLE 레퍼런스](../sql-reference/sql-statements/table_bucket_part_index/ALTER_TABLE.md) +- BE 파라미터: `json_flat_null_factor`, `json_flat_sparsity_factor`, + `json_flat_column_max`, `enable_compaction_flat_json` diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java index 5a558b9b66a6ef..9b3f9d9f32f8bf 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobExecutor.java @@ -522,10 +522,8 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties)) { boolean isSuccess = schemaChangeHandler.updateFlatJsonConfigMeta(db, table.getId(), properties, TTabletMetaType.FLAT_JSON_CONFIG); if (!isSuccess) { diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java index 5577a382ab0439..c3cd0f87c7de32 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java @@ -2516,6 +2516,27 @@ public void updateTableMeta(Database db, String tableName, Map p } } + // Extracts the JSON column name from a key of the form + // flat_json.column_paths. (returns col_name) + // flat_json.column_paths..add (returns col_name) + // flat_json.column_paths..remove (returns col_name) + // Returns empty string if the key does not match. Used by ALTER to detect stale + // per-column properties that should be erased before leader->follower replication. + private static String extractColumnNameForFlatJsonKey(String key) { + if (!key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + return ""; + } + String suffix = key.substring(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + return suffix.substring(0, suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + } + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + return suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + } + return suffix; + } + public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map properties, TTabletMetaType metaType) { FlatJsonConfig newFlatJsonConfig; @@ -2551,10 +2572,8 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map columnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); - if (!columnPaths.equals(newFlatJsonConfig.getFlatJsonColumnPaths())) { - newFlatJsonConfig.setFlatJsonColumnPaths(columnPaths); + // Per-column full-replace: flat_json.column_paths.= "..." + java.util.Map> replaceMap = + PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + for (java.util.Map.Entry> entry : replaceMap.entrySet()) { + java.util.List existing = newFlatJsonConfig.getColumnPaths(entry.getKey()); + if (!entry.getValue().equals(existing)) { + newFlatJsonConfig.setColumnPaths(entry.getKey(), entry.getValue()); hasChanged = true; } } - // Incremental add: append paths not already in the list - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD)) { - java.util.List toAdd = PropertyAnalyzer.analyzeFlatJsonColumnPaths( - java.util.Collections.singletonMap( - PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, - properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD))); - java.util.List current = new java.util.ArrayList<>(newFlatJsonConfig.getFlatJsonColumnPaths()); - for (String path : toAdd) { + // Per-column incremental add: flat_json.column_paths..add = "..." + java.util.Map> addMap = PropertyAnalyzer.analyzeFlatJsonColumnPathsOps( + properties, PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD); + for (java.util.Map.Entry> entry : addMap.entrySet()) { + java.util.List current = new java.util.ArrayList<>( + newFlatJsonConfig.getColumnPaths(entry.getKey())); + boolean localChanged = false; + for (String path : entry.getValue()) { if (!current.contains(path)) { current.add(path); - hasChanged = true; + localChanged = true; } } - if (hasChanged) { - newFlatJsonConfig.setFlatJsonColumnPaths(current); + if (localChanged) { + newFlatJsonConfig.setColumnPaths(entry.getKey(), current); + hasChanged = true; } } - // Incremental remove: delete matching paths from the list - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE)) { - java.util.List toRemove = PropertyAnalyzer.analyzeFlatJsonColumnPaths( - java.util.Collections.singletonMap( - PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, - properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE))); - java.util.List current = new java.util.ArrayList<>(newFlatJsonConfig.getFlatJsonColumnPaths()); - if (current.removeAll(toRemove)) { - newFlatJsonConfig.setFlatJsonColumnPaths(current); + // Per-column incremental remove: flat_json.column_paths..remove = "..." + java.util.Map> removeMap = PropertyAnalyzer.analyzeFlatJsonColumnPathsOps( + properties, PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE); + for (java.util.Map.Entry> entry : removeMap.entrySet()) { + java.util.List current = new java.util.ArrayList<>( + newFlatJsonConfig.getColumnPaths(entry.getKey())); + if (current.removeAll(entry.getValue())) { + newFlatJsonConfig.setColumnPaths(entry.getKey(), current); hasChanged = true; } } @@ -2623,6 +2644,16 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map entry here flushes the + // stale key so the leader's properties map stays in sync with its config.) + olapTable.getTableProperty().getProperties().keySet().removeIf(k -> + k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX) && + !newFlatJsonConfig.getFlatJsonColumnPaths().containsKey( + extractColumnNameForFlatJsonKey(k))); if (!hasChanged) { LOG.info("table {} flat json config is same as the previous config, so nothing need to do", olapTable.getName()); return true; diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java index 84d11e2f0fe69a..2459b99b4b4350 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java @@ -23,8 +23,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; public class FlatJsonConfig implements Writable { @SerializedName("flatJsonEnable") @@ -39,13 +41,13 @@ public class FlatJsonConfig implements Writable { @SerializedName("flatJsonColumnMax") private int flatJsonColumnMax; - // Paths that must always be flattened, regardless of sparsity. - // Stored in the internal dot-separated format (no leading "$."). + // Per-JSON-column force-flatten paths: -> list of dot-separated paths + // (no leading "$."). Empty/absent entry means the column has no forced paths. @SerializedName("flatJsonColumnPaths") - private List flatJsonColumnPaths; + private Map> flatJsonColumnPaths; - // Upper bound on the number of force-path columns (independent of flatJsonColumnMax). - // 0 means "use system default" (Config.flat_json_column_paths_max). + // Upper bound on force-flatten columns per JSON column (independent of flatJsonColumnMax). + // 0 means "use BE default". @SerializedName("flatJsonColumnPathsMax") private int flatJsonColumnPathsMax; @@ -54,7 +56,7 @@ public FlatJsonConfig(boolean enabled, double nullFactor, double sparsityFactor, this.flatJsonNullFactor = nullFactor; this.flatJsonSparsityFactor = sparsityFactor; this.flatJsonColumnMax = columnMax; - this.flatJsonColumnPaths = Collections.emptyList(); + this.flatJsonColumnPaths = new LinkedHashMap<>(); this.flatJsonColumnPathsMax = 0; } @@ -63,7 +65,7 @@ public FlatJsonConfig(FlatJsonConfig config) { this.flatJsonNullFactor = config.getFlatJsonNullFactor(); this.flatJsonSparsityFactor = config.getFlatJsonSparsityFactor(); this.flatJsonColumnMax = config.getFlatJsonColumnMax(); - this.flatJsonColumnPaths = new ArrayList<>(config.getFlatJsonColumnPaths()); + this.flatJsonColumnPaths = deepCopyPaths(config.getFlatJsonColumnPaths()); this.flatJsonColumnPathsMax = config.getFlatJsonColumnPathsMax(); } @@ -89,14 +91,15 @@ public void buildFromProperties(Map properties) { flatJsonColumnMax = Integer.parseInt(properties.get( PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)); } - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { - flatJsonColumnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); - } + // Replace (not merge) per-column paths from the properties map. This is the EditLog + // replay path on follower FEs: leader emits the full state in toProperties(), so this + // must overwrite any stale in-memory entries. Callers that want incremental ops + // (.add/.remove) resolve them BEFORE serializing to properties. + Map> perCol = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + flatJsonColumnPaths = new LinkedHashMap<>(perCol); if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { int max = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); - if (max >= 0) { - flatJsonColumnPathsMax = max; - } + flatJsonColumnPathsMax = Math.max(max, 0); } } @@ -132,12 +135,32 @@ public void setFlatJsonColumnMax(int flatJsonColumnMax) { this.flatJsonColumnMax = flatJsonColumnMax; } - public List getFlatJsonColumnPaths() { - return flatJsonColumnPaths == null ? Collections.emptyList() : flatJsonColumnPaths; + public Map> getFlatJsonColumnPaths() { + return flatJsonColumnPaths == null ? Collections.emptyMap() : flatJsonColumnPaths; + } + + public void setFlatJsonColumnPaths(Map> paths) { + this.flatJsonColumnPaths = paths == null ? new LinkedHashMap<>() : new LinkedHashMap<>(paths); } - public void setFlatJsonColumnPaths(List paths) { - this.flatJsonColumnPaths = paths == null ? Collections.emptyList() : paths; + // Returns the path list for a single JSON column (empty list if absent). + public List getColumnPaths(String columnName) { + Map> map = getFlatJsonColumnPaths(); + List list = map.get(columnName); + return list == null ? Collections.emptyList() : list; + } + + // Replaces the path list for a single JSON column. Passing an empty/null list REMOVES + // the entry so the column falls back to pure sparsity-based flattening. + public void setColumnPaths(String columnName, List paths) { + if (flatJsonColumnPaths == null) { + flatJsonColumnPaths = new LinkedHashMap<>(); + } + if (paths == null || paths.isEmpty()) { + flatJsonColumnPaths.remove(columnName); + } else { + flatJsonColumnPaths.put(columnName, new ArrayList<>(paths)); + } } public int getFlatJsonColumnPathsMax() { @@ -148,37 +171,46 @@ public void setFlatJsonColumnPathsMax(int max) { this.flatJsonColumnPathsMax = max; } + // Serializes this config back into a flat properties map for EditLog persistence. + // + // CRITICAL: the EditLog replay path on follower FEs uses + // TableProperty.modifyTableProperties(map) which does putAll() (merge, not replace) into + // the existing properties map. If we conditionally omit keys for empty/zero values, + // followers would retain stale values after a user removes all paths or resets _max to 0, + // diverging from the leader's state. To keep leader and followers in sync we: + // 1. Always emit the global _max key (so 0 reliably resets it). + // 2. Emit one key per surviving column. Keys that previously existed but are now + // absent must be cleared on the follower; SchemaChangeHandler's ALTER path + // explicitly erases stale keys before calling toProperties() (see there for details). public Map toProperties() { Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE, String.valueOf(flatJsonEnable)); - - // Only include other flat JSON properties if flat JSON is enabled if (flatJsonEnable) { properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR, String.valueOf(flatJsonNullFactor)); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR, String.valueOf(flatJsonSparsityFactor)); properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, String.valueOf(flatJsonColumnMax)); - List fps = getFlatJsonColumnPaths(); - if (!fps.isEmpty()) { - properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, String.join(",", fps)); - } - if (flatJsonColumnPathsMax > 0) { - properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX, - String.valueOf(flatJsonColumnPathsMax)); + properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX, + String.valueOf(flatJsonColumnPathsMax)); + if (flatJsonColumnPaths != null) { + for (Map.Entry> entry : flatJsonColumnPaths.entrySet()) { + String key = PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX + entry.getKey(); + properties.put(key, String.join(",", entry.getValue())); + } } } return properties; } - public TFlatJsonConfig toTFlatJsonConfig() { + public TFlatJsonConfig toThrift() { TFlatJsonConfig tFlatJsonConfig = new TFlatJsonConfig(); tFlatJsonConfig.setFlat_json_enable(flatJsonEnable); tFlatJsonConfig.setFlat_json_null_factor(flatJsonNullFactor); tFlatJsonConfig.setFlat_json_sparsity_factor(flatJsonSparsityFactor); tFlatJsonConfig.setFlat_json_column_max(flatJsonColumnMax); - List fps = getFlatJsonColumnPaths(); - if (!fps.isEmpty()) { - tFlatJsonConfig.setFlat_json_column_paths(fps); + Map> paths = getFlatJsonColumnPaths(); + if (!paths.isEmpty()) { + tFlatJsonConfig.setFlat_json_column_paths(new TreeMap<>(paths)); } if (flatJsonColumnPathsMax > 0) { tFlatJsonConfig.setFlat_json_column_paths_max(flatJsonColumnPathsMax); @@ -186,6 +218,16 @@ public TFlatJsonConfig toTFlatJsonConfig() { return tFlatJsonConfig; } + private static Map> deepCopyPaths(Map> src) { + Map> dst = new LinkedHashMap<>(); + if (src != null) { + for (Map.Entry> e : src.entrySet()) { + dst.put(e.getKey(), new ArrayList<>(e.getValue())); + } + } + return dst; + } + @Override public String toString() { return String.format("{ flat_json_enable : %b,\n " + diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index a316692ad7edce..9c158491842324 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -3155,13 +3155,16 @@ public Map getUniqueProperties() { properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, flatJsonColumnMax); } - // flat json force paths - String flatJsonColumnPaths = tableProperties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS); - if (!Strings.isNullOrEmpty(flatJsonColumnPaths)) { - properties.put(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, flatJsonColumnPaths); + // flat json per-column force paths: forward any key under the prefix verbatim + for (Map.Entry entry : tableProperties.entrySet()) { + String key = entry.getKey(); + if (key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX) + && !Strings.isNullOrEmpty(entry.getValue())) { + properties.put(key, entry.getValue()); + } } - // flat json force paths max + // flat json force paths global max String flatJsonColumnPathsMax = tableProperties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); if (!Strings.isNullOrEmpty(flatJsonColumnPathsMax)) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java index d17609127d3e72..492f1782f28846 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java @@ -559,8 +559,8 @@ public TableProperty buildFlatJsonConfig() { properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties)) { boolean enableFlatJson = PropertyAnalyzer.analyzeFlatJsonEnabled(properties); // In gsonPostProcess, we should be tolerant of existing properties even when flat_json.enable is false. @@ -575,9 +575,12 @@ public TableProperty buildFlatJsonConfig() { PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, Config.flat_json_column_max); flatJsonConfig = new FlatJsonConfig(enableFlatJson, flatJsonNullFactor, flatJsonSparsityFactory, flatJsonColumnMax); - List columnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); - if (!columnPaths.isEmpty()) { - flatJsonConfig.setFlatJsonColumnPaths(columnPaths); + // Per-column force paths. analyzeFlatJsonColumnPaths only picks up full-replace + // keys (flat_json.column_paths.); .add/.remove are incremental ops handled + // by SchemaChangeHandler before reaching this path. + Map> perCol = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); + if (!perCol.isEmpty()) { + flatJsonConfig.setFlatJsonColumnPaths(perCol); } int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); if (columnPathsMax >= 0) { diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index e6c8c3b161ffb0..ed9c9afb513f6f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -177,16 +177,21 @@ public class PropertyAnalyzer { public static final String PROPERTIES_FLAT_JSON_COLUMN_MAX = "flat_json.column.max"; - // Comma-separated list of JSON paths that must always be flattened, - // regardless of sparsity (e.g. "event_ts,event_id" or "$.event_ts,$.event_id"). - public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS = "flat_json.column_paths"; - - // Upper bound on the number of user-specified columns (independent of flat_json.column.max). - public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX = "flat_json.column_paths.max"; - - // Incremental add/remove operations for column_paths (ALTER TABLE SET only). - public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD = "flat_json.column_paths.add"; - public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE = "flat_json.column_paths.remove"; + // Per-JSON-column force-flatten paths are expressed as keys of the form: + // flat_json.column_paths. = "$.path1, $.path2" (full replace) + // flat_json.column_paths..add = "$.path3" (incremental add) + // flat_json.column_paths..remove = "$.path1" (incremental remove) + // Reserved suffixes: "add", "remove" (cannot be used as json column names for the .add/.remove ops). + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX = "flat_json.column_paths."; + + // Operation suffixes for incremental updates (ALTER TABLE SET only). + public static final String FLAT_JSON_COLUMN_PATHS_OP_ADD = "add"; + public static final String FLAT_JSON_COLUMN_PATHS_OP_REMOVE = "remove"; + + // Global upper bound on force-flatten columns per JSON column. Distinct from flat_json.column.max + // which caps auto-detected sparse-derived columns. Uses underscore (not dot) to avoid + // collision with a user-supplied JSON column literally named "max". + public static final String PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX = "flat_json.column_paths_max"; public static final String PROPERTIES_STORAGE_TYPE_COLUMN = "column"; public static final String PROPERTIES_STORAGE_TYPE_COLUMN_WITH_ROW = "column_with_row"; @@ -624,11 +629,30 @@ public static boolean analyzeFlatJsonEnabled(Map properties) { return flatJsonEnabled; } - public static java.util.List analyzeFlatJsonColumnPaths(Map properties) { - if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { + // Returns true if the properties map contains ANY flat_json.column_paths.* key (replace/add/remove) + // or the global flat_json.column_paths_max key. + public static boolean hasFlatJsonColumnPathsProperty(Map properties) { + if (properties == null) { + return false; + } + if (properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + return true; + } + for (String key : properties.keySet()) { + if (key.startsWith(PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + return true; + } + } + return false; + } + + // Parses a comma-separated list of JSON paths. Strips the optional "$." prefix so the + // returned strings are in internal dot-separated form ("a.b.c"). Empty tokens are skipped. + private static java.util.List parseFlatJsonPathList(String raw) { + if (raw == null) { return java.util.Collections.emptyList(); } - String raw = properties.get(PROPERTIES_FLAT_JSON_COLUMN_PATHS).trim(); + raw = raw.trim(); if (raw.isEmpty()) { return java.util.Collections.emptyList(); } @@ -645,6 +669,69 @@ public static java.util.List analyzeFlatJsonColumnPaths(Map[.]" into (columnName, op). + // op is "" for bare keys (full replace), or "add"/"remove" for incremental ops. + // Returns null if the key is not a flat_json.column_paths.* key. + private static String[] splitColumnPathsKey(String key) { + if (!key.startsWith(PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + return null; + } + String suffix = key.substring(PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + if (suffix.isEmpty()) { + return null; + } + // Check if the key ends with ".add" or ".remove" (operation suffix). + if (suffix.endsWith("." + FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + String col = suffix.substring(0, suffix.length() - FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + if (!col.isEmpty()) { + return new String[] {col, FLAT_JSON_COLUMN_PATHS_OP_ADD}; + } + } else if (suffix.endsWith("." + FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + String col = suffix.substring(0, suffix.length() - FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + if (!col.isEmpty()) { + return new String[] {col, FLAT_JSON_COLUMN_PATHS_OP_REMOVE}; + } + } + return new String[] {suffix, ""}; + } + + // Extracts per-column full-replace entries: "flat_json.column_paths." -> List. + // Keys with .add/.remove suffix are NOT included here (use analyzeFlatJsonColumnPathsOps). + public static java.util.Map> analyzeFlatJsonColumnPaths( + Map properties) { + java.util.Map> result = new java.util.HashMap<>(); + if (properties == null) { + return result; + } + for (Map.Entry entry : properties.entrySet()) { + String[] parts = splitColumnPathsKey(entry.getKey()); + if (parts == null || !parts[1].isEmpty()) { + continue; + } + result.put(parts[0], parseFlatJsonPathList(entry.getValue())); + } + return result; + } + + // Extracts per-column incremental op entries. + // op = "add" -> keys of the form "flat_json.column_paths..add" + // op = "remove" -> keys of the form "flat_json.column_paths..remove" + public static java.util.Map> analyzeFlatJsonColumnPathsOps( + Map properties, String op) { + java.util.Map> result = new java.util.HashMap<>(); + if (properties == null) { + return result; + } + for (Map.Entry entry : properties.entrySet()) { + String[] parts = splitColumnPathsKey(entry.getKey()); + if (parts == null || !op.equals(parts[1])) { + continue; + } + result.put(parts[0], parseFlatJsonPathList(entry.getValue())); + } + return result; + } + public static int analyzeFlatJsonColumnPathsMax(Map properties) { if (properties == null || !properties.containsKey(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { return -1; @@ -653,10 +740,10 @@ public static int analyzeFlatJsonColumnPathsMax(Map properties) try { max = Integer.parseInt(properties.get(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)); } catch (NumberFormatException e) { - throw new SemanticException("flat_json.column_paths.max: " + e.getMessage()); + throw new SemanticException(PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX + ": " + e.getMessage()); } if (max < 0) { - throw new SemanticException("Illegal flat_json.column_paths.max: " + max); + throw new SemanticException("Illegal " + PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX + ": " + max); } return max; } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java index df4de283fe2738..371cf6451f757e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java @@ -916,8 +916,8 @@ private void processFlatJsonConfig(Map properties, OlapTable tab if (!enableFlatJson && (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX))) { + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties))) { throw new DdlException("flat JSON configuration must be set after enabling flat JSON."); } @@ -927,15 +927,22 @@ private void processFlatJsonConfig(Map properties, OlapTable tab PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR, Config.flat_json_sparsity_factory); int flatJsonColumnMax = PropertyAnalyzer.analyzeIntProp(properties, PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX, Config.flat_json_column_max); - java.util.List columnPaths = PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); - properties.remove(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS); + // Per-column force-flatten paths (only full-replace keys are valid at CREATE time; + // .add/.remove are ALTER-only ops). + java.util.Map> perCol = + PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + + // Remove the per-column keys from the raw properties map so downstream handlers + // don't re-process them, but let them be re-emitted later via FlatJsonConfig.toProperties(). + properties.keySet().removeIf(k -> + k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)); properties.remove(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); FlatJsonConfig flatJsonConfig = new FlatJsonConfig(enableFlatJson, flatJsonNullFactor, flatJsonSparsityFactory, flatJsonColumnMax); - if (!columnPaths.isEmpty()) { - flatJsonConfig.setFlatJsonColumnPaths(columnPaths); + if (!perCol.isEmpty()) { + flatJsonConfig.setFlatJsonColumnPaths(perCol); } if (columnPathsMax >= 0) { flatJsonConfig.setFlatJsonColumnPathsMax(columnPathsMax); @@ -953,7 +960,7 @@ private boolean hasFlatJsonProperties(Map properties) { properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX); + properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java index ca9efa1739f85b..234d5dab539336 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java @@ -451,30 +451,50 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, " haven't been enabled"); } } - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE) || - properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties)) { if (table instanceof OlapTable) { OlapTable olapTable = (OlapTable) table; if (olapTable.getFlatJsonConfig() == null || !olapTable.getFlatJsonConfig().getFlatJsonEnable()) { ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, "Property " + PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE + " haven't been enabled"); } - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS)) { - PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); - } - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD)) { - PropertyAnalyzer.analyzeFlatJsonColumnPaths( - java.util.Collections.singletonMap( - PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, - properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_ADD))); + // Collect the table's JSON column names so we can reject unknown targets early. + java.util.Set jsonColumnNames = new java.util.HashSet<>(); + for (Column col : olapTable.getBaseSchema()) { + if (col.getType() != null && col.getType().isJsonType()) { + jsonColumnNames.add(col.getName()); + } } - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE)) { + // Validate every column_paths.* key references an existing JSON column. The + // per-column prefix covers both the bare "" replace form and the + // ".add"/".remove" incremental forms. + for (String key : properties.keySet()) { + if (!key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + continue; + } + String suffix = key.substring(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + String columnName; + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + columnName = suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + } else if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + columnName = suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + } else { + columnName = suffix; + } + if (columnName.isEmpty()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Invalid property key '" + key + "': missing JSON column name"); + } + if (!jsonColumnNames.contains(columnName)) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Property '" + key + "' references unknown or non-JSON column '" + columnName + "'"); + } + // Run path-format validation (throws SemanticException on bad input). PropertyAnalyzer.analyzeFlatJsonColumnPaths( - java.util.Collections.singletonMap( - PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS, - properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_REMOVE))); + java.util.Collections.singletonMap(key, properties.get(key))); } if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 7ec2bac4058b09..da207df02eb4db 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -280,12 +280,21 @@ message BinlogConfigPB { optional int64 binlog_max_size = 4; } +// Per-JSON-column force-flatten paths. column_name is the JSON column's +// identifier in the table schema; paths are dot-separated (no leading "$."). +message FlatJsonColumnPathsEntryPB { + optional string column_name = 1; + repeated string paths = 2; +} + message FlatJsonConfigPB { optional bool flat_json_enable = 1; optional double flat_json_null_factor = 2; optional double flat_json_sparsity_factor = 3; optional int64 flat_json_max_column_max = 4; - repeated string flat_json_column_paths = 5; + // Per-JSON-column paths that must always be flattened, regardless of sparsity. + repeated FlatJsonColumnPathsEntryPB flat_json_column_paths = 5; + // Upper bound on force-flattened columns per JSON column (0 => server default). optional int64 flat_json_column_paths_max = 6; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 4a3d959921bdc0..35b1fbf67ff89e 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -90,7 +90,8 @@ struct TFlatJsonConfig { 2: optional double flat_json_null_factor; 3: optional double flat_json_sparsity_factor; 4: optional i64 flat_json_column_max; - 5: optional list flat_json_column_paths; + // Per-JSON-column force-flatten paths: column_name -> list of dot-separated paths (no leading "$."). + 5: optional map> flat_json_column_paths; 6: optional i64 flat_json_column_paths_max; } From 4d9c55b31e368bec9227520062a76e93ce5d7765 Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Wed, 22 Apr 2026 20:51:47 +0900 Subject: [PATCH 5/9] [Revert] Restore original public API removed/renamed in prior refactor - Rename toThrift() back to toTFlatJsonConfig() in FlatJsonConfig.java; CreateReplicaTask.java still calls the original name and the rename would cause a compilation error. - Restore the 4-parameter update(bool, double, double, int) helper in flat_json_config.h that was removed in the refactor commit; other parts of the codebase may rely on it and the original author had intent behind exposing it. Co-Authored-By: Claude Sonnet 4.6 --- be/src/storage/flat_json_config.h | 8 ++++++++ .../main/java/com/starrocks/catalog/FlatJsonConfig.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/be/src/storage/flat_json_config.h b/be/src/storage/flat_json_config.h index 8f886fc8fe213c..99db73ac69ebb5 100644 --- a/be/src/storage/flat_json_config.h +++ b/be/src/storage/flat_json_config.h @@ -140,6 +140,14 @@ class FlatJsonConfig { } } + // Update function using four parameters (kept for backward compatibility with original API) + void update(bool enable, double nullFactor, double sparsityFactor, int maxColumnMax) { + _flat_json_enable = enable; + _flat_json_null_factor = nullFactor; + _flat_json_sparsity_factor = sparsityFactor; + _flat_json_max_column_max = maxColumnMax; + } + // Copy Assignment FlatJsonConfig& operator=(const FlatJsonConfig& other) { if (this != &other) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java index 2459b99b4b4350..fd03fd3d07e853 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/FlatJsonConfig.java @@ -202,7 +202,7 @@ public Map toProperties() { return properties; } - public TFlatJsonConfig toThrift() { + public TFlatJsonConfig toTFlatJsonConfig() { TFlatJsonConfig tFlatJsonConfig = new TFlatJsonConfig(); tFlatJsonConfig.setFlat_json_enable(flatJsonEnable); tFlatJsonConfig.setFlat_json_null_factor(flatJsonNullFactor); From 4aeea89dc38da0551670a060e2ad9a2345837525 Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Wed, 22 Apr 2026 21:16:20 +0900 Subject: [PATCH 6/9] [BugFix] Purge stale flat_json.column_paths.* keys before EditLog replay merge modifyTableProperties() uses putAll (merge, not replace). When a user removes all forced paths for a JSON column, the corresponding flat_json.column_paths.key is absent from FlatJsonConfig.toProperties(). On follower FEs the stale key survives the putAll and causes buildFromProperties() to reconstruct a FlatJsonConfig with ghost paths, silently forcing paths that should no longer be columnized. Fix: purge all flat_json.column_paths.* keys from the existing properties map before the putAll when replaying OP_MODIFY_FLAT_JSON_CONFIG, mirroring the SchemaChangeHandler.removeIf() that already does this on the leader. Co-Authored-By: Claude Sonnet 4.6 --- .../src/main/java/com/starrocks/server/LocalMetastore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 5e5b8c3f6950d1..e8c7cd5649add5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -4332,6 +4332,7 @@ public void modifyFlatJsonMeta(Database db, OlapTable table, FlatJsonConfig flat flatJsonConfig.toProperties() ); GlobalStateMgr.getCurrentState().getEditLog().logModifyFlatJsonConfig(info, wal -> { + table.getTableProperty().modifyTableProperties(flatJsonConfig.toProperties()); table.setFlatJsonConfig(flatJsonConfig); }); } @@ -4594,6 +4595,10 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation olapTable.setHasDelete(); } else { TableProperty tableProperty = olapTable.getTableProperty(); + if (opCode == OperationType.OP_MODIFY_FLAT_JSON_CONFIG) { + tableProperty.getProperties().keySet() + .removeIf(k -> k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)); + } tableProperty.modifyTableProperties(properties); tableProperty.buildProperty(opCode); From 202d48cd5a0d8a5731ca6e5c3a19e27325595b3b Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Wed, 22 Apr 2026 22:38:23 +0900 Subject: [PATCH 7/9] [BugFix] Validate JSON column existence for flat_json.column_paths at CREATE TABLE CREATE TABLE did not verify that column names referenced in flat_json.column_paths.* properties are actual JSON-typed columns in the table schema. Unlike ALTER TABLE (which already validates via AlterTableClauseAnalyzer), CREATE TABLE silently accepted invalid column names and stored them, causing BE to silently ignore the configured paths. Added the same JSON-column existence check used in AlterTableClauseAnalyzer to processFlatJsonConfig() in OlapTableFactory so that CREATE TABLE fails fast with a clear error when a non-existent or non-JSON column is referenced. --- .../com/starrocks/server/OlapTableFactory.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java index 371cf6451f757e..9b15bcec3e5de2 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java @@ -933,6 +933,22 @@ private void processFlatJsonConfig(Map properties, OlapTable tab PropertyAnalyzer.analyzeFlatJsonColumnPaths(properties); int columnPathsMax = PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + // Validate that every column named in perCol is a JSON-typed column in the schema. + if (!perCol.isEmpty()) { + java.util.Set jsonColumnNames = new java.util.HashSet<>(); + for (Column col : table.getBaseSchema()) { + if (col.getType() != null && col.getType().isJsonType()) { + jsonColumnNames.add(col.getName()); + } + } + for (String colName : perCol.keySet()) { + if (!jsonColumnNames.contains(colName)) { + throw new DdlException( + "flat_json.column_paths references unknown or non-JSON column: '" + colName + "'"); + } + } + } + // Remove the per-column keys from the raw properties map so downstream handlers // don't re-process them, but let them be re-emitted later via FlatJsonConfig.toProperties(). properties.keySet().removeIf(k -> From d02002bd204d78bc5ada3978e99e27ea888a88e3 Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Thu, 23 Apr 2026 21:19:05 +0900 Subject: [PATCH 8/9] fix(flat_json): close validation gap and race condition in column_paths ALTER Three issues fixed: 1. AlterTableClauseAnalyzer: when flat_json.enable and flat_json.column_paths.* were present in the same ALTER TABLE, the else-if chain caused column_paths validation (JSON column existence check, path format) to be silently skipped. Extracted the validation into validateFlatJsonColumnPathsProperties() and call it from both the enable branch and the dedicated column_paths branch. 2. AlterTableClauseAnalyzer: the properties.size() != 1 guard did not include FLAT_JSON_PROPERTY_PREFIX, so users could not bundle flat_json.enable and flat_json.column_paths.* in a single ALTER TABLE statement. Added the prefix to the existing multi-property allowlist alongside dynamic_partition and binlog. 3. SchemaChangeHandler: stale flat_json.column_paths.* key removal ran without holding the write lock, creating a narrow window where SHOW CREATE TABLE could observe the old keys already erased but the new values not yet written by the WAL lambda. Moved the removal inside the WRITE lock block immediately before modifyFlatJsonMeta so the removal and WAL update are atomic. Co-Authored-By: Claude Sonnet 4.6 --- .../starrocks/alter/SchemaChangeHandler.java | 20 ++-- .../analyzer/AlterTableClauseAnalyzer.java | 96 +++++++++++-------- 2 files changed, 64 insertions(+), 52 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java index c3cd0f87c7de32..01ae29443b37f4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java @@ -2644,16 +2644,6 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map entry here flushes the - // stale key so the leader's properties map stays in sync with its config.) - olapTable.getTableProperty().getProperties().keySet().removeIf(k -> - k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX) && - !newFlatJsonConfig.getFlatJsonColumnPaths().containsKey( - extractColumnNameForFlatJsonKey(k))); if (!hasChanged) { LOG.info("table {} flat json config is same as the previous config, so nothing need to do", olapTable.getName()); return true; @@ -2661,6 +2651,16 @@ public boolean updateFlatJsonConfigMeta(Database db, Long tableId, Map + k.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX) && + !newFlatJsonConfig.getFlatJsonColumnPaths().containsKey( + extractColumnNameForFlatJsonKey(k))); GlobalStateMgr.getCurrentState().getLocalMetastore().modifyFlatJsonMeta(db, olapTable, newFlatJsonConfig); } catch (Exception e) { isModifiedSuccess = false; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java index 234d5dab539336..3a4691f4b84189 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java @@ -204,6 +204,48 @@ public Void visitTableRenameClause(TableRenameClause clause, ConnectContext cont return null; } + // Validates flat_json.column_paths.* (and column_paths_max) keys in the given properties map + // against the table's JSON columns. Throws SemanticException on any violation. + // Called from both the flat_json.enable branch and the dedicated column_paths branch so that + // bundling enable + column_paths in a single ALTER TABLE does not bypass validation. + private void validateFlatJsonColumnPathsProperties(Map properties, OlapTable olapTable) { + java.util.Set jsonColumnNames = new java.util.HashSet<>(); + for (Column col : olapTable.getBaseSchema()) { + if (col.getType() != null && col.getType().isJsonType()) { + jsonColumnNames.add(col.getName()); + } + } + for (String key : properties.keySet()) { + if (!key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { + continue; + } + String suffix = key.substring(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); + String columnName; + if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD)) { + columnName = suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); + } else if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { + columnName = suffix.substring(0, + suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); + } else { + columnName = suffix; + } + if (columnName.isEmpty()) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Invalid property key '" + key + "': missing JSON column name"); + } + if (!jsonColumnNames.contains(columnName)) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "Property '" + key + "' references unknown or non-JSON column '" + columnName + "'"); + } + PropertyAnalyzer.analyzeFlatJsonColumnPaths( + java.util.Collections.singletonMap(key, properties.get(key))); + } + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { + PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); + } + } + @Override public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, ConnectContext context) { Map properties = clause.getProperties(); @@ -217,7 +259,8 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, if (properties.size() != 1 && !(TableProperty.isSamePrefixProperties(properties, TableProperty.DYNAMIC_PARTITION_PROPERTY_PREFIX) - || TableProperty.isSamePrefixProperties(properties, TableProperty.BINLOG_PROPERTY_PREFIX))) { + || TableProperty.isSamePrefixProperties(properties, TableProperty.BINLOG_PROPERTY_PREFIX) + || TableProperty.isSamePrefixProperties(properties, TableProperty.FLAT_JSON_PROPERTY_PREFIX))) { ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, "Can only set one table property at a time"); } @@ -430,8 +473,16 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, e.getMessage()); } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE)) { - // Allow setting flat_json.enable to true or false + // Validate the enable flag itself. PropertyAnalyzer.analyzeFlatJsonEnabled(properties); + // When column_paths properties are bundled in the same ALTER, validate them here. + // The else-if chain would otherwise skip column_paths validation entirely when + // flat_json.enable is also present. + if (table instanceof OlapTable && + (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || + PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties))) { + validateFlatJsonColumnPathsProperties(properties, (OlapTable) table); + } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_SPARSITY_FACTOR) || properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_MAX)) { @@ -459,46 +510,7 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, "Property " + PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE + " haven't been enabled"); } - // Collect the table's JSON column names so we can reject unknown targets early. - java.util.Set jsonColumnNames = new java.util.HashSet<>(); - for (Column col : olapTable.getBaseSchema()) { - if (col.getType() != null && col.getType().isJsonType()) { - jsonColumnNames.add(col.getName()); - } - } - // Validate every column_paths.* key references an existing JSON column. The - // per-column prefix covers both the bare "" replace form and the - // ".add"/".remove" incremental forms. - for (String key : properties.keySet()) { - if (!key.startsWith(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX)) { - continue; - } - String suffix = key.substring(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_PREFIX.length()); - String columnName; - if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD)) { - columnName = suffix.substring(0, - suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_ADD.length() - 1); - } else if (suffix.endsWith("." + PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE)) { - columnName = suffix.substring(0, - suffix.length() - PropertyAnalyzer.FLAT_JSON_COLUMN_PATHS_OP_REMOVE.length() - 1); - } else { - columnName = suffix; - } - if (columnName.isEmpty()) { - ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, - "Invalid property key '" + key + "': missing JSON column name"); - } - if (!jsonColumnNames.contains(columnName)) { - ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, - "Property '" + key + "' references unknown or non-JSON column '" + columnName + "'"); - } - // Run path-format validation (throws SemanticException on bad input). - PropertyAnalyzer.analyzeFlatJsonColumnPaths( - java.util.Collections.singletonMap(key, properties.get(key))); - } - if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX)) { - PropertyAnalyzer.analyzeFlatJsonColumnPathsMax(properties); - } + validateFlatJsonColumnPathsProperties(properties, olapTable); } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY)) { if (!properties.get(PropertyAnalyzer.PROPERTIES_COMPACTION_STRATEGY).equalsIgnoreCase("default") && From 4010d6f9c7d661a91c31722c67f8f15433edeb7f Mon Sep 17 00:00:00 2001 From: Moweon Lee Date: Thu, 23 Apr 2026 21:38:03 +0900 Subject: [PATCH 9/9] fix: reject flat_json.column_paths when flat_json.enable=false in bundled ALTER When flat_json.enable=false and flat_json.column_paths.* are combined in a single ALTER TABLE, the Analyzer previously called validateFlatJsonColumnPathsProperties without first verifying that enable=true, allowing an invalid combination to pass FE validation. The execution layer (updateFlatJsonConfigMeta) would then throw a RuntimeException instead of a proper SemanticException. Add an explicit guard in the enable branch: if flat_json.enable is not "true" and column_paths properties are present, report a SemanticException immediately so the error is clear and caught at the right layer. Co-Authored-By: Claude Sonnet 4.6 --- .../com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java index 3a4691f4b84189..0c34ebda891eac 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AlterTableClauseAnalyzer.java @@ -481,6 +481,11 @@ public Void visitModifyTablePropertiesClause(ModifyTablePropertiesClause clause, if (table instanceof OlapTable && (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_COLUMN_PATHS_MAX) || PropertyAnalyzer.hasFlatJsonColumnPathsProperty(properties))) { + if (!"true".equalsIgnoreCase(properties.get(PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE))) { + ErrorReport.reportSemanticException(ErrorCode.ERR_COMMON_ERROR, + "flat_json.column_paths can only be set when " + + PropertyAnalyzer.PROPERTIES_FLAT_JSON_ENABLE + " is true"); + } validateFlatJsonColumnPathsProperties(properties, (OlapTable) table); } } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_FLAT_JSON_NULL_FACTOR) ||