From 8a34c4506bb5fe161ee629abd0c73354013abe0d Mon Sep 17 00:00:00 2001 From: alvin-phoenix-ai Date: Thu, 25 Jun 2026 16:33:08 -0700 Subject: [PATCH 1/2] [Refactor] Move snapshot to data_workflows Signed-off-by: alvin-phoenix-ai --- be/module_boundary_manifest.json | 3 --- be/src/agent/agent_task.cpp | 2 +- be/src/agent/task_worker_pool.cpp | 1 - be/src/data_workflows/CMakeLists.txt | 3 +++ .../snapshot}/lake_snapshot_loader.cpp | 6 ++--- .../snapshot}/lake_snapshot_loader.h | 0 .../snapshot}/snapshot_loader.cpp | 24 ++---------------- .../snapshot}/snapshot_loader.h | 25 ++++++++----------- be/src/exec/data_sinks/export_sink.cpp | 2 +- be/src/exec/data_sinks/file_result_writer.cpp | 2 +- be/src/exec/file_scanner/file_scanner.cpp | 2 +- be/src/exec/file_scanner/parquet_scanner.cpp | 1 - .../pipeline/sink/export_sink_operator.cpp | 2 +- be/src/fs/CMakeLists.txt | 1 - be/src/platform/CMakeLists.txt | 1 + be/src/{fs => platform}/fs_broker.cpp | 2 +- be/src/{fs => platform}/fs_broker.h | 0 be/src/runtime/CMakeLists.txt | 2 -- be/src/service/service_be/lake_service.cpp | 2 +- be/test/CMakeLists.txt | 3 +-- be/test/data_workflows/CMakeLists.txt | 1 + .../snapshot}/snapshot_loader_test.cpp | 2 +- be/test/platform/CMakeLists.txt | 1 + be/test/{fs => platform}/fs_broker_test.cpp | 2 +- 24 files changed, 32 insertions(+), 58 deletions(-) rename be/src/{runtime => data_workflows/snapshot}/lake_snapshot_loader.cpp (99%) rename be/src/{runtime => data_workflows/snapshot}/lake_snapshot_loader.h (100%) rename be/src/{runtime => data_workflows/snapshot}/snapshot_loader.cpp (97%) rename be/src/{runtime => data_workflows/snapshot}/snapshot_loader.h (83%) rename be/src/{fs => platform}/fs_broker.cpp (99%) rename be/src/{fs => platform}/fs_broker.h (100%) rename be/test/{runtime => data_workflows/snapshot}/snapshot_loader_test.cpp (98%) rename be/test/{fs => platform}/fs_broker_test.cpp (99%) diff --git a/be/module_boundary_manifest.json b/be/module_boundary_manifest.json index 7b0c2288b5b734..170c52c188ad9c 100644 --- a/be/module_boundary_manifest.json +++ b/be/module_boundary_manifest.json @@ -160,7 +160,6 @@ "allowed_include_prefixes": ["fs/", "io/", "common/", "base/", "gutil/", "gen_cpp/"], "forbidden_include_prefixes": ["fs/azure/", "fs/credential/", "fs/hdfs/", "fs/s3/"], "forbidden_includes": [ - "fs/fs_broker.h", "fs/fs_options_helper.h", "fs/fs_s3.h", "fs/key_cache.h", @@ -307,7 +306,6 @@ "runtime/hive_table_sink.h", "runtime/iceberg_table_sink.h", "runtime/jdbc_driver_manager.h", - "runtime/lake_snapshot_loader.h", "runtime/lake_tablets_channel.h", "runtime/load_channel.h", "runtime/load_channel_mgr.h", @@ -324,7 +322,6 @@ "runtime/result_sink.h", "runtime/schema_table_sink.h", "runtime/small_file_mgr.h", - "runtime/snapshot_loader.h", "runtime/table_function_table_sink.h", "runtime/tablets_channel.h", "runtime/variable_result_writer.h" diff --git a/be/src/agent/agent_task.cpp b/be/src/agent/agent_task.cpp index 26ace4e34427a2..5f4dfd1bb5c917 100644 --- a/be/src/agent/agent_task.cpp +++ b/be/src/agent/agent_task.cpp @@ -31,11 +31,11 @@ #include "data_workflows/consistency/engine_checksum_task.h" #include "data_workflows/migration/engine_storage_migration_task.h" #include "data_workflows/schema_change/engine_alter_tablet_task.h" +#include "data_workflows/snapshot/snapshot_loader.h" #include "gutil/strings/join.h" #include "io/io_profiler.h" #include "runtime/current_thread.h" #include "runtime/exec_env.h" -#include "runtime/snapshot_loader.h" #include "storage/lake/replication_txn_manager.h" #include "storage/lake/schema_change.h" #include "storage/lake/tablet_manager.h" diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index a996c3d5826817..28ed0ae6f91eac 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -71,7 +71,6 @@ #include "gen_cpp/Types_types.h" #include "runtime/current_thread.h" #include "runtime/exec_env.h" -#include "runtime/snapshot_loader.h" #include "storage/data_dir.h" #include "storage/lake/tablet_manager.h" #include "storage/primitive/storage_ids.h" diff --git a/be/src/data_workflows/CMakeLists.txt b/be/src/data_workflows/CMakeLists.txt index 1273852f7571b3..6da582c86ae971 100644 --- a/be/src/data_workflows/CMakeLists.txt +++ b/be/src/data_workflows/CMakeLists.txt @@ -30,6 +30,8 @@ set(DATA_WORKFLOWS_FILES load/tablet_writer/local_tablets_channel.cpp load/tablet_writer/tablets_channel.cpp migration/engine_storage_migration_task.cpp + snapshot/lake_snapshot_loader.cpp + snapshot/snapshot_loader.cpp schema_change/engine_alter_tablet_task.cpp ) @@ -42,6 +44,7 @@ target_link_libraries(DataWorkflows PUBLIC Exec Runtime StoragePrimitive + Platform Common Base Gutil diff --git a/be/src/runtime/lake_snapshot_loader.cpp b/be/src/data_workflows/snapshot/lake_snapshot_loader.cpp similarity index 99% rename from be/src/runtime/lake_snapshot_loader.cpp rename to be/src/data_workflows/snapshot/lake_snapshot_loader.cpp index b4e8930808fff2..30020bffc76b9d 100644 --- a/be/src/runtime/lake_snapshot_loader.cpp +++ b/be/src/data_workflows/snapshot/lake_snapshot_loader.cpp @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include "data_workflows/snapshot/lake_snapshot_loader.h" #include "base/container/raw_container.h" #include "base/network/network_util.h" #include "common/config_rpc_client_fwd.h" -#include "fs/fs_broker.h" +#include "data_workflows/snapshot/snapshot_loader.h" #include "fs/fs_util.h" #include "gen_cpp/TFileBrokerService.h" #include "gen_cpp/lake_service.pb.h" +#include "platform/fs_broker.h" #include "runtime/exec_env.h" -#include "runtime/snapshot_loader.h" #include "storage/lake/filenames.h" #include "storage/lake/lake_proto_normalizer.h" #include "storage/lake/tablet.h" diff --git a/be/src/runtime/lake_snapshot_loader.h b/be/src/data_workflows/snapshot/lake_snapshot_loader.h similarity index 100% rename from be/src/runtime/lake_snapshot_loader.h rename to be/src/data_workflows/snapshot/lake_snapshot_loader.h diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/data_workflows/snapshot/snapshot_loader.cpp similarity index 97% rename from be/src/runtime/snapshot_loader.cpp rename to be/src/data_workflows/snapshot/snapshot_loader.cpp index 20759029e6a024..702e90b1fed027 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/data_workflows/snapshot/snapshot_loader.cpp @@ -12,27 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// This file is based on code available under the Apache license here: -// https://github.com/apache/incubator-doris/blob/master/be/src/runtime/snapshot_loader.cpp - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/snapshot_loader.h" +#include "data_workflows/snapshot/snapshot_loader.h" #include #include @@ -43,7 +23,6 @@ #include "common/logging.h" #include "common/system/master_info.h" #include "fs/fs.h" -#include "fs/fs_broker.h" #include "fs/fs_factory.h" #include "fs/fs_util.h" #include "gen_cpp/FileBrokerService_types.h" @@ -52,6 +31,7 @@ #include "gen_cpp/HeartbeatService_types.h" #include "gen_cpp/TFileBrokerService.h" #include "platform/broker_mgr.h" +#include "platform/fs_broker.h" #include "runtime/exec_env.h" #include "storage/index/index_descriptor.h" #ifndef __APPLE__ diff --git a/be/src/runtime/snapshot_loader.h b/be/src/data_workflows/snapshot/snapshot_loader.h similarity index 83% rename from be/src/runtime/snapshot_loader.h rename to be/src/data_workflows/snapshot/snapshot_loader.h index eea5da0643d81a..e3bc9549346647 100644 --- a/be/src/runtime/snapshot_loader.h +++ b/be/src/data_workflows/snapshot/snapshot_loader.h @@ -1,19 +1,16 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at +// Copyright 2021-present StarRocks, Inc. All rights reserved. // -// http://www.apache.org/licenses/LICENSE-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #pragma once diff --git a/be/src/exec/data_sinks/export_sink.cpp b/be/src/exec/data_sinks/export_sink.cpp index 89ce39ff4f731d..136b03d72f821b 100644 --- a/be/src/exec/data_sinks/export_sink.cpp +++ b/be/src/exec/data_sinks/export_sink.cpp @@ -44,9 +44,9 @@ #include "exprs/expr.h" #include "exprs/expr_executor.h" #include "exprs/expr_factory.h" -#include "fs/fs_broker.h" #include "fs/fs_factory.h" #include "gutil/strings/substitute.h" +#include "platform/fs_broker.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" diff --git a/be/src/exec/data_sinks/file_result_writer.cpp b/be/src/exec/data_sinks/file_result_writer.cpp index 483056361bb072..e3eb43f589f9b5 100644 --- a/be/src/exec/data_sinks/file_result_writer.cpp +++ b/be/src/exec/data_sinks/file_result_writer.cpp @@ -45,9 +45,9 @@ #include "exec/plain_text_builder.h" #include "formats/csv/converter.h" #include "formats/io/formatted_output_stream.h" -#include "fs/fs_broker.h" #include "fs/fs_factory.h" #include "gutil/strings/substitute.h" +#include "platform/fs_broker.h" #include "runtime/runtime_state.h" namespace starrocks { diff --git a/be/src/exec/file_scanner/file_scanner.cpp b/be/src/exec/file_scanner/file_scanner.cpp index 81f9884b6f0533..6e606fab3befcc 100644 --- a/be/src/exec/file_scanner/file_scanner.cpp +++ b/be/src/exec/file_scanner/file_scanner.cpp @@ -34,10 +34,10 @@ #include "exprs/expr_executor.h" #include "exprs/expr_factory.h" #include "fs/fs.h" -#include "fs/fs_broker.h" #include "fs/fs_factory.h" #include "gutil/strings/substitute.h" #include "io/compressed_input_stream.h" +#include "platform/fs_broker.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" diff --git a/be/src/exec/file_scanner/parquet_scanner.cpp b/be/src/exec/file_scanner/parquet_scanner.cpp index 5ae92de167ab83..77d95ef0129725 100644 --- a/be/src/exec/file_scanner/parquet_scanner.cpp +++ b/be/src/exec/file_scanner/parquet_scanner.cpp @@ -23,7 +23,6 @@ #include "compute_env/load_path/rejected_record_writer.h" #include "exprs/cast_expr.h" #include "exprs/column_ref.h" -#include "fs/fs_broker.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" diff --git a/be/src/exec/pipeline/sink/export_sink_operator.cpp b/be/src/exec/pipeline/sink/export_sink_operator.cpp index f92d1efab30d61..fd64cd015ceb99 100644 --- a/be/src/exec/pipeline/sink/export_sink_operator.cpp +++ b/be/src/exec/pipeline/sink/export_sink_operator.cpp @@ -28,8 +28,8 @@ #include "exprs/expr_factory.h" #include "formats/csv/converter.h" #include "formats/io/formatted_output_stream.h" -#include "fs/fs_broker.h" #include "fs/fs_factory.h" +#include "platform/fs_broker.h" #include "runtime/runtime_state.h" namespace starrocks::pipeline { diff --git a/be/src/fs/CMakeLists.txt b/be/src/fs/CMakeLists.txt index 57924c6d75d429..6c91618bde26af 100644 --- a/be/src/fs/CMakeLists.txt +++ b/be/src/fs/CMakeLists.txt @@ -40,7 +40,6 @@ target_link_libraries(FSCore PUBLIC IO) set(EXEC_FILES fs_provider_bootstrap.cpp fs_options_helper.cpp - fs_broker.cpp fs_s3.cpp azure/azblob_uri.cpp azure/fs_azblob.cpp diff --git a/be/src/platform/CMakeLists.txt b/be/src/platform/CMakeLists.txt index 5f873e3fd957f3..aa042eaad4da64 100644 --- a/be/src/platform/CMakeLists.txt +++ b/be/src/platform/CMakeLists.txt @@ -22,6 +22,7 @@ set(PLATFORM_FILES aws/poco_common.cpp aws/poco_http_client.cpp aws/poco_http_client_factory.cpp + fs_broker.cpp broker_mgr.cpp download_util.cpp key_cache.cpp diff --git a/be/src/fs/fs_broker.cpp b/be/src/platform/fs_broker.cpp similarity index 99% rename from be/src/fs/fs_broker.cpp rename to be/src/platform/fs_broker.cpp index 2f31af118ba87a..2dc045331f03f7 100644 --- a/be/src/fs/fs_broker.cpp +++ b/be/src/platform/fs_broker.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "fs/fs_broker.h" +#include "platform/fs_broker.h" #include diff --git a/be/src/fs/fs_broker.h b/be/src/platform/fs_broker.h similarity index 100% rename from be/src/fs/fs_broker.h rename to be/src/platform/fs_broker.h diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 221c4426cff4ad..707c8cd7be57e3 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -90,7 +90,6 @@ set(RUNTIME_FILES runtime_state_helper.cpp rejected_record_sync_daemon.cpp fragment_mgr.cpp - snapshot_loader.cpp stream_load/transaction_mgr.cpp stream_load/stream_load_executor.cpp batch_write/isomorphic_batch_write.cpp @@ -105,7 +104,6 @@ set(RUNTIME_FILES external_scan_context_mgr.cpp runtime_filter_worker.cpp runtime_filter_serde.cpp - lake_snapshot_loader.cpp command_executor.cpp diagnose_daemon.cpp ) diff --git a/be/src/service/service_be/lake_service.cpp b/be/src/service/service_be/lake_service.cpp index 3094f07c5b95b6..673e7dfe9b0ab4 100644 --- a/be/src/service/service_be/lake_service.cpp +++ b/be/src/service/service_be/lake_service.cpp @@ -35,13 +35,13 @@ #include "common/thread/thread.h" #include "common/thread/threadpool.h" #include "data_workflows/load/tablet_writer/load_channel_mgr.h" +#include "data_workflows/snapshot/lake_snapshot_loader.h" #include "exec/write_combined_txn_log.h" #include "fs/fs_factory.h" #include "fs/fs_util.h" #include "gen_cpp/tablet_schema.pb.h" #include "gutil/strings/join.h" #include "runtime/exec_env.h" -#include "runtime/lake_snapshot_loader.h" #include "storage/lake/compaction_policy.h" #include "storage/lake/compaction_scheduler.h" #include "storage/lake/compaction_task.h" diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 857481728f9aae..56952602bd8e73 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -39,7 +39,7 @@ set(EXEC_FILES ./fs/azure/fs_azblob_test.cpp ./fs/azure/utils_test.cpp ./fs/credential/cloud_configuration_factory_test.cpp - ./fs/fs_broker_test.cpp + ./platform/fs_broker_test.cpp ./fs/fs_hdfs_test.cpp ./fs/fs_posix_test.cpp ./fs/fs_memory_test.cpp @@ -270,7 +270,6 @@ set(EXEC_FILES #./runtime/routine_load_task_executor_test.cpp ./runtime/routine_load/data_consumer_test.cpp ./runtime/small_file_mgr_test.cpp - ./runtime/snapshot_loader_test.cpp ./compute_env/load/time_bounded_stream_load_pipe_test.cpp ./runtime/merge_cascade_test.cpp ./exec/data_sinks/memory_scratch_sink_test_issue_8676.cpp diff --git a/be/test/data_workflows/CMakeLists.txt b/be/test/data_workflows/CMakeLists.txt index 9e357716534bca..94a7fa28a5446b 100644 --- a/be/test/data_workflows/CMakeLists.txt +++ b/be/test/data_workflows/CMakeLists.txt @@ -24,6 +24,7 @@ set(DATA_WORKFLOWS_TEST_FILES load/tablet_writer/load_channel_test.cpp load/tablet_writer/local_tablets_channel_test.cpp push_handler_test.cpp + snapshot/snapshot_loader_test.cpp ) add_library(starrocks_data_workflows_test_objs OBJECT ${DATA_WORKFLOWS_TEST_FILES}) diff --git a/be/test/runtime/snapshot_loader_test.cpp b/be/test/data_workflows/snapshot/snapshot_loader_test.cpp similarity index 98% rename from be/test/runtime/snapshot_loader_test.cpp rename to be/test/data_workflows/snapshot/snapshot_loader_test.cpp index d49f13e0a8d52d..d5e2ccf27dc3ec 100644 --- a/be/test/runtime/snapshot_loader_test.cpp +++ b/be/test/data_workflows/snapshot/snapshot_loader_test.cpp @@ -26,7 +26,7 @@ #define private public // hack complier #define protected public -#include "runtime/snapshot_loader.h" +#include "data_workflows/snapshot/snapshot_loader.h" #undef protected #undef private diff --git a/be/test/platform/CMakeLists.txt b/be/test/platform/CMakeLists.txt index 928935f89b33b9..d0623e1b4798e7 100644 --- a/be/test/platform/CMakeLists.txt +++ b/be/test/platform/CMakeLists.txt @@ -15,6 +15,7 @@ set(PLATFORM_TEST_FILES ../base/cxa_throw_wrap.cpp download_util_test.cpp + fs_broker_test.cpp key_cache_test.cpp llm/llm_cache_test.cpp llm/model_config_test.cpp diff --git a/be/test/fs/fs_broker_test.cpp b/be/test/platform/fs_broker_test.cpp similarity index 99% rename from be/test/fs/fs_broker_test.cpp rename to be/test/platform/fs_broker_test.cpp index 966d508f18a498..0c9551ab88b7eb 100644 --- a/be/test/fs/fs_broker_test.cpp +++ b/be/test/platform/fs_broker_test.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "fs/fs_broker.h" +#include "platform/fs_broker.h" #include #include From 478f829b4da8f8c6abc261e47b2693df7031c11e Mon Sep 17 00:00:00 2001 From: alvin-phoenix-ai Date: Thu, 25 Jun 2026 18:32:37 -0700 Subject: [PATCH 2/2] Fix Comment Signed-off-by: alvin-phoenix-ai --- .../snapshot/snapshot_loader.cpp | 20 +++++++++++++++++++ .../data_workflows/snapshot/snapshot_loader.h | 17 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/be/src/data_workflows/snapshot/snapshot_loader.cpp b/be/src/data_workflows/snapshot/snapshot_loader.cpp index 702e90b1fed027..02160b1aa217a4 100644 --- a/be/src/data_workflows/snapshot/snapshot_loader.cpp +++ b/be/src/data_workflows/snapshot/snapshot_loader.cpp @@ -12,6 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. +// This file is based on code available under the Apache license here: +// https://github.com/apache/incubator-doris/blob/master/be/src/runtime/snapshot_loader.cpp + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "data_workflows/snapshot/snapshot_loader.h" #include diff --git a/be/src/data_workflows/snapshot/snapshot_loader.h b/be/src/data_workflows/snapshot/snapshot_loader.h index e3bc9549346647..0b0020e2b80f21 100644 --- a/be/src/data_workflows/snapshot/snapshot_loader.h +++ b/be/src/data_workflows/snapshot/snapshot_loader.h @@ -12,6 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include