[opt](optimizer) Remove unused code for unify code

.github/CODEOWNERS

@@ -16,3 +16,4 @@
 #
 be/src/io/* @platoneko @gavinchou @dataroaring
 fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman
+**/pom.xml @CalvinKirs @morningman

.github/workflows/comment-to-trigger-teamcity.yml

@@ -150,6 +150,7 @@ jobs:
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'"  ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_fe_ut }}" \
@@ -158,7 +159,7 @@ jobs:
             "feut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run feut"
+          echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run feut"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -175,6 +176,7 @@ jobs:
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_be_ut }}" \
@@ -183,7 +185,7 @@ jobs:
             "beut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run beut"
+          echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run beut"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -197,15 +199,16 @@ jobs:
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_cloud_ut }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "cloudut" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch is not master, skip run cloudut"
+          echo "PR target branch is not in (master, branch-3.0), skip run cloudut"
         fi
 
     - name: "Trigger or Skip compile"
@@ -279,16 +282,17 @@ jobs:
           echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
         fi
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
-          echo "PR target branch in (master), need run cloud_p0"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+          echo "PR target branch is in (master, branch-3.0), need run cloud_p0"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_cloud_p0 }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "cloud_p0" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch not in (master), skip run cloud_p0"
+          echo "PR target branch is not in (master, branch-3.0), skip run cloud_p0"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -305,16 +309,17 @@ jobs:
           echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit
         fi
         set -x
-        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
-          echo "PR target branch in (master), need run cloud_p1"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+          echo "PR target branch is in (master, branch-3.0), need run cloud_p1"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_cloud_p1 }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "cloud_p1" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch not in (master), skip run cloud_p1"
+          echo "PR target branch is not in (master, branch-3.0), skip run cloud_p1"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -341,16 +346,17 @@ jobs:
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
         if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
               "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
-          echo "PR target branch in (master, branch-2.0), need run performance"
+          echo "PR target branch is in (master, branch-2.0, branch-3.0), need run performance"
           trigger_or_skip_build \
             "${{ steps.changes.outputs.changed_performance }}" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
             "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
             "performance" \
             "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
         else
-          echo "PR target branch not in (master, branch-2.0), skip run performance"
+          echo "PR target branch is not in (master, branch-2.0, branch-3.0), skip run performance"
           trigger_or_skip_build \
             "false" \
             "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \

.github/workflows/labeler/scope-label-conf.yml

@@ -23,3 +23,6 @@ meta-change:
   - fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
   - gensrc/thrift/*
   - gensrc/proto/*
+
+doing:
+  - '**'

.github/workflows/lfs-warning.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Check Large File'
+
+on: [push, pull_request_target]
+
+jobs:
+  large-file-checker:
+    name: "Check large file"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+
+      - name: "Checkout lfs-warning commit"
+        run: |
+          rm -rf ./.github/actions/lfs-warning
+          git clone https://github.com/ppremk/lfs-warning .github/actions/lfs-warning
+          pushd .github/actions/lfs-warning &>/dev/null
+          git checkout 4b98a8a5e6c429c23c34eee02d71553bca216425
+          popd &>/dev/null
+
+      - name: "Check Large File"
+        uses: ./.github/actions/lfs-warning
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filesizelimit: 1MB
+

be/CMakeLists.txt

@@ -157,6 +157,7 @@ set(BOOST_VERSION "1.81.0")
 
 if (NOT APPLE)
     find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system date_time)
+    find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system container)
 else()
     find_package(Boost ${BOOST_VERSION} COMPONENTS system date_time)
     find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
@@ -298,12 +299,11 @@ if (COMPILER_CLANG)
                         -Wno-implicit-float-conversion
                         -Wno-implicit-int-conversion
                         -Wno-sign-conversion
+                        -Wno-missing-field-initializers
+                        -Wno-unused-const-variable
                         -Wno-shorten-64-to-32)
     if (USE_LIBCPP)
         add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
-        if (NOT OS_MACOSX)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-lstdc++>)
-        endif()
         add_definitions(-DUSE_LIBCPP)
     endif()
 endif ()
@@ -373,29 +373,29 @@ endif()
 # For CMAKE_BUILD_TYPE=Debug
 if (OS_MACOSX AND ARCH_ARM)
     # Using -O0 may meet ARM64 branch out of range errors when linking with tcmalloc.
-    set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -Og")
+    set(CXX_FLAGS_DEBUG "-Og")
 else()
-    set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -O0")
+    set(CXX_FLAGS_DEBUG "-O0")
 endif()
 
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
 #   -DNDEBUG: Turn off dchecks/asserts/debug only code.
-set(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG")
-set(CXX_FLAGS_ASAN "${CXX_GCC_FLAGS} -O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
-set(CXX_FLAGS_LSAN "${CXX_GCC_FLAGS} -O0 -fsanitize=leak -DLEAK_SANITIZER")
+set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
+set(CXX_FLAGS_ASAN "-O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
+set(CXX_FLAGS_LSAN "-O0 -fsanitize=leak -DLEAK_SANITIZER")
 ## Use for BE-UT
-set(CXX_FLAGS_ASAN_UT "${CXX_GCC_FLAGS} -O0 -fsanitize=address -DADDRESS_SANITIZER")
+set(CXX_FLAGS_ASAN_UT "-O0 -fsanitize=address -DADDRESS_SANITIZER")
 
 # Set the flags to the undefined behavior sanitizer, also known as "ubsan"
 # Turn on sanitizer and debug symbols to get stack traces:
-set(CXX_FLAGS_UBSAN "${CXX_GCC_FLAGS} -O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
+set(CXX_FLAGS_UBSAN "-O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
 
 # Set the flags to the thread sanitizer, also known as "tsan"
 # Turn on sanitizer and debug symbols to get stack traces:
 # Use -Wno-builtin-declaration-mismatch to mute warnings like "new declaration ‘__tsan_atomic16 __tsan_atomic16_fetch_nand(..."
 # If use -O0 to compile, BE will stack overflow when start. https://github.com/apache/doris/issues/8868
-set(CXX_FLAGS_TSAN "${CXX_GCC_FLAGS} -O1 -fsanitize=thread -DTHREAD_SANITIZER -Wno-missing-declarations")
+set(CXX_FLAGS_TSAN "-O1 -fsanitize=thread -DTHREAD_SANITIZER -Wno-missing-declarations")
 
 # Set compile flags based on the build type.
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
@@ -517,6 +517,7 @@ find_package(absl)
 # add it here first.
 set(COMMON_THIRDPARTY
     Boost::date_time
+    Boost::container
     ${COMMON_THIRDPARTY}
 )
 
@@ -559,7 +560,6 @@ endif()
 if (OS_MACOSX)
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}
-        Boost::container
         bfd
         iberty
         intl
@@ -603,9 +603,11 @@ if (NOT OS_MACOSX)
         ${DORIS_DEPENDENCIES}
         -static-libstdc++
         -static-libgcc
-        -lstdc++fs
         -lresolv
     )
+    if (NOT (USE_LIBCPP AND COMPILER_CLANG))
+        set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs)
+    endif()
 else()
     set(DORIS_LINK_LIBS
         ${DORIS_LINK_LIBS}

be/src/agent/task_worker_pool.cpp

@@ -104,6 +104,10 @@ std::unordered_map<TTaskType::type, std::unordered_set<int64_t>> s_task_signatur
 
 std::atomic_ulong s_report_version(time(nullptr) * 10000);
 
+void increase_report_version() {
+    s_report_version.fetch_add(1, std::memory_order_relaxed);
+}
+
 // FIXME(plat1ko): Paired register and remove task info
 bool register_task_info(const TTaskType::type task_type, int64_t signature) {
     if (task_type == TTaskType::type::PUSH_STORAGE_POLICY ||
@@ -214,7 +218,7 @@ void alter_tablet(StorageEngine& engine, const TAgentTaskRequest& agent_task_req
     }
 
     if (status.ok()) {
-        s_report_version.fetch_add(1, std::memory_order_relaxed);
+        increase_report_version();
     }
 
     // Return result to fe
@@ -290,7 +294,7 @@ void alter_cloud_tablet(CloudStorageEngine& engine, const TAgentTaskRequest& age
     }
 
     if (status.ok()) {
-        s_report_version.fetch_add(1, std::memory_order_relaxed);
+        increase_report_version();
     }
 
     // Return result to fe
@@ -1536,7 +1540,7 @@ void create_tablet_callback(StorageEngine& engine, const TAgentTaskRequest& req)
                 .tag("tablet_id", create_tablet_req.tablet_id)
                 .error(status);
     } else {
-        s_report_version.fetch_add(1, std::memory_order_relaxed);
+        increase_report_version();
         // get path hash of the created tablet
         TabletSharedPtr tablet;
         {
@@ -1631,7 +1635,7 @@ void push_callback(StorageEngine& engine, const TAgentTaskRequest& req) {
                 .tag("signature", req.signature)
                 .tag("tablet_id", push_req.tablet_id)
                 .tag("push_type", push_req.push_type);
-        ++s_report_version;
+        increase_report_version();
         finish_task_request.__set_finish_tablet_infos(tablet_infos);
     } else {
         LOG_WARNING("failed to execute push task")
@@ -1677,7 +1681,7 @@ void cloud_push_callback(CloudStorageEngine& engine, const TAgentTaskRequest& re
                 .tag("signature", req.signature)
                 .tag("tablet_id", push_req.tablet_id)
                 .tag("push_type", push_req.push_type);
-        ++s_report_version;
+        increase_report_version();
         auto& tablet_info = finish_task_request.finish_tablet_infos.emplace_back();
         // Just need tablet_id
         tablet_info.tablet_id = push_req.tablet_id;
@@ -1974,6 +1978,10 @@ void clone_callback(StorageEngine& engine, const TMasterInfo& master_info,
         LOG_INFO("successfully clone tablet")
                 .tag("signature", req.signature)
                 .tag("tablet_id", clone_req.tablet_id);
+        if (engine_task.is_new_tablet()) {
+            increase_report_version();
+            finish_task_request.__set_report_version(s_report_version);
+        }
         finish_task_request.__set_finish_tablet_infos(tablet_infos);
     }
 

be/src/agent/workload_group_listener.cpp

@@ -61,6 +61,9 @@ void WorkloadGroupListener::handle_topic_info(const std::vector<TopicInfo>& topi
         // 4 create and update task scheduler
         wg->upsert_task_scheduler(&workload_group_info, _exec_env);
 
+        // 5 upsert io throttle
+        wg->upsert_scan_io_throttle(&workload_group_info);
+
         LOG(INFO) << "[topic_publish_wg]update workload group finish, wg info="
                   << wg->debug_string() << ", enable_cpu_hard_limit="
                   << (_exec_env->workload_group_mgr()->enable_cpu_hard_limit() ? "true" : "false")

be/src/cloud/cloud_base_compaction.cpp

@@ -237,7 +237,13 @@ Status CloudBaseCompaction::execute_compact() {
 
     using namespace std::chrono;
     auto start = steady_clock::now();
-    RETURN_IF_ERROR(CloudCompactionMixin::execute_compact());
+    auto res = CloudCompactionMixin::execute_compact();
+    if (!res.ok()) {
+        LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << res
+                     << ", tablet=" << _tablet->tablet_id()
+                     << ", output_version=" << _output_version;
+        return res;
+    }
     LOG_INFO("finish CloudBaseCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
              duration_cast<milliseconds>(steady_clock::now() - start).count())
             .tag("job_id", _uuid)

be/src/cloud/cloud_cumulative_compaction.cpp

@@ -172,7 +172,13 @@ Status CloudCumulativeCompaction::execute_compact() {
 
     using namespace std::chrono;
     auto start = steady_clock::now();
-    RETURN_IF_ERROR(CloudCompactionMixin::execute_compact());
+    auto res = CloudCompactionMixin::execute_compact();
+    if (!res.ok()) {
+        LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << res
+                     << ", tablet=" << _tablet->tablet_id()
+                     << ", output_version=" << _output_version;
+        return res;
+    }
     LOG_INFO("finish CloudCumulativeCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
              duration_cast<milliseconds>(steady_clock::now() - start).count())
             .tag("job_id", _uuid)

be/src/cloud/cloud_full_compaction.cpp

@@ -149,7 +149,13 @@ Status CloudFullCompaction::execute_compact() {
 
     using namespace std::chrono;
     auto start = steady_clock::now();
-    RETURN_IF_ERROR(CloudCompactionMixin::execute_compact());
+    auto res = CloudCompactionMixin::execute_compact();
+    if (!res.ok()) {
+        LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << res
+                     << ", tablet=" << _tablet->tablet_id()
+                     << ", output_version=" << _output_version;
+        return res;
+    }
     LOG_INFO("finish CloudFullCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
              duration_cast<milliseconds>(steady_clock::now() - start).count())
             .tag("job_id", _uuid)