diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/CounterNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/CounterNode.java deleted file mode 100644 index bb4d4a5f1c956f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/CounterNode.java +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.profile; - -import org.apache.doris.common.Pair; -import org.apache.doris.common.TreeNode; - -public class CounterNode extends TreeNode { - private Pair counter; - - public void setCounter(String key, String value) { - counter = Pair.of(key, value); - } - - public Pair getCounter() { - return counter; - } - - public String toTree(int indent) { - StringBuilder sb = new StringBuilder(); - sb.append(debugString(indent)); - for (CounterNode node : getChildren()) { - sb.append("\n").append(node.debugString(indent + 4)); - } - return sb.toString(); - } - - public String debugString(int indent) { - if (counter == null) { - return printIndent(indent) + " - Counters:"; - } - return printIndent(indent) + " - " + counter.first + ": " + counter.second; - } - - private String printIndent(int indent) { - String res = ""; - for (int i = 0; i < indent; i++) { - res += " "; - } - return res; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecNodeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecNodeNode.java deleted file mode 100644 index 4cc12b1be7b3d9..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecNodeNode.java +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.profile; - -public class ExecNodeNode extends ProfileTreeNode { - - public ExecNodeNode(String name, String id) { - super(name, id); - } - - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/MultiProfileTreeBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/MultiProfileTreeBuilder.java deleted file mode 100644 index 4e150ae6a04eeb..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/MultiProfileTreeBuilder.java +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.profile; - -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Pair; -import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Counter; -import org.apache.doris.common.util.RuntimeProfile; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.commons.lang3.tuple.Triple; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - - -/** - * MultiProfileTreeBuilder saves a set of ProfileTreeBuilder. - * For a query profile, there is usually only one ExecutionProfile node. - * For a load job profile, it may produce multiple subtasks, so there may be multiple ExecutionProfile nodes. - *

- * Each ExecutionProfile node corresponds to a ProfileTreeBuilder - */ -public class MultiProfileTreeBuilder { - public static final String PROFILE_NAME_EXECUTION = "Execution Profile"; - private static final Set PROFILE_ROOT_NAMES; - private static final String EXECUTION_ID_PATTERN_STR = "^Execution Profile (.*)"; - private static final Pattern EXECUTION_ID_PATTERN; - - private RuntimeProfile profileRoot; - private Map idToSingleProfile = Maps.newHashMap(); - private Map idToSingleTreeBuilder = Maps.newHashMap(); - - static { - PROFILE_ROOT_NAMES = Sets.newHashSet(); - PROFILE_ROOT_NAMES.add("Query"); - PROFILE_ROOT_NAMES.add("BrokerLoadJob"); - EXECUTION_ID_PATTERN = Pattern.compile(EXECUTION_ID_PATTERN_STR); - } - - public MultiProfileTreeBuilder(RuntimeProfile root) { - this.profileRoot = root; - } - - public void build() throws UserException { - unwrapProfile(); - buildTrees(); - } - - private void unwrapProfile() throws UserException { - if (PROFILE_ROOT_NAMES.stream().anyMatch(n -> profileRoot.getName().startsWith(n))) { - List> children = profileRoot.getChildList(); - boolean find = false; - for (Pair pair : children) { - if (pair.first.getName().startsWith(PROFILE_NAME_EXECUTION)) { - String executionProfileId = getExecutionProfileId(pair.first.getName()); - idToSingleProfile.put(executionProfileId, pair.first); - find = true; - } - } - if (!find) { - throw new UserException("Invalid profile. Expected " + PROFILE_NAME_EXECUTION); - } - } - } - - private String getExecutionProfileId(String executionName) throws UserException { - Matcher m = EXECUTION_ID_PATTERN.matcher(executionName); - if (!m.find() || m.groupCount() != 1) { - throw new UserException("Invalid execution profile name: " + executionName); - } - return m.group(1); - } - - private void buildTrees() throws UserException { - for (Map.Entry entry : idToSingleProfile.entrySet()) { - ProfileTreeBuilder builder = new ProfileTreeBuilder(entry.getValue()); - builder.build(); - idToSingleTreeBuilder.put(entry.getKey(), builder); - } - } - - public List> getSubTaskInfo() { - List> rows = Lists.newArrayList(); - for (Map.Entry entry : idToSingleProfile.entrySet()) { - List row = Lists.newArrayList(); - Counter activeCounter = entry.getValue().getCounterTotalTime(); - row.add(entry.getKey()); - row.add(RuntimeProfile.printCounter(activeCounter.getValue(), activeCounter.getType())); - rows.add(row); - } - return rows; - } - - public List> getInstanceList(String executionId, String fragmentId) - throws AnalysisException { - ProfileTreeBuilder singleBuilder = getExecutionProfileTreeBuilder(executionId); - return singleBuilder.getInstanceList(fragmentId); - } - - public ProfileTreeNode getInstanceTreeRoot(String executionId, String fragmentId, String instanceId) - throws AnalysisException { - ProfileTreeBuilder singleBuilder = getExecutionProfileTreeBuilder(executionId); - return singleBuilder.getInstanceTreeRoot(fragmentId, instanceId); - } - - public ProfileTreeNode getFragmentTreeRoot(String executionId) throws AnalysisException { - ProfileTreeBuilder singleBuilder = getExecutionProfileTreeBuilder(executionId); - return singleBuilder.getFragmentTreeRoot(); - } - - public List getFragmentInstances(String executionId) - throws AnalysisException { - return getExecutionProfileTreeBuilder(executionId).getFragmentsInstances(); - } - - private ProfileTreeBuilder getExecutionProfileTreeBuilder(String executionId) throws AnalysisException { - ProfileTreeBuilder singleBuilder = idToSingleTreeBuilder.get(executionId); - if (singleBuilder == null) { - throw new AnalysisException("Can not find execution profile: " + executionId); - } - return singleBuilder; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index bd12bbb7828229..5381cc26bd225a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -20,7 +20,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.common.util.ProfileManager; import org.apache.doris.common.util.RuntimeProfile; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.trees.plans.AbstractPlan; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java similarity index 92% rename from fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java rename to fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java index dfd54e473f8a4d..3177b5738edf37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileManager.java @@ -15,21 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.common.util; +package org.apache.doris.common.profile; import org.apache.doris.catalog.Env; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.AuthenticationException; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; -import org.apache.doris.common.profile.ExecutionProfile; -import org.apache.doris.common.profile.MultiProfileTreeBuilder; -import org.apache.doris.common.profile.Profile; -import org.apache.doris.common.profile.ProfileTreeBuilder; -import org.apache.doris.common.profile.ProfileTreeNode; -import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.load.loadv2.LoadJob; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.qe.CoordInterface; @@ -90,7 +85,6 @@ public ProfileElement(Profile profile) { private final Profile profile; public Map infoStrings = Maps.newHashMap(); - public MultiProfileTreeBuilder builder = null; public String errMsg = ""; public StatsErrorEstimator statsErrorEstimator; @@ -443,67 +437,6 @@ public void checkAuthByUserAndQueryId(String user, String queryId) throws Authen } } - public ProfileTreeNode getFragmentProfileTree(String queryID, String executionId) throws AnalysisException { - MultiProfileTreeBuilder builder; - readLock.lock(); - try { - ProfileElement element = queryIdToProfileMap.get(queryID); - if (element == null || element.builder == null) { - throw new AnalysisException("failed to get fragment profile tree. err: " - + (element == null ? "not found" : element.errMsg)); - } - builder = element.builder; - } finally { - readLock.unlock(); - } - return builder.getFragmentTreeRoot(executionId); - } - - public ProfileTreeNode getInstanceProfileTree(String queryID, String executionId, - String fragmentId, String instanceId) - throws AnalysisException { - MultiProfileTreeBuilder builder; - readLock.lock(); - try { - ProfileElement element = queryIdToProfileMap.get(queryID); - if (element == null || element.builder == null) { - throw new AnalysisException("failed to get instance profile tree. err: " - + (element == null ? "not found" : element.errMsg)); - } - builder = element.builder; - } finally { - readLock.unlock(); - } - - return builder.getInstanceTreeRoot(executionId, fragmentId, instanceId); - } - - // Return the tasks info of the specified load job - // Columns: TaskId, ActiveTime - public List> getLoadJobTaskList(String jobId) throws AnalysisException { - MultiProfileTreeBuilder builder = getMultiProfileTreeBuilder(jobId); - return builder.getSubTaskInfo(); - } - - public List getFragmentsAndInstances(String queryId) - throws AnalysisException { - return getMultiProfileTreeBuilder(queryId).getFragmentInstances(queryId); - } - - private MultiProfileTreeBuilder getMultiProfileTreeBuilder(String jobId) throws AnalysisException { - readLock.lock(); - try { - ProfileElement element = queryIdToProfileMap.get(jobId); - if (element == null || element.builder == null) { - throw new AnalysisException("failed to get task ids. err: " - + (element == null ? "not found" : element.errMsg)); - } - return element.builder; - } finally { - readLock.unlock(); - } - } - public String getQueryIdByTraceId(String traceId) { readLock.lock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java deleted file mode 100644 index a5beb880f81eac..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java +++ /dev/null @@ -1,465 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.profile; - -import org.apache.doris.common.Pair; -import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Counter; -import org.apache.doris.common.util.RuntimeProfile; -import org.apache.doris.thrift.TUnit; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import lombok.Getter; -import lombok.Setter; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.Formatter; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * This class is used to build a tree from the query runtime profile - * It will build tree for the entire query, and also tree for each instance, - * So that user can view the profile tree by query id or by instance id. - * - * Each runtime profile of a query should be built once and be read every where. - */ -public class ProfileTreeBuilder { - private static final Logger LOG = LogManager.getLogger(ProfileTreeBuilder.class); - - private static final String PROFILE_NAME_DATA_STREAM_SENDER = "DataStreamSender"; - private static final String PROFILE_NAME_VDATA_STREAM_SENDER = "VDataStreamSender"; - private static final String PROFILE_NAME_DATA_BUFFER_SENDER = "DataBufferSender"; - private static final String PROFILE_NAME_VDATA_BUFFER_SENDER = "VDataBufferSender"; - private static final String PROFILE_NAME_VOLAP_TABLE_SINK = "VOlapTableSink"; - private static final String PROFILE_NAME_BLOCK_MGR = "BlockMgr"; - private static final String PROFILE_NAME_BUFFER_POOL = "Buffer pool"; - private static final String PROFILE_NAME_EXCHANGE_NODE = "EXCHANGE_NODE"; - public static final String FINAL_SENDER_ID = "-1"; - private static final String PROFILE_NAME_VEXCHANGE_NODE = "VEXCHANGE_NODE"; - public static final String DATA_BUFFER_SENDER_ID = "-1"; - public static final String UNKNOWN_ID = "-2"; - - private RuntimeProfile profileRoot; - - // auxiliary structure to link different fragments - private List exchangeNodes = Lists.newArrayList(); - private List senderNodes = Lists.newArrayList(); - - // fragment id -> instance id -> instance tree root - private Map> instanceTreeMap = Maps.newHashMap(); - // fragment id -> (instance id, instance host, instance active time) - private Map>> instanceActiveTimeMap = Maps.newHashMap(); - - // the tree root of the entire query profile tree - private ProfileTreeNode fragmentTreeRoot; - - private List fragmentsInstances = Lists.newArrayList(); - - // Match string like: - // EXCHANGE_NODE (id=3):(Active: 103.899ms, % non-child: 2.27%) - // Extract "EXCHANGE_NODE" and "3" - private static final String EXEC_NODE_NAME_ID_PATTERN_STR = "^(.*) .*id=([0-9]+).*"; - private static final Pattern EXEC_NODE_NAME_ID_PATTERN; - - // Match string like: - // Fragment 0: - // Extract "0" - private static final String FRAGMENT_ID_PATTERN_STR = "^Fragment ([0-9]+).*"; - private static final Pattern FRAGMENT_ID_PATTERN; - - // Match string like: - // Instance e0f7390f5363419e-b416a2a7999608b6 - // (host=TNetworkAddress(hostname:192.168.1.1, port:9060)):(Active: 1s858ms, % - // non-child: 0.02%) - // Extract "e0f7390f5363419e-b416a2a7999608b6", "192.168.1.1", "9060" - private static final String INSTANCE_PATTERN_STR = "^Instance (.*) \\(.*hostname:(.*), port:([0-9]+).*"; - private static final Pattern INSTANCE_PATTERN; - - static { - EXEC_NODE_NAME_ID_PATTERN = Pattern.compile(EXEC_NODE_NAME_ID_PATTERN_STR); - FRAGMENT_ID_PATTERN = Pattern.compile(FRAGMENT_ID_PATTERN_STR); - INSTANCE_PATTERN = Pattern.compile(INSTANCE_PATTERN_STR); - } - - public ProfileTreeBuilder(RuntimeProfile root) { - this.profileRoot = root; - } - - public ProfileTreeNode getFragmentTreeRoot() { - return fragmentTreeRoot; - } - - public ProfileTreeNode getInstanceTreeRoot(String fragmentId, String instanceId) { - if (!instanceTreeMap.containsKey(fragmentId)) { - return null; - } - return instanceTreeMap.get(fragmentId).get(instanceId); - } - - public List> getInstanceList(String fragmentId) { - return instanceActiveTimeMap.get(fragmentId); - } - - public List getFragmentsInstances() { - return fragmentsInstances; - } - - public void build() throws UserException { - reset(); - checkProfile(); - analyzeAndBuild(); - assembleFragmentTrees(); - } - - private void reset() { - exchangeNodes.clear(); - senderNodes.clear(); - instanceTreeMap.clear(); - instanceActiveTimeMap.clear(); - fragmentTreeRoot = null; - fragmentsInstances.clear(); - } - - private void checkProfile() throws UserException { - if (!profileRoot.getName().startsWith(MultiProfileTreeBuilder.PROFILE_NAME_EXECUTION)) { - throw new UserException("Invalid profile. Expected " + MultiProfileTreeBuilder.PROFILE_NAME_EXECUTION); - } - } - - private void analyzeAndBuild() throws UserException { - List> childrenFragment = profileRoot.getChildList(); - for (Pair pair : childrenFragment) { - String name = pair.first.getName(); - if (name.equals("Fragments")) { - analyzeAndBuildFragmentTrees(pair.first); - } else if (name.equals("LoadChannels")) { - analyzeAndBuildLoadChannels(pair.first); - } else { - throw new UserException("Invalid execution profile name: " + name); - } - } - } - - private void analyzeAndBuildLoadChannels(RuntimeProfile loadChannelsProfile) throws UserException { - List> childrenFragment = loadChannelsProfile.getChildList(); - for (Pair pair : childrenFragment) { - analyzeAndBuildLoadChannel(pair.first); - } - } - - private void analyzeAndBuildLoadChannel(RuntimeProfile loadChannelsProfil) throws UserException { - // TODO, `show load profile` add load channel profile, or add `show load channel - // profile`. - } - - private void analyzeAndBuildFragmentTrees(RuntimeProfile fragmentsProfile) throws UserException { - List> childrenFragment = fragmentsProfile.getChildList(); - for (Pair pair : childrenFragment) { - analyzeAndBuildFragmentTree(pair.first); - } - } - - private void analyzeAndBuildFragmentTree(RuntimeProfile fragmentProfile) throws UserException { - String fragmentId = getFragmentId(fragmentProfile); - List> fragmentChildren = fragmentProfile.getChildList(); - if (fragmentChildren.isEmpty()) { - throw new UserException("Empty instance in fragment: " + fragmentProfile.getName()); - } - - // 1. Get max active time of instances in this fragment - List> instanceIdAndActiveTimeList = Lists.newArrayList(); - List instances = Lists.newArrayList(); - Map instanceIdToInstance = Maps.newHashMap(); - long maxActiveTimeNs = 0; - for (Pair pair : fragmentChildren) { - Triple instanceIdAndActiveTime = getInstanceIdHostAndActiveTime(pair.first); - instanceIdToInstance.put(instanceIdAndActiveTime.getLeft(), - new Instance(instanceIdAndActiveTime.getMiddle(), - RuntimeProfile.printCounter(instanceIdAndActiveTime.getRight(), TUnit.TIME_NS))); - maxActiveTimeNs = Math.max(instanceIdAndActiveTime.getRight(), maxActiveTimeNs); - instanceIdAndActiveTimeList.add(instanceIdAndActiveTime); - instances.add(instanceIdAndActiveTime.getLeft()); - } - instanceActiveTimeMap.put(fragmentId, instanceIdAndActiveTimeList); - fragmentsInstances.add(new FragmentInstances(fragmentId, - RuntimeProfile.printCounter(maxActiveTimeNs, TUnit.TIME_NS), instanceIdToInstance)); - - // 2. Build tree for all fragments - // All instance in a fragment are same, so use first instance to build the - // fragment tree - RuntimeProfile instanceProfile = fragmentChildren.get(0).first; - ProfileTreeNode instanceTreeRoot = buildSingleInstanceTree(instanceProfile, fragmentId, null); - instanceTreeRoot.setMaxInstanceActiveTime(RuntimeProfile.printCounter(maxActiveTimeNs, TUnit.TIME_NS)); - if (instanceTreeRoot.id.equals(FINAL_SENDER_ID)) { - fragmentTreeRoot = instanceTreeRoot; - } - - // 3. Build tree for each single instance - int i = 0; - Map instanceTrees = Maps.newHashMap(); - for (Pair pair : fragmentChildren) { - String instanceId = instanceIdAndActiveTimeList.get(i).getLeft(); - ProfileTreeNode singleInstanceTreeNode = buildSingleInstanceTree(pair.first, fragmentId, instanceId); - instanceTrees.put(instanceId, singleInstanceTreeNode); - i++; - } - this.instanceTreeMap.put(fragmentId, instanceTrees); - } - - // If instanceId is null, which means this profile tree node is for building the - // entire fragment tree. - // So that we need to add sender and exchange node to the auxiliary structure. - private ProfileTreeNode buildSingleInstanceTree(RuntimeProfile instanceProfile, String fragmentId, - String instanceId) throws UserException { - List> instanceChildren = instanceProfile.getChildList(); - ProfileTreeNode senderNode = null; - List childrenNodes = Lists.newArrayList(); - for (Pair pair : instanceChildren) { - RuntimeProfile profile = pair.first; - if (profile.getName().startsWith(PROFILE_NAME_DATA_STREAM_SENDER) - || profile.getName().startsWith(PROFILE_NAME_VDATA_STREAM_SENDER) - || profile.getName().startsWith(PROFILE_NAME_VDATA_BUFFER_SENDER) - || profile.getName().startsWith(PROFILE_NAME_DATA_BUFFER_SENDER) - || profile.getName().startsWith(PROFILE_NAME_VOLAP_TABLE_SINK)) { - senderNode = buildTreeNode(profile, null, fragmentId, instanceId); - if (instanceId == null) { - senderNodes.add(senderNode); - } - } else if (profile.getName().startsWith(PROFILE_NAME_BLOCK_MGR) - || profile.getName().startsWith(PROFILE_NAME_BUFFER_POOL)) { - // skip BlockMgr nad Buffer pool profile - continue; - } else { - // This should be an ExecNode profile - childrenNodes.add(buildTreeNode(profile, null, fragmentId, instanceId)); - } - } - if (senderNode == null || childrenNodes.isEmpty()) { - // FE will constantly update the total profile after receiving the instance - // profile reported by BE. - // Writing a profile will result in an empty instance profile until all instance - // profiles are received - // at least once. - // Issue: https://github.com/apache/doris/issues/10095 - StringBuilder sb = new StringBuilder(); - instanceProfile.prettyPrint(sb, ""); - if (LOG.isDebugEnabled()) { - LOG.debug( - "Invalid instance profile, sender is null: {}," - + "childrenNodes is empty: {}, instance profile: {}", - (senderNode == null), childrenNodes.isEmpty(), sb.toString()); - } - throw new UserException("Invalid instance profile, without sender or exec node: \n" + instanceProfile); - } - for (ProfileTreeNode execNode : childrenNodes) { - senderNode.addChild(execNode); - execNode.setParentNode(senderNode); - execNode.setFragmentAndInstanceId(fragmentId, instanceId); - } - - senderNode.setFragmentAndInstanceId(fragmentId, instanceId); - - return senderNode; - } - - private ProfileTreeNode buildTreeNode(RuntimeProfile profile, ProfileTreeNode root, - String fragmentId, String instanceId) { - String name = profile.getName(); - if (name.startsWith(PROFILE_NAME_BUFFER_POOL)) { - // skip Buffer pool, and buffer pool does not has child - return null; - } - String finalSenderName = checkAndGetFinalSenderName(name); - Matcher m = EXEC_NODE_NAME_ID_PATTERN.matcher(name); - String extractName; - String extractId; - if ((!m.find() && finalSenderName == null) || m.groupCount() != 2) { - // DataStreamBuffer name like: - // "DataBufferSender - // (dst_fragment_instance_id=d95356f9219b4831-986b4602b41683ca):" - // So it has no id. - // Other profile should has id like: - // EXCHANGE_NODE (id=3):(Active: 103.899ms, % non-child: 2.27%) - // HASH_JOIN_NODE (id=2):(Active: 972.329us, % non-child: 0.00%) - extractName = name; - extractId = UNKNOWN_ID; - } else { - extractName = finalSenderName != null ? finalSenderName : m.group(1); - extractId = finalSenderName != null ? FINAL_SENDER_ID : m.group(2); - } - Counter activeCounter = profile.getCounterTotalTime(); - ExecNodeNode node = new ExecNodeNode(extractName, extractId); - node.setActiveTime(RuntimeProfile.printCounter(activeCounter.getValue(), activeCounter.getType())); - try (Formatter fmt = new Formatter()) { - node.setNonChild(fmt.format("%.2f", profile.getLocalTimePercent()).toString()); - } - - if (!profile.getInfoStrings().isEmpty()) { - ArrayList infoStrings = new ArrayList(); - for (Map.Entry entry : profile.getInfoStrings().entrySet()) { - infoStrings.add(entry.getKey() + ": " + entry.getValue()); - } - node.setInfoStrings(infoStrings); - } - CounterNode rootCounterNode = new CounterNode(); - buildCounterNode(profile, RuntimeProfile.ROOT_COUNTER, rootCounterNode); - node.setCounterNode(rootCounterNode); - - if (root != null) { - root.addChild(node); - node.setParentNode(root); - } - - if ((node.name.equals(PROFILE_NAME_EXCHANGE_NODE) - || node.name.equals(PROFILE_NAME_VEXCHANGE_NODE)) && instanceId == null) { - exchangeNodes.add(node); - } - - // The children in profile is reversed, so traverse it from last to first - List> children = profile.getChildList(); - for (int i = children.size() - 1; i >= 0; i--) { - Pair pair = children.get(i); - ProfileTreeNode execNode = buildTreeNode(pair.first, node, fragmentId, instanceId); - if (execNode != null) { - // For buffer pool profile, buildTreeNode will return null - execNode.setFragmentAndInstanceId(fragmentId, instanceId); - } - } - return node; - } - - // Check if the given node name is from final node, like DATA_BUFFER_SENDER or - // OLAP_TABLE_SINK - // If yes, return that name, if not, return null; - private String checkAndGetFinalSenderName(String name) { - if (name.startsWith(PROFILE_NAME_DATA_BUFFER_SENDER)) { - return PROFILE_NAME_DATA_BUFFER_SENDER; - } else if (name.startsWith(PROFILE_NAME_VOLAP_TABLE_SINK)) { - return PROFILE_NAME_VOLAP_TABLE_SINK; - } else if (name.startsWith(PROFILE_NAME_VDATA_BUFFER_SENDER)) { - return PROFILE_NAME_VDATA_BUFFER_SENDER; - } else { - return null; - } - } - - private void buildCounterNode(RuntimeProfile profile, String counterName, CounterNode root) { - Map> childCounterMap = profile.getChildCounterMap(); - Set childCounterSet = childCounterMap.get(counterName); - if (childCounterSet == null) { - return; - } - - Map counterMap = profile.getCounterMap(); - for (String childCounterName : childCounterSet) { - Counter counter = counterMap.get(childCounterName); - CounterNode counterNode = new CounterNode(); - if (root != null) { - root.addChild(counterNode); - } - counterNode.setCounter(childCounterName, - RuntimeProfile.printCounter(counter.getValue(), counter.getType())); - buildCounterNode(profile, childCounterName, counterNode); - } - return; - } - - private void assembleFragmentTrees() throws UserException { - for (ProfileTreeNode senderNode : senderNodes) { - if (senderNode.id.equals(FINAL_SENDER_ID)) { - // this is result sender, skip it. - continue; - } - ProfileTreeNode exchangeNode = findExchangeNode(senderNode.id); - exchangeNode.addChild(senderNode); - senderNode.setParentNode(exchangeNode); - } - } - - private ProfileTreeNode findExchangeNode(String senderId) throws UserException { - for (ProfileTreeNode node : exchangeNodes) { - if (node.id.equals(senderId)) { - return node; - } - } - throw new UserException("Failed to find fragment for sender id: " + senderId); - } - - private String getFragmentId(RuntimeProfile fragmentProfile) throws UserException { - String name = fragmentProfile.getName(); - Matcher m = FRAGMENT_ID_PATTERN.matcher(name); - if (!m.find() || m.groupCount() != 1) { - throw new UserException("Invalid fragment profile name: " + name); - } - return m.group(1); - } - - private Triple getInstanceIdHostAndActiveTime(RuntimeProfile instanceProfile) - throws UserException { - long activeTimeNs = instanceProfile.getCounterTotalTime().getValue(); - String name = instanceProfile.getName(); - Matcher m = INSTANCE_PATTERN.matcher(name); - if (!m.find() || m.groupCount() != 3) { - throw new UserException("Invalid instance profile name: " + name); - } - return new ImmutableTriple<>(m.group(1), m.group(2) + ":" + m.group(3), activeTimeNs); - } - - @Getter - @Setter - public static class FragmentInstances { - @JsonProperty("fragment_id") - private String fragmentId; - @JsonProperty("time") - private String maxActiveTimeNs; - @JsonProperty("instance_id") - private Map instanceIdToInstance; - - public FragmentInstances(String fragmentId, String maxActiveTimeNs, - Map instanceIdToInstance) { - this.fragmentId = fragmentId; - this.maxActiveTimeNs = maxActiveTimeNs; - this.instanceIdToInstance = instanceIdToInstance; - } - } - - @Getter - @Setter - public static class Instance { - @JsonProperty("host") - private String host; - @JsonProperty("active_time") - private String activeTime; - - public Instance(String host, String activeTime) { - this.host = host; - this.activeTime = activeTime; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeNode.java deleted file mode 100644 index c6ef8b5c82b865..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeNode.java +++ /dev/null @@ -1,201 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.profile; - -import org.apache.doris.common.TreeNode; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; - -import java.util.List; - -public class ProfileTreeNode extends TreeNode { - - protected String name; - protected String id; - protected CounterNode counterNode; - protected String activeTime; - protected List infoStrings = Lists.newArrayList(); - protected String nonChild; - - protected String fragmentId = ""; - protected String instanceId = ""; - - // This is used to record the max activeTime of all instances in a fragment. - // Usually recorded on the Sender node. - protected String maxInstanceActiveTime = ""; - - protected ProfileTreeNode parentNode; - - protected ProfileTreeNode(String name, String id) { - this.name = name; - this.id = id; - } - - public void setParentNode(ProfileTreeNode parentNode) { - this.parentNode = parentNode; - } - - public ProfileTreeNode getParentNode() { - return parentNode; - } - - public void setCounterNode(CounterNode counterNode) { - this.counterNode = counterNode; - } - - public CounterNode getCounterNode() { - return counterNode; - } - - public String getId() { - return id; - } - - public String getName() { - return name; - } - - public void setActiveTime(String activeTime) { - this.activeTime = activeTime; - } - - public String getActiveTime() { - return activeTime; - } - - public void setInfoStrings(List infoStrings) { - this.infoStrings = infoStrings; - } - - public List getInfoStrings() { - return infoStrings; - } - - public void setNonChild(String nonChild) { - this.nonChild = nonChild; - } - - public String getNonChild() { - return nonChild; - } - - public String getIdentity() { - if (id.equals(ProfileTreeBuilder.UNKNOWN_ID)) { - return "[" + name + "]"; - } - return "[" + id + ": " + name + "]"; - } - - public void setFragmentAndInstanceId(String fragmentId, String instanceId) { - this.fragmentId = fragmentId; - this.instanceId = instanceId; - } - - public void setMaxInstanceActiveTime(String maxInstanceActiveTime) { - this.maxInstanceActiveTime = maxInstanceActiveTime; - } - - public String getMaxInstanceActiveTime() { - return maxInstanceActiveTime; - } - - public String debugTree(int indent, ProfileTreePrinter.PrintLevel level) { - StringBuilder sb = new StringBuilder(printIndent(indent)); - sb.append(debugString(indent, level)); - if (!getChildren().isEmpty()) { - int childSize = getChildren().size(); - for (int i = 0; i < childSize; i++) { - ProfileTreeNode node = getChild(i); - sb.append("\n").append(node.debugTree(indent + 4, level)); - } - } - return sb.toString(); - } - - public String debugString(int indent, ProfileTreePrinter.PrintLevel level) { - String indentStr = printIndent(indent); - StringBuilder sb = new StringBuilder(); - sb.append(indentStr).append(getIdentity()).append("\n"); - if (level == ProfileTreePrinter.PrintLevel.FRAGMENT) { - sb.append(indentStr).append("Fragment: ").append(fragmentId).append("\n"); - if (!Strings.isNullOrEmpty(maxInstanceActiveTime)) { - sb.append(indentStr).append("MaxActiveTime: ").append(maxInstanceActiveTime).append("\n"); - } - } - if (level == ProfileTreePrinter.PrintLevel.INSTANCE) { - sb.append("(Active: ").append(activeTime).append(", "); - sb.append("non-child: ").append(nonChild).append(")").append("\n"); - if (!infoStrings.isEmpty()) { - String infoStringIndent = printIndent(indent + 1); - sb.append(infoStringIndent).append(" - Info:").append("\n"); - infoStringIndent = printIndent(indent + 5); - for (String info : infoStrings) { - sb.append(infoStringIndent).append(" - ").append(info).append("\n"); - } - } - // print counters - sb.append(counterNode.toTree(indent + 1)); - } - return sb.toString(); - } - - public JSONObject debugStringInJson(ProfileTreePrinter.PrintLevel level, String nodeLevel) { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("id", nodeLevel); - JSONObject title = new JSONObject(); - if (!id.equals(ProfileTreeBuilder.UNKNOWN_ID)) { - title.put("id", id); - } - title.put("name", name); - jsonObject.put("title", title); - if (level == ProfileTreePrinter.PrintLevel.FRAGMENT) { - jsonObject.put("fragment", fragmentId); - JSONArray labels = new JSONArray(); - if (!Strings.isNullOrEmpty(maxInstanceActiveTime)) { - JSONObject label = new JSONObject(); - label.put("name", "MaxActiveTime"); - label.put("value", maxInstanceActiveTime); - labels.add(label); - } - jsonObject.put("labels", labels); - } - if (level == ProfileTreePrinter.PrintLevel.INSTANCE) { - jsonObject.put("active", activeTime); - jsonObject.put("non-child", nonChild); - JSONArray counters = new JSONArray(); - for (CounterNode node : counterNode.getChildren()) { - JSONObject counter = new JSONObject(); - counter.put(node.getCounter().first, node.getCounter().second); - counters.add(counter); - } - jsonObject.put("counters", counters); - } - return jsonObject; - } - - private String printIndent(int indent) { - String res = ""; - for (int i = 0; i < indent; i++) { - res += " "; - } - return res; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreePrinter.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreePrinter.java deleted file mode 100644 index 5d03b406485e42..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreePrinter.java +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.profile; - -import hu.webarticum.treeprinter.BorderTreeNodeDecorator; -import hu.webarticum.treeprinter.SimpleTreeNode; -import hu.webarticum.treeprinter.TraditionalTreePrinter; -import org.apache.commons.lang3.StringUtils; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; - -public class ProfileTreePrinter { - - public enum PrintLevel { - FRAGMENT, INSTANCE - } - - // Fragment tree only print the entire query plan tree with node name - // and some other brief info. - public static String printFragmentTree(ProfileTreeNode root) { - SimpleTreeNode rootNode = buildNode(root, PrintLevel.FRAGMENT); - StringBuilder sb = new StringBuilder(); - sb.append("\n"); - new TraditionalTreePrinter().print(new BorderTreeNodeDecorator(rootNode), sb); - return sb.toString(); - } - - // Instance tree will print the details of the tree of a single instance - public static String printInstanceTree(ProfileTreeNode root) { - SimpleTreeNode rootNode = buildNode(root, PrintLevel.INSTANCE); - StringBuilder sb = new StringBuilder(); - sb.append("\n"); - new TraditionalTreePrinter().print(new BorderTreeNodeDecorator(rootNode), sb); - return sb.toString(); - } - - private static SimpleTreeNode buildNode(ProfileTreeNode profileNode, PrintLevel level) { - SimpleTreeNode node = new SimpleTreeNode(profileNode.debugString(0, level)); - for (ProfileTreeNode child : profileNode.getChildren()) { - node.addChild(buildNode(child, level)); - } - return node; - } - - public static JSONObject printFragmentTreeInJson(ProfileTreeNode root, ProfileTreePrinter.PrintLevel level) { - JSONObject object = new JSONObject(); - JSONArray jsonNodes = new JSONArray(); - JSONArray edges = new JSONArray(); - object.put("nodes", jsonNodes); - object.put("edges", edges); - buildNodeInJson(root, level, "", "", jsonNodes, edges); - return object; - } - - private static void buildNodeInJson(ProfileTreeNode profileNode, PrintLevel level, String sourceNodeId, - String targetNodeId, JSONArray jsonNodes, JSONArray edges) { - boolean isFrist = false; - if (StringUtils.isBlank(sourceNodeId)) { - isFrist = true; - targetNodeId = "1"; - } - jsonNodes.add(profileNode.debugStringInJson(level, targetNodeId)); - int i = 0; - for (ProfileTreeNode child : profileNode.getChildren()) { - buildNodeInJson(child, level, targetNodeId, targetNodeId + i++, jsonNodes, edges); - } - if (!isFrist) { - JSONObject edge = new JSONObject(); - edge.put("id", "e" + targetNodeId); - edge.put("source", sourceNodeId); - edge.put("target", targetNodeId); - edges.add(edge); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java index d86ab6019926a5..2b6dd1fc4c46e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java @@ -17,8 +17,8 @@ package org.apache.doris.httpv2.controller; +import org.apache.doris.common.profile.ProfileManager; import org.apache.doris.common.profile.SummaryProfile; -import org.apache.doris.common.util.ProfileManager; import org.apache.doris.httpv2.entity.ResponseBody; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java index 85d97879602f33..a470608446a76b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ProfileAction.java @@ -17,7 +17,7 @@ package org.apache.doris.httpv2.rest; -import org.apache.doris.common.util.ProfileManager; +import org.apache.doris.common.profile.ProfileManager; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java index e5f19f90a1a9fe..625b307c43ed88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java @@ -25,11 +25,9 @@ import org.apache.doris.common.Status; import org.apache.doris.common.proc.CurrentQueryStatementsProcNode; import org.apache.doris.common.proc.ProcResult; -import org.apache.doris.common.profile.ProfileTreeNode; -import org.apache.doris.common.profile.ProfileTreePrinter; +import org.apache.doris.common.profile.ProfileManager; +import org.apache.doris.common.profile.ProfileManager.ProfileElement; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.common.util.ProfileManager; -import org.apache.doris.common.util.ProfileManager.ProfileElement; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.rest.RestBaseController; import org.apache.doris.mysql.privilege.Auth; @@ -51,7 +49,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; -import org.json.simple.JSONObject; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -77,9 +74,8 @@ * 2. /sql/{query_id} * 3. /profile/{format}/{query_id} * 4. /trace_id/{trace_id} - * 5. /profile/fragments/{query_id} - * 6. /current_queries - * 7. /kill/{query_id} + * 5. /current_queries + * 6. /kill/{query_id} */ @RestController @RequestMapping("/rest/v2/manager/query") @@ -255,20 +251,13 @@ private Stream> filterQueriesByUserAndQueryId(Stream> * 1. Text: return the entire profile of the specified query id * eg: {"profile": "text_xxx"} *

- * 2. Graph: return the profile in ascii graph. If fragmentId and instanceId are specified, it will - * return the instance profile, otherwise, it will return the fragment profile. - * eg: {"profile" : "graph_xxx"} - *

- * 3. Json: return the profile in json. If fragmentId and instanceId are specified, it will - * return the instance profile, otherwise, it will return the fragment profile. + * 2. Json: return the profile in json. * Json format is mainly used for front-end UI drawing. * eg: {"profile" : "json_xxx"} */ @RequestMapping(path = "/profile/{format}/{query_id}", method = RequestMethod.GET) public Object queryProfileText(HttpServletRequest request, HttpServletResponse response, @PathVariable("format") String format, @PathVariable("query_id") String queryId, - @RequestParam(value = FRAGMENT_ID, required = false) String fragmentId, - @RequestParam(value = INSTANCE_ID, required = false) String instanceId, @RequestParam(value = IS_ALL_NODE_PARA, required = false, defaultValue = "true") boolean isAllNode) { executeCheckPassword(request, response); @@ -282,10 +271,8 @@ public Object queryProfileText(HttpServletRequest request, HttpServletResponse r if (format.equals("text")) { return getTextProfile(request, queryId, isAllNode); - } else if (format.equals("graph")) { - return getGraphProfile(request, queryId, fragmentId, instanceId, isAllNode); } else if (format.equals("json")) { - return getJsonProfile(request, queryId, fragmentId, instanceId, isAllNode); + return getJsonProfile(request, queryId, isAllNode); } else { return ResponseEntityBuilder.badRequest("Invalid profile format: " + format); } @@ -359,47 +346,6 @@ public ResponseEntity getStats(@PathVariable(value = "id") String id) { return ResponseEntity.ok(GsonUtils.GSON.toJson(statsErrorEstimator)); } - @RequestMapping(path = "/profile/fragments/{query_id}", method = RequestMethod.GET) - public Object fragments(HttpServletRequest request, HttpServletResponse response, - @PathVariable("query_id") String queryId, - @RequestParam(value = IS_ALL_NODE_PARA, required = false, defaultValue = "true") boolean isAllNode) { - executeCheckPassword(request, response); - - if (isAllNode) { - String httpPath = "/rest/v2/manager/query/profile/fragments/" + queryId; - ImmutableMap arguments = - ImmutableMap.builder().put(IS_ALL_NODE_PARA, "false").build(); - List> frontends = HttpUtils.getFeList(); - ImmutableMap header = ImmutableMap.builder() - .put(NodeAction.AUTHORIZATION, request.getHeader(NodeAction.AUTHORIZATION)).build(); - for (Pair ipPort : frontends) { - String url = HttpUtils.concatUrl(ipPort, httpPath, arguments); - try { - String responseJson = HttpUtils.doGet(url, header); - int code = JsonParser.parseString(responseJson).getAsJsonObject().get("code").getAsInt(); - if (code == HttpUtils.REQUEST_SUCCESS_CODE) { - return responseJson; - } - } catch (Exception e) { - LOG.warn(e); - } - } - } else { - try { - checkAuthByUserAndQueryId(queryId); - } catch (AuthenticationException e) { - return ResponseEntityBuilder.badRequest(e.getMessage()); - } - - try { - return ResponseEntityBuilder.ok(ProfileManager.getInstance().getFragmentsAndInstances(queryId)); - } catch (AnalysisException e) { - return ResponseEntityBuilder.badRequest(e.getMessage()); - } - } - return ResponseEntityBuilder.badRequest("not found query id"); - } - @NotNull private ResponseEntity getTextProfile(HttpServletRequest request, String queryId, boolean isAllNode) { Map profileMap = Maps.newHashMap(); @@ -415,52 +361,16 @@ private ResponseEntity getTextProfile(HttpServletRequest request, String queryId } @NotNull - private ResponseEntity getGraphProfile(HttpServletRequest request, String queryId, String fragmentId, - String instanceId, boolean isAllNode) { + private ResponseEntity getJsonProfile(HttpServletRequest request, String queryId, boolean isAllNode) { Map graph = Maps.newHashMap(); - List results; if (isAllNode) { - return getProfileFromAllFrontends(request, "graph", queryId, fragmentId, instanceId); + return getProfileFromAllFrontends(request, "json", queryId, null, null); } else { try { - if (Strings.isNullOrEmpty(fragmentId) || Strings.isNullOrEmpty(instanceId)) { - ProfileTreeNode treeRoot = ProfileManager.getInstance().getFragmentProfileTree(queryId, queryId); - results = Lists.newArrayList(ProfileTreePrinter.printFragmentTree(treeRoot)); - } else { - ProfileTreeNode treeRoot = ProfileManager.getInstance().getInstanceProfileTree(queryId, queryId, - fragmentId, instanceId); - results = Lists.newArrayList(ProfileTreePrinter.printInstanceTree(treeRoot)); - } - graph.put("graph", results.get(0)); - } catch (Exception e) { - LOG.warn("get profile graph error, queryId:{}, fragementId:{}, instanceId:{}", queryId, fragmentId, - instanceId, e); - } - } - return ResponseEntityBuilder.ok(graph); - } - - @NotNull - private ResponseEntity getJsonProfile(HttpServletRequest request, String queryId, String fragmentId, - String instanceId, boolean isAllNode) { - Map graph = Maps.newHashMap(); - if (isAllNode) { - return getProfileFromAllFrontends(request, "json", queryId, fragmentId, instanceId); - } else { - try { - JSONObject json; - if (Strings.isNullOrEmpty(fragmentId) || Strings.isNullOrEmpty(instanceId)) { - String brief = ProfileManager.getInstance().getProfileBrief(queryId); - graph.put("profile", brief); - } else { - ProfileTreeNode treeRoot = ProfileManager.getInstance() - .getInstanceProfileTree(queryId, queryId, fragmentId, instanceId); - json = ProfileTreePrinter.printFragmentTreeInJson(treeRoot, ProfileTreePrinter.PrintLevel.INSTANCE); - graph.put("profile", json.toJSONString()); - } + String brief = ProfileManager.getInstance().getProfileBrief(queryId); + graph.put("profile", brief); } catch (Exception e) { - LOG.warn("get profile graph error, queryId:{}, fragementId:{}, instanceId:{}", queryId, fragmentId, - instanceId, e); + LOG.warn("get profile graph error, queryId:{}", queryId, e); } } return ResponseEntityBuilder.ok(graph); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 5e1b085b239474..872d0c5396a76d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -36,12 +36,12 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.profile.Profile; +import org.apache.doris.common.profile.ProfileManager.ProfileType; import org.apache.doris.common.profile.SummaryProfile.SummaryBuilder; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.MetaLockUtils; -import org.apache.doris.common.util.ProfileManager.ProfileType; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.property.constants.S3Properties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsErrorEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsErrorEstimator.java index 34206b1ea788cb..60439a43928306 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsErrorEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsErrorEstimator.java @@ -18,8 +18,8 @@ package org.apache.doris.nereids.stats; import org.apache.doris.common.Pair; +import org.apache.doris.common.profile.ProfileManager; import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.common.util.ProfileManager; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.planner.PlanNode; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 68718de0f86a5b..b4303bea4a6302 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -24,7 +24,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.util.ProfileManager.ProfileType; +import org.apache.doris.common.profile.ProfileManager.ProfileType; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 8bdbc57f9993b9..296ba4ac15c7ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -136,7 +136,7 @@ import org.apache.doris.cloud.load.CopyJob; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.ProfileManager; +import org.apache.doris.common.profile.ProfileManager; import org.apache.doris.job.exception.JobException; import org.apache.doris.load.EtlStatus; import org.apache.doris.load.FailMsg; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java index 667d15de1671bf..d9b4583d71f20e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java @@ -23,8 +23,8 @@ import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; import org.apache.doris.common.profile.ExecutionProfile; +import org.apache.doris.common.profile.ProfileManager; import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.common.util.ProfileManager; import org.apache.doris.metric.MetricRepo; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TNetworkAddress; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 234d5f0e610780..08ad231168b9f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -113,6 +113,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.profile.Profile; +import org.apache.doris.common.profile.ProfileManager.ProfileType; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.profile.SummaryProfile.SummaryBuilder; import org.apache.doris.common.util.DebugPointUtil; @@ -120,7 +121,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.common.util.ProfileManager.ProfileType; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util;