Skip to content

Commit 63ae0f3

Browse files
authored
Add SHOW RECEIVERS support (#17869)
* Add show receivers support * Fix show receivers runtime edge cases * Fix receiver handshake body duplication * Fix show receivers schema alignment * Add show receivers visibility and HA tests * Add show receivers edge case tests * Add show receivers schema filter tests * Add show receivers permission IT * Fix pipe handshake request body reuse * Add show receivers test coverage * Add show receivers initial transfer time tests * Add show receivers lifecycle test coverage * Add sort * Add aggregation UT * Add information_schema.receivers UT * Fix pipe IT * fix-information-ci * Fix * recv-fix * ignore ts * i18n * Fix ci failure * Added ut/it * Update IoTDBShowReceiversLifecycleIT.java * Update StatementGeneratorTest.java * Added tests * Add test * Test * Delete use pipe ut * Update IoTDBShowReceiversLifecycleIT.java * Fix show receivers pipe cleanup * Fix wd * Tag write-back receiver runtime separately * Add show receivers integration coverage * Fix show receivers runtime metadata tests * Fix receiver runtime cleanup test * Remove unused session runtime hash code * Fix old pipe sender receiver runtime handshake * Fix null-safe tree user audit comparison * [IT] Fix receiver test merge selection * [Pipe] Preserve receiver sessions across region task lifecycle * Address SHOW RECEIVERS review comments
1 parent a467cf3 commit 63ae0f3

86 files changed

Lines changed: 6477 additions & 727 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.iotdb.common.rpc.thrift.TSStatus;
2525
import org.apache.iotdb.commons.client.ClientPoolFactory;
2626
import org.apache.iotdb.commons.client.IClientManager;
27-
import org.apache.iotdb.commons.client.exception.ClientManagerException;
2827
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
2928
import org.apache.iotdb.commons.cluster.NodeStatus;
3029
import org.apache.iotdb.commons.exception.PortOccupiedException;
@@ -40,7 +39,6 @@
4039
import org.apache.iotdb.isession.SessionConfig;
4140
import org.apache.iotdb.isession.pool.ISessionPool;
4241
import org.apache.iotdb.isession.pool.ITableSessionPool;
43-
import org.apache.iotdb.it.env.EnvFactory;
4442
import org.apache.iotdb.it.env.cluster.EnvUtils;
4543
import org.apache.iotdb.it.env.cluster.config.MppClusterConfig;
4644
import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
@@ -1629,7 +1627,7 @@ public void ensureNodeStatus(
16291627
Throwable lastException = null;
16301628
for (int i = 0; i < retryCount; i++) {
16311629
try (final SyncConfigNodeIServiceClient client =
1632-
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
1630+
(SyncConfigNodeIServiceClient) getLeaderConfigNodeConnection()) {
16331631
final List<String> errorMessages = new ArrayList<>(nodes.size());
16341632
final Map<String, Integer> nodeIds = new HashMap<>(nodes.size());
16351633
final TShowClusterResp showClusterResp = client.showCluster();
@@ -1685,7 +1683,7 @@ public void ensureNodeStatus(
16851683
} else {
16861684
lastException = new IllegalStateException(String.join(". ", errorMessages));
16871685
}
1688-
} catch (final TException | ClientManagerException | IOException | InterruptedException e) {
1686+
} catch (final TException | IOException | InterruptedException e) {
16891687
lastException = e;
16901688
}
16911689
try {
Lines changed: 364 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,364 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic;
21+
22+
import org.apache.iotdb.common.rpc.thrift.TSStatus;
23+
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
24+
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
25+
import org.apache.iotdb.db.it.utils.TestUtils;
26+
import org.apache.iotdb.isession.SessionConfig;
27+
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
28+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
29+
import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic;
30+
import org.apache.iotdb.itbase.env.BaseEnv;
31+
import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT;
32+
import org.apache.iotdb.rpc.TSStatusCode;
33+
34+
import org.awaitility.Awaitility;
35+
import org.junit.Assert;
36+
import org.junit.Test;
37+
import org.junit.experimental.categories.Category;
38+
import org.junit.runner.RunWith;
39+
40+
import java.sql.Connection;
41+
import java.sql.ResultSet;
42+
import java.sql.SQLException;
43+
import java.sql.Statement;
44+
import java.util.Arrays;
45+
import java.util.Collections;
46+
import java.util.HashMap;
47+
import java.util.Map;
48+
import java.util.concurrent.TimeUnit;
49+
50+
@RunWith(IoTDBTestRunner.class)
51+
@Category({MultiClusterIT2DualTreeAutoBasic.class})
52+
public class IoTDBShowReceiversLifecycleIT extends AbstractPipeDualTreeModelAutoIT {
53+
54+
@Override
55+
protected void setupConfig() {
56+
super.setupConfig();
57+
senderEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true);
58+
receiverEnv.getConfig().getCommonConfig().setPipeAirGapReceiverEnabled(true);
59+
}
60+
61+
@Test
62+
public void testShowReceiversPipeIdsDisappearAfterDropPipe() throws Exception {
63+
final String database = "root.show_receivers_lifecycle";
64+
final String pipeName = "show_receivers_lifecycle_pipe";
65+
66+
createThriftPipe(database, pipeName);
67+
68+
assertShowReceivers("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName);
69+
70+
TestUtils.executeNonQueries(
71+
senderEnv, Collections.singletonList("drop pipe " + pipeName), null);
72+
73+
assertShowReceiversDoesNotContainPipe("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName);
74+
}
75+
76+
@Test
77+
public void testShowReceiversPipeIdsDisappearAfterStopPipe() throws Exception {
78+
final String database = "root.show_receivers_lifecycle_stop";
79+
final String pipeName = "show_receivers_lifecycle_stop_pipe";
80+
81+
createThriftPipe(database, pipeName);
82+
83+
assertShowReceivers("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName);
84+
85+
TestUtils.executeNonQueries(
86+
senderEnv, Collections.singletonList("stop pipe " + pipeName), null);
87+
88+
assertShowReceiversDoesNotContainPipe("show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName);
89+
}
90+
91+
@Test
92+
public void testShowReceiversIncludesDataNodeAndConfigNodeReceivers() throws Exception {
93+
final String database = "root.show_receivers_config_node";
94+
final String pipeName = "show_receivers_config_node_pipe";
95+
96+
createThriftPipe(database, pipeName, "all");
97+
98+
TestUtils.executeNonQueries(
99+
senderEnv,
100+
Arrays.asList(
101+
"create timeseries " + database + ".d1.s2 with datatype=BOOLEAN, encoding=PLAIN",
102+
"insert into " + database + ".d1(time, s1, s2) values (2, 2, true)",
103+
"flush"),
104+
null);
105+
106+
TestUtils.assertDataEventuallyOnEnv(
107+
receiverEnv,
108+
"count timeseries " + database + ".**",
109+
"count(timeseries),",
110+
Collections.singleton("2,"));
111+
TestUtils.assertDataEventuallyOnEnv(
112+
receiverEnv,
113+
"select count(s1) from " + database + ".d1",
114+
"count(" + database + ".d1.s1),",
115+
Collections.singleton("2,"));
116+
117+
assertShowReceiversContainDataNodeAndConfigNode(
118+
"show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName);
119+
}
120+
121+
@Test
122+
public void testShowReceiversShowsAirGapProtocol() throws Exception {
123+
final String database = "root.show_receivers_air_gap";
124+
final String pipeName = "show_receivers_air_gap_pipe";
125+
126+
createAirGapPipe(database, pipeName);
127+
128+
TestUtils.assertDataEventuallyOnEnv(
129+
receiverEnv,
130+
"select count(s1) from " + database + ".d1",
131+
"count(" + database + ".d1.s1),",
132+
Collections.singleton("1,"));
133+
134+
assertShowReceiversContainProtocol(
135+
"show receivers", BaseEnv.TREE_SQL_DIALECT, pipeName, "air_gap");
136+
}
137+
138+
private void createThriftPipe(final String database, final String pipeName) throws Exception {
139+
createThriftPipe(database, pipeName, "data.insert");
140+
}
141+
142+
private void createThriftPipe(
143+
final String database, final String pipeName, final String sourceInclusion) throws Exception {
144+
createPipe(database, pipeName, sourceInclusion, "iotdb-thrift-sink", false);
145+
}
146+
147+
private void createAirGapPipe(final String database, final String pipeName) throws Exception {
148+
createPipe(database, pipeName, "data.insert", "iotdb-air-gap-sink", true);
149+
}
150+
151+
private void createPipe(
152+
final String database,
153+
final String pipeName,
154+
final String sourceInclusion,
155+
final String sinkName,
156+
final boolean useAirGapPort)
157+
throws Exception {
158+
TestUtils.executeNonQueries(
159+
senderEnv,
160+
Arrays.asList(
161+
"create database " + database,
162+
"create timeseries " + database + ".d1.s1 with datatype=INT32, encoding=PLAIN",
163+
"insert into " + database + ".d1(time, s1) values (1, 1)"),
164+
null);
165+
awaitUntilFlush(senderEnv);
166+
167+
final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
168+
169+
final Map<String, String> sourceAttributes = new HashMap<>();
170+
final Map<String, String> processorAttributes = new HashMap<>();
171+
final Map<String, String> sinkAttributes = new HashMap<>();
172+
173+
sourceAttributes.put("source.path", database + ".**");
174+
sourceAttributes.put("source.inclusion", sourceInclusion);
175+
sourceAttributes.put("user", SessionConfig.DEFAULT_USER);
176+
177+
sinkAttributes.put("sink", sinkName);
178+
sinkAttributes.put("sink.batch.enable", "false");
179+
sinkAttributes.put("sink.ip", receiverDataNode.getIp());
180+
sinkAttributes.put(
181+
"sink.port",
182+
Integer.toString(
183+
useAirGapPort
184+
? receiverDataNode.getPipeAirGapReceiverPort()
185+
: receiverDataNode.getPort()));
186+
187+
try (final SyncConfigNodeIServiceClient client =
188+
(SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
189+
TSStatus status =
190+
client.createPipe(
191+
new TCreatePipeReq(pipeName, sinkAttributes)
192+
.setExtractorAttributes(sourceAttributes)
193+
.setProcessorAttributes(processorAttributes));
194+
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
195+
196+
status = client.startPipe(pipeName);
197+
Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
198+
}
199+
}
200+
201+
private void assertShowReceivers(
202+
final String sql, final String sqlDialect, final String pipeName) {
203+
Awaitility.await()
204+
.pollInSameThread()
205+
.pollDelay(1L, TimeUnit.SECONDS)
206+
.pollInterval(1L, TimeUnit.SECONDS)
207+
.atMost(60L, TimeUnit.SECONDS)
208+
.untilAsserted(() -> Assert.assertTrue(hasExpectedReceiver(sql, sqlDialect, pipeName)));
209+
}
210+
211+
private boolean hasExpectedReceiver(
212+
final String sql, final String sqlDialect, final String pipeName) throws SQLException {
213+
try (final Connection connection =
214+
receiverEnv.getConnection(
215+
SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect);
216+
final Statement statement = connection.createStatement();
217+
final ResultSet resultSet = statement.executeQuery(sql)) {
218+
while (resultSet.next()) {
219+
final String senderClusterId = getString(resultSet, "SenderClusterId", "sender_cluster_id");
220+
final String senderAddress = getString(resultSet, "SenderAddress", "sender_address");
221+
final String userName = getString(resultSet, "UserName", "user_name");
222+
final String senderPorts = getString(resultSet, "SenderPorts", "sender_ports");
223+
final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids");
224+
if ("DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type"))
225+
&& "thrift".equals(getString(resultSet, "Protocol", "protocol"))
226+
&& senderClusterId != null
227+
&& !senderClusterId.isEmpty()
228+
&& senderAddress != null
229+
&& !senderAddress.isEmpty()
230+
&& SessionConfig.DEFAULT_USER.equals(userName)
231+
&& senderPorts != null
232+
&& !senderPorts.isEmpty()
233+
&& getInt(resultSet, "ConnectionCount", "connection_count") >= 1
234+
&& getInt(resultSet, "PipeCount", "pipe_count") >= 1
235+
&& pipeIds != null
236+
&& pipeIds.contains(pipeName + "@")
237+
&& getString(resultSet, "LastHandshakeTime", "last_handshake_time") != null
238+
&& getString(resultSet, "LastTransferTime", "last_transfer_time") != null) {
239+
return true;
240+
}
241+
}
242+
return false;
243+
}
244+
}
245+
246+
private void assertShowReceiversDoesNotContainPipe(
247+
final String sql, final String sqlDialect, final String pipeName) {
248+
Awaitility.await()
249+
.pollInSameThread()
250+
.pollDelay(1L, TimeUnit.SECONDS)
251+
.pollInterval(1L, TimeUnit.SECONDS)
252+
.atMost(60L, TimeUnit.SECONDS)
253+
.untilAsserted(() -> Assert.assertFalse(containsPipe(sql, sqlDialect, pipeName)));
254+
}
255+
256+
private boolean containsPipe(final String sql, final String sqlDialect, final String pipeName)
257+
throws SQLException {
258+
try (final Connection connection =
259+
receiverEnv.getConnection(
260+
SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect);
261+
final Statement statement = connection.createStatement();
262+
final ResultSet resultSet = statement.executeQuery(sql)) {
263+
while (resultSet.next()) {
264+
final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids");
265+
if (pipeIds != null && pipeIds.contains(pipeName + "@")) {
266+
return true;
267+
}
268+
}
269+
return false;
270+
}
271+
}
272+
273+
private void assertShowReceiversContainDataNodeAndConfigNode(
274+
final String sql, final String sqlDialect, final String pipeName) {
275+
Awaitility.await()
276+
.pollInSameThread()
277+
.pollDelay(1L, TimeUnit.SECONDS)
278+
.pollInterval(1L, TimeUnit.SECONDS)
279+
.atMost(60L, TimeUnit.SECONDS)
280+
.untilAsserted(
281+
() -> Assert.assertTrue(hasDataNodeAndConfigNodeReceivers(sql, sqlDialect, pipeName)));
282+
}
283+
284+
private boolean hasDataNodeAndConfigNodeReceivers(
285+
final String sql, final String sqlDialect, final String pipeName) throws SQLException {
286+
boolean hasDataNode = false;
287+
boolean hasConfigNode = false;
288+
try (final Connection connection =
289+
receiverEnv.getConnection(
290+
SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect);
291+
final Statement statement = connection.createStatement();
292+
final ResultSet resultSet = statement.executeQuery(sql)) {
293+
while (resultSet.next()) {
294+
final String receiverNodeType =
295+
getString(resultSet, "ReceiverNodeType", "receiver_node_type");
296+
final String protocol = getString(resultSet, "Protocol", "protocol");
297+
final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids");
298+
if (!"thrift".equals(protocol) || pipeIds == null || !pipeIds.contains(pipeName + "@")) {
299+
continue;
300+
}
301+
if ("DataNode".equals(receiverNodeType)) {
302+
hasDataNode = true;
303+
} else if ("ConfigNode".equals(receiverNodeType)) {
304+
hasConfigNode = true;
305+
}
306+
}
307+
}
308+
return hasDataNode && hasConfigNode;
309+
}
310+
311+
private void assertShowReceiversContainProtocol(
312+
final String sql, final String sqlDialect, final String pipeName, final String protocol) {
313+
Awaitility.await()
314+
.pollInSameThread()
315+
.pollDelay(1L, TimeUnit.SECONDS)
316+
.pollInterval(1L, TimeUnit.SECONDS)
317+
.atMost(60L, TimeUnit.SECONDS)
318+
.untilAsserted(
319+
() ->
320+
Assert.assertTrue(
321+
hasDataNodeReceiverWithProtocol(sql, sqlDialect, pipeName, protocol)));
322+
}
323+
324+
private boolean hasDataNodeReceiverWithProtocol(
325+
final String sql, final String sqlDialect, final String pipeName, final String protocol)
326+
throws SQLException {
327+
try (final Connection connection =
328+
receiverEnv.getConnection(
329+
SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD, sqlDialect);
330+
final Statement statement = connection.createStatement();
331+
final ResultSet resultSet = statement.executeQuery(sql)) {
332+
while (resultSet.next()) {
333+
final String pipeIds = getString(resultSet, "PipeIDs", "pipe_ids");
334+
if ("DataNode".equals(getString(resultSet, "ReceiverNodeType", "receiver_node_type"))
335+
&& protocol.equals(getString(resultSet, "Protocol", "protocol"))
336+
&& pipeIds != null
337+
&& pipeIds.contains(pipeName + "@")) {
338+
return true;
339+
}
340+
}
341+
}
342+
return false;
343+
}
344+
345+
private static String getString(
346+
final ResultSet resultSet, final String treeColumnName, final String tableColumnName)
347+
throws SQLException {
348+
try {
349+
return resultSet.getString(treeColumnName);
350+
} catch (final SQLException ignored) {
351+
return resultSet.getString(tableColumnName);
352+
}
353+
}
354+
355+
private static int getInt(
356+
final ResultSet resultSet, final String treeColumnName, final String tableColumnName)
357+
throws SQLException {
358+
try {
359+
return resultSet.getInt(treeColumnName);
360+
} catch (final SQLException ignored) {
361+
return resultSet.getInt(tableColumnName);
362+
}
363+
}
364+
}

0 commit comments

Comments
 (0)