Skip to content

Commit da07a8e

Browse files
VGalaxiescodexCodexCaideyipijt2594838
authored
Subscription: support table-model column-filter for topics (#17936)
* WIP: subscription table-model column-filter (impl + UT/IT) Independent ColumnFilter.g4 grammar; parser/validator/evaluator/binder/matcher/TabletColumnPruner; node-commons ColumnMetadata; column pruning in consensus WAL + pipe tablet + TsFile reseal paths; legacy 'column' regex removed; timeSelected flag wired end-to-end (per-table); CREATE/ALTER TOPIC validation + DataNode refreshColumnFilter cache hot-refresh; UT + IT coverage. Implements V2.0 subscription column-filter design. NOT YET VERIFIED: end-to-end datanode/confignode compile and full IT run. * Use generated grammar for column filter parsing * Add subscription column filter IT follow-ups * Subscription: stabilize consensus column-filter IT draining * Subscription: add tree-view column-filter IT * Fix subscription column-filter review issues * fix * Address subscription column filter review comments * Address subscription column filter i18n comments * Fix consensus subscription WAL tail lag detection * Update ConsensusSubscriptionSetupHandler.java * Fix CI * Fix subscription column filter edge cases * spotless * Relax consensus column filter alter tests * Resolve merge conflict in DataNodeTableCache.java * Fix double living visibility lookup * Enforce metadata lease fencing for table snapshots --------- Co-authored-by: Codex <codex@openai.com> Co-authored-by: Codex <codex@localhost> Co-authored-by: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Co-authored-by: Jiang Tian <jt2594838@163.com>
1 parent 86741a8 commit da07a8e

66 files changed

Lines changed: 7189 additions & 379 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎integration-test/src/test/java/org/apache/iotdb/subscription/it/consensus/local/tablemodel/ConsensusSubscriptionTableITSupport.java‎

Lines changed: 293 additions & 43 deletions
Large diffs are not rendered by default.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.subscription.it.consensus.local.tablemodel;
21+
22+
import org.apache.iotdb.consensus.ConsensusFactory;
23+
import org.apache.iotdb.isession.ITableSession;
24+
import org.apache.iotdb.it.env.EnvFactory;
25+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
26+
import org.apache.iotdb.itbase.category.TableClusterIT;
27+
import org.apache.iotdb.rpc.subscription.config.TopicConstant;
28+
import org.apache.iotdb.session.subscription.ISubscriptionTableSession;
29+
import org.apache.iotdb.session.subscription.SubscriptionTableSessionBuilder;
30+
import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumer;
31+
import org.apache.iotdb.session.subscription.consumer.table.SubscriptionTablePullConsumerBuilder;
32+
import org.apache.iotdb.subscription.it.AbstractSubscriptionIT;
33+
34+
import org.junit.After;
35+
import org.junit.Assert;
36+
import org.junit.Before;
37+
import org.junit.Test;
38+
import org.junit.experimental.categories.Category;
39+
import org.junit.runner.RunWith;
40+
41+
import java.util.Collections;
42+
import java.util.LinkedHashSet;
43+
import java.util.Locale;
44+
import java.util.Properties;
45+
import java.util.Set;
46+
47+
@RunWith(IoTDBTestRunner.class)
48+
@Category({TableClusterIT.class})
49+
public class IoTDBConsensusSubscriptionColumnFilterClusterIT extends AbstractSubscriptionIT {
50+
51+
private static final long OWNER_LEASE_DURATION_MS = 60_000L;
52+
53+
@Override
54+
@Before
55+
public void setUp() throws Exception {
56+
super.setUp();
57+
EnvFactory.getEnv()
58+
.getConfig()
59+
.getCommonConfig()
60+
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
61+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
62+
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
63+
.setSchemaReplicationFactor(1)
64+
.setDataReplicationFactor(2)
65+
.setAutoCreateSchemaEnabled(true)
66+
.setSubscriptionEnabled(true)
67+
.setPipeMemoryManagementEnabled(false)
68+
.setIsPipeEnableMemoryCheck(false)
69+
.setSubscriptionOwnerLeaseDurationMsMin(1000);
70+
EnvFactory.getEnv().initClusterEnvironment(1, 3);
71+
}
72+
73+
@Override
74+
@After
75+
public void tearDown() throws Exception {
76+
EnvFactory.getEnv().cleanClusterEnvironment();
77+
super.tearDown();
78+
}
79+
80+
@Test
81+
public void testAlterColumnFilterRebindsAfterOwnerTransferOnThreeDataNodes() throws Exception {
82+
final ConsensusSubscriptionTableITSupport.TestIdentifiers ids =
83+
ConsensusSubscriptionTableITSupport.newIdentifiers("cluster_owner_rebind");
84+
final String database = ids.getDatabase();
85+
final String table = "t1";
86+
final String schema = "tag1 STRING TAG, s1 INT64 FIELD, s2 DOUBLE FIELD, s3 BOOLEAN FIELD";
87+
SubscriptionTablePullConsumer ownerConsumer = null;
88+
89+
try {
90+
ConsensusSubscriptionTableITSupport.createDatabaseAndTable(database, table, schema);
91+
ConsensusSubscriptionTableITSupport.insertRows(database, table, 0L, 1, true, true, true);
92+
createOwnedConsensusTopic(ids.getTopic(), database, table, "column_name = \"s1\"");
93+
94+
ownerConsumer = createOwnerConsumer(ids.consumer("owner1"), ids.consumerGroup("owner"), 1L);
95+
ownerConsumer.subscribe(ids.getTopic());
96+
97+
final Set<String> rowsBeforeTransfer =
98+
ConsensusSubscriptionTableITSupport.insertRows(
99+
database, table, 100L, 3, true, true, true);
100+
final ConsensusSubscriptionTableITSupport.ConsumedRecords beforeTransfer =
101+
ConsensusSubscriptionTableITSupport.pollAndCommitUntilAtLeast(
102+
ownerConsumer, rowsBeforeTransfer.size(), 60);
103+
104+
ConsensusSubscriptionTableITSupport.assertExactRowKeys(rowsBeforeTransfer, beforeTransfer);
105+
Assert.assertEquals(
106+
Collections.singleton(
107+
ConsensusSubscriptionTableITSupport.normalizeColumnSignature("tag1", "s1")),
108+
beforeTransfer.getSeenColumnSignatures());
109+
110+
ownerConsumer.unsubscribe(ids.getTopic());
111+
ownerConsumer.close();
112+
ownerConsumer = null;
113+
114+
addColumn(database, table, "s4 INT32 FIELD");
115+
transferOwner(ids.getTopic(), "owner2", 2L);
116+
ConsensusSubscriptionTableITSupport.alterConsensusTopicColumnFilter(
117+
ids.getTopic(), "category = \"FIELD\"");
118+
119+
ownerConsumer = createOwnerConsumer(ids.consumer("owner2"), ids.consumerGroup("owner"), 2L);
120+
ownerConsumer.subscribe(ids.getTopic());
121+
122+
final Set<String> rowsAfterTransfer = insertRowsWithS4(database, table, 200L, 3);
123+
final ConsensusSubscriptionTableITSupport.ConsumedRecords afterTransfer =
124+
ConsensusSubscriptionTableITSupport.pollAndCommitUntilContains(
125+
ownerConsumer, rowsAfterTransfer, 80);
126+
127+
Assert.assertTrue(
128+
"Missing post-transfer rows. Consumed records: " + afterTransfer,
129+
afterTransfer.getRowKeys().containsAll(rowsAfterTransfer));
130+
Assert.assertTrue(
131+
afterTransfer
132+
.getSeenColumnSignatures()
133+
.contains(
134+
ConsensusSubscriptionTableITSupport.normalizeColumnSignature(
135+
"tag1", "s1", "s2", "s3", "s4")));
136+
} finally {
137+
ConsensusSubscriptionTableITSupport.cleanup(ownerConsumer, ids.getTopic(), database);
138+
}
139+
}
140+
141+
private static void createOwnedConsensusTopic(
142+
final String topicName, final String database, final String table, final String columnFilter)
143+
throws Exception {
144+
try (final ISubscriptionTableSession session =
145+
new SubscriptionTableSessionBuilder()
146+
.host(EnvFactory.getEnv().getIP())
147+
.port(Integer.parseInt(EnvFactory.getEnv().getPort()))
148+
.build()) {
149+
session.open();
150+
session.dropTopicIfExists(topicName);
151+
152+
final Properties config = new Properties();
153+
config.put(TopicConstant.MODE_KEY, TopicConstant.MODE_CONSENSUS_VALUE);
154+
config.put(TopicConstant.FORMAT_KEY, TopicConstant.FORMAT_SESSION_DATA_SETS_HANDLER_VALUE);
155+
config.put(TopicConstant.DATABASE_KEY, database);
156+
config.put(TopicConstant.TABLE_KEY, table);
157+
config.put(TopicConstant.COLUMN_FILTER_KEY, columnFilter);
158+
config.put(TopicConstant.OWNER_ID_KEY, "owner1");
159+
config.put(TopicConstant.OWNER_EPOCH_KEY, "1");
160+
config.put(
161+
TopicConstant.OWNER_LEASE_DURATION_MS_KEY, String.valueOf(OWNER_LEASE_DURATION_MS));
162+
session.createTopic(topicName, config);
163+
}
164+
}
165+
166+
private static SubscriptionTablePullConsumer createOwnerConsumer(
167+
final String consumerId, final String consumerGroupId, final long ownerEpoch)
168+
throws Exception {
169+
final SubscriptionTablePullConsumer consumer =
170+
(SubscriptionTablePullConsumer)
171+
new SubscriptionTablePullConsumerBuilder()
172+
.host(EnvFactory.getEnv().getIP())
173+
.port(Integer.parseInt(EnvFactory.getEnv().getPort()))
174+
.consumerId(consumerId)
175+
.consumerGroupId(consumerGroupId)
176+
.ownerId(ownerEpoch == 1L ? "owner1" : "owner2")
177+
.ownerEpoch(ownerEpoch)
178+
.heartbeatIntervalMs(1000)
179+
.endpointsSyncIntervalMs(5000)
180+
.autoCommit(false)
181+
.build();
182+
consumer.open();
183+
return consumer;
184+
}
185+
186+
private static void transferOwner(
187+
final String topicName, final String ownerId, final long ownerEpoch) throws Exception {
188+
try (final ISubscriptionTableSession session =
189+
new SubscriptionTableSessionBuilder()
190+
.host(EnvFactory.getEnv().getIP())
191+
.port(Integer.parseInt(EnvFactory.getEnv().getPort()))
192+
.build()) {
193+
session.open();
194+
session.alterTopicOwner(topicName, ownerId, ownerEpoch, OWNER_LEASE_DURATION_MS);
195+
}
196+
}
197+
198+
private static void addColumn(final String database, final String table, final String column)
199+
throws Exception {
200+
try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
201+
session.executeNonQueryStatement("use " + database);
202+
session.executeNonQueryStatement("alter table " + table + " add column " + column);
203+
}
204+
}
205+
206+
private static Set<String> insertRowsWithS4(
207+
final String database, final String table, final long startTimestamp, final int rowCount)
208+
throws Exception {
209+
final Set<String> rowKeys = new LinkedHashSet<>();
210+
try (final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
211+
session.executeNonQueryStatement("use " + database);
212+
for (int row = 0; row < rowCount; row++) {
213+
final long timestamp = startTimestamp + row;
214+
session.executeNonQueryStatement(
215+
String.format(
216+
Locale.ROOT,
217+
"insert into %s(tag1, s1, s2, s3, s4, time) "
218+
+ "values ('tag_%d', %d, %.1f, %s, %d, %d)",
219+
table,
220+
timestamp,
221+
timestamp * 10L,
222+
timestamp + 0.5d,
223+
timestamp % 2 == 0 ? "true" : "false",
224+
timestamp,
225+
timestamp));
226+
rowKeys.add(ConsensusSubscriptionTableITSupport.rowKey(database, table, timestamp));
227+
}
228+
session.executeNonQueryStatement("flush");
229+
}
230+
return rowKeys;
231+
}
232+
}

0 commit comments

Comments
 (0)