Skip to content

Commit 022fb2c

Browse files
committed
Fix DataStream provider lineage edge cases
Restrict upstream traversal to DataStreamScanProvider pipelines with one distinct source, preserving existing lineage for fromDataStream conversions. Fall back to source-provided lineage when the provider exposes none. Add regression coverage, remove redundant tests, and document LineageVertexProvider support for DataStream providers.
1 parent 627fe48 commit 022fb2c

4 files changed

Lines changed: 249 additions & 148 deletions

File tree

‎docs/content/docs/internals/data_lineage.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@ the extended interfaces for Table and DataStream independently. The interface an
4343
{{< img src="/fig/lineage_interfaces.png" alt="Lineage Data Model" width="80%">}}
4444

4545
By default, Table related lineage interfaces or classes are used in Flink Table environment, thus Flink users doesn't need to touch these interfaces. The Flink community will gradually support all
46-
of the common connectors, such as Kafka, JDBC, Cassandra, Hive. If you have a customized connector defined, you need to have customized source/sink implementations of the LineageVertexProvider interface.
46+
of the common connectors, such as Kafka, JDBC, Cassandra, Hive.
47+
48+
If you have a customized connector defined, you need to have customized source/sink implementations of the LineageVertexProvider interface. When a connector uses `DataStreamScanProvider` or `DataStreamSinkProvider`, the provider class can also implement `LineageVertexProvider` to supply lineage metadata.
49+
4750
Within a LineageVertex, a list of Lineage Datasets are defined as metadata for Flink source/sink.
4851

4952

‎flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecTableSourceScan.java‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.flink.streaming.api.operators.StreamSource;
3737
import org.apache.flink.streaming.api.transformations.LegacySourceTransformation;
3838
import org.apache.flink.streaming.api.transformations.PartitionTransformation;
39+
import org.apache.flink.streaming.api.transformations.SourceTransformation;
3940
import org.apache.flink.streaming.api.transformations.SourceTransformationWrapper;
4041
import org.apache.flink.streaming.api.transformations.TransformationWithLineage;
4142
import org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner;
@@ -75,6 +76,7 @@
7576
import java.util.Arrays;
7677
import java.util.List;
7778
import java.util.Optional;
79+
import java.util.stream.Collectors;
7880

7981
import static org.apache.flink.runtime.state.KeyGroupRangeAssignment.DEFAULT_LOWER_BOUND_MAX_PARALLELISM;
8082

@@ -205,6 +207,31 @@ protected Transformation<RowData> translateToPlanInternal(
205207
provider.getClass().getSimpleName() + " is unsupported now.");
206208
}
207209

210+
TransformationWithLineage<?> lineageTransform = null;
211+
if (sourceTransform instanceof TransformationWithLineage) {
212+
lineageTransform = (TransformationWithLineage<?>) sourceTransform;
213+
} else if (provider instanceof DataStreamScanProvider) {
214+
// Only attach table lineage to a pipeline with one distinct source. Replacing an
215+
// individual source's lineage in a multi-source pipeline would be ambiguous.
216+
final List<Transformation<?>> sources =
217+
sourceTransform.getTransitivePredecessors().stream()
218+
.filter(
219+
t ->
220+
t instanceof SourceTransformation
221+
|| t instanceof LegacySourceTransformation)
222+
.distinct()
223+
.collect(Collectors.toList());
224+
if (sources.size() == 1) {
225+
lineageTransform = (TransformationWithLineage<?>) sources.get(0);
226+
}
227+
}
228+
229+
// The source transformation may already hold connector lineage (e.g. Kafka) even when
230+
// the provider does not expose it. Reuse it to preserve the namespace and facets.
231+
if (!lineageVertex.isPresent() && lineageTransform != null) {
232+
lineageVertex = Optional.ofNullable(lineageTransform.getLineageVertex());
233+
}
234+
208235
LineageDataset tableLineageDataset =
209236
TableLineageUtils.createTableLineageDataset(
210237
tableSourceSpec.getContextResolvedTable(), lineageVertex);
@@ -216,21 +243,8 @@ protected Transformation<RowData> translateToPlanInternal(
216243
? Boundedness.BOUNDED
217244
: Boundedness.CONTINUOUS_UNBOUNDED);
218245

219-
if (sourceTransform instanceof TransformationWithLineage) {
220-
((TransformationWithLineage<RowData>) sourceTransform)
221-
.setLineageVertex(sourceLineageVertex);
222-
} else {
223-
// When DataStreamScanProvider returns a multi-operator pipeline, the outermost
224-
// transformation may not implement TransformationWithLineage. Search the transitive
225-
// predecessors to find a SourceTransformation or LegacySourceTransformation that does.
226-
for (Transformation<?> predecessor : sourceTransform.getTransitivePredecessors()) {
227-
if (predecessor != sourceTransform
228-
&& predecessor instanceof TransformationWithLineage) {
229-
((TransformationWithLineage<?>) predecessor)
230-
.setLineageVertex(sourceLineageVertex);
231-
break;
232-
}
233-
}
246+
if (lineageTransform != null) {
247+
lineageTransform.setLineageVertex(sourceLineageVertex);
234248
}
235249

236250
if (sourceParallelismConfigured) {

‎flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/lineage/TableLineageUtilsTest.java‎

Lines changed: 0 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,7 @@
1818

1919
package org.apache.flink.table.planner.lineage;
2020

21-
import org.apache.flink.streaming.api.datastream.DataStream;
22-
import org.apache.flink.streaming.api.datastream.DataStreamSink;
23-
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
24-
import org.apache.flink.streaming.api.lineage.DefaultLineageDataset;
2521
import org.apache.flink.streaming.api.lineage.LineageDataset;
26-
import org.apache.flink.streaming.api.lineage.LineageVertex;
27-
import org.apache.flink.streaming.api.lineage.LineageVertexProvider;
2822
import org.apache.flink.table.api.DataTypes;
2923
import org.apache.flink.table.api.Schema;
3024
import org.apache.flink.table.catalog.Catalog;
@@ -35,17 +29,12 @@
3529
import org.apache.flink.table.catalog.ObjectIdentifier;
3630
import org.apache.flink.table.catalog.ResolvedCatalogTable;
3731
import org.apache.flink.table.catalog.ResolvedSchema;
38-
import org.apache.flink.table.connector.ProviderContext;
39-
import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
40-
import org.apache.flink.table.connector.source.DataStreamScanProvider;
41-
import org.apache.flink.table.data.RowData;
4232

4333
import org.junit.jupiter.api.Test;
4434

4535
import java.util.Arrays;
4636
import java.util.Collections;
4737
import java.util.HashMap;
48-
import java.util.List;
4938
import java.util.Map;
5039
import java.util.Optional;
5140

@@ -126,124 +115,4 @@ void testCreateTableLineageDatasetWithoutCatalog() {
126115
assertThat(tableLineageDataset.catalogContext().getCatalogName()).isEmpty();
127116
assertThat(tableLineageDataset.name()).isEqualTo(objectIdentifier.asSummaryString());
128117
}
129-
130-
@Test
131-
void testExtractLineageDatasetFromDataStreamScanProvider() {
132-
DataStreamScanProvider provider =
133-
new DataStreamScanProvider() {
134-
@Override
135-
public DataStream<RowData> produceDataStream(
136-
ProviderContext providerContext, StreamExecutionEnvironment execEnv) {
137-
return null;
138-
}
139-
140-
@Override
141-
public boolean isBounded() {
142-
return false;
143-
}
144-
};
145-
146-
// A plain DataStreamScanProvider without LineageVertexProvider should return empty
147-
Optional<LineageVertex> result = TableLineageUtils.extractLineageDataset(provider);
148-
assertThat(result).isEmpty();
149-
}
150-
151-
@Test
152-
void testExtractLineageDatasetFromDataStreamScanProviderWithLineage() {
153-
DataStreamScanProvider provider =
154-
new TestDataStreamScanProviderWithLineage("test-source", "test://namespace");
155-
156-
Optional<LineageVertex> result = TableLineageUtils.extractLineageDataset(provider);
157-
assertThat(result).isPresent();
158-
assertThat(result.get().datasets()).hasSize(1);
159-
assertThat(result.get().datasets().get(0).name()).isEqualTo("test-source");
160-
assertThat(result.get().datasets().get(0).namespace()).isEqualTo("test://namespace");
161-
}
162-
163-
@Test
164-
void testExtractLineageDatasetFromDataStreamSinkProvider() {
165-
DataStreamSinkProvider provider =
166-
new DataStreamSinkProvider() {
167-
@Override
168-
public DataStreamSink<?> consumeDataStream(
169-
ProviderContext providerContext, DataStream<RowData> dataStream) {
170-
return null;
171-
}
172-
};
173-
174-
// A plain DataStreamSinkProvider without LineageVertexProvider should return empty
175-
Optional<LineageVertex> result = TableLineageUtils.extractLineageDataset(provider);
176-
assertThat(result).isEmpty();
177-
}
178-
179-
@Test
180-
void testExtractLineageDatasetFromDataStreamSinkProviderWithLineage() {
181-
DataStreamSinkProvider provider =
182-
new TestDataStreamSinkProviderWithLineage("test-sink", "test://namespace");
183-
184-
Optional<LineageVertex> result = TableLineageUtils.extractLineageDataset(provider);
185-
assertThat(result).isPresent();
186-
assertThat(result.get().datasets()).hasSize(1);
187-
assertThat(result.get().datasets().get(0).name()).isEqualTo("test-sink");
188-
assertThat(result.get().datasets().get(0).namespace()).isEqualTo("test://namespace");
189-
}
190-
191-
/**
192-
* A {@link DataStreamScanProvider} that also implements {@link LineageVertexProvider}, similar
193-
* to how Apache Paimon's PaimonDataStreamScanProvider works.
194-
*/
195-
private static class TestDataStreamScanProviderWithLineage
196-
implements DataStreamScanProvider, LineageVertexProvider {
197-
private final String name;
198-
private final String namespace;
199-
200-
TestDataStreamScanProviderWithLineage(String name, String namespace) {
201-
this.name = name;
202-
this.namespace = namespace;
203-
}
204-
205-
@Override
206-
public DataStream<RowData> produceDataStream(
207-
ProviderContext providerContext, StreamExecutionEnvironment execEnv) {
208-
return null;
209-
}
210-
211-
@Override
212-
public boolean isBounded() {
213-
return false;
214-
}
215-
216-
@Override
217-
public LineageVertex getLineageVertex() {
218-
return () ->
219-
List.of(new DefaultLineageDataset(name, namespace, Collections.emptyMap()));
220-
}
221-
}
222-
223-
/**
224-
* A {@link DataStreamSinkProvider} that also implements {@link LineageVertexProvider}, similar
225-
* to how Apache Paimon's PaimonDataStreamSinkProvider works.
226-
*/
227-
private static class TestDataStreamSinkProviderWithLineage
228-
implements DataStreamSinkProvider, LineageVertexProvider {
229-
private final String name;
230-
private final String namespace;
231-
232-
TestDataStreamSinkProviderWithLineage(String name, String namespace) {
233-
this.name = name;
234-
this.namespace = namespace;
235-
}
236-
237-
@Override
238-
public DataStreamSink<?> consumeDataStream(
239-
ProviderContext providerContext, DataStream<RowData> dataStream) {
240-
return null;
241-
}
242-
243-
@Override
244-
public LineageVertex getLineageVertex() {
245-
return () ->
246-
List.of(new DefaultLineageDataset(name, namespace, Collections.emptyMap()));
247-
}
248-
}
249118
}

0 commit comments

Comments
 (0)