From 0910fc925dbe28965f1e15aff8aba5faa651630a Mon Sep 17 00:00:00 2001 From: Liu Liu Date: Mon, 3 Aug 2026 10:53:14 +0800 Subject: [PATCH 01/34] [FLINK-40187][python] Introduce DataType class in DataFrame API (#28843) --- .../reference/pyflink.dataframe/datatype.rst | 20 + flink-python/pyflink/dataframe/datatype.py | 387 +++++++++++++++++- .../pyflink/dataframe/tests/test_datatype.py | 372 ++++++++++++++++- flink-python/pyflink/table/expression.py | 11 +- .../pyflink/table/tests/test_types.py | 22 +- flink-python/pyflink/table/types.py | 28 +- 6 files changed, 807 insertions(+), 33 deletions(-) diff --git a/flink-python/docs/reference/pyflink.dataframe/datatype.rst b/flink-python/docs/reference/pyflink.dataframe/datatype.rst index 9d9b19b5cf550..c439b3a4a47b6 100644 --- a/flink-python/docs/reference/pyflink.dataframe/datatype.rst +++ b/flink-python/docs/reference/pyflink.dataframe/datatype.rst @@ -34,5 +34,25 @@ Example:: :toctree: api/ DataType + DataType.int8 + DataType.int16 + DataType.int32 DataType.int64 + DataType.float32 + DataType.float64 + DataType.decimal DataType.string + DataType.fixed_size_string + DataType.binary + DataType.fixed_size_binary + DataType.bool + DataType.null + DataType.date + DataType.time + DataType.timestamp + DataType.timestamp_ltz + DataType.list + DataType.map + DataType.struct + DataType.not_null + DataType.nullable diff --git a/flink-python/pyflink/dataframe/datatype.py b/flink-python/pyflink/dataframe/datatype.py index c22f4ba6d6819..84676885a6f53 100644 --- a/flink-python/pyflink/dataframe/datatype.py +++ b/flink-python/pyflink/dataframe/datatype.py @@ -16,11 +16,33 @@ # limitations under the License. ################################################################################ -from pyflink.table.types import DataType as TableDataType, DataTypes +import datetime +import decimal +import types +from functools import partial +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, get_args, get_origin + +from pyflink.table.types import DataType as TableDataType, DataTypes, NullType from pyflink.util.api_stability_decorators import PublicEvolving __all__ = ["DataType"] +_PEP_604_UNION_TYPE = getattr(types, "UnionType", None) + +_BASIC_TYPE_HINT_FACTORIES: Dict[Any, Callable[[], TableDataType]] = { + bool: DataTypes.BOOLEAN, + int: DataTypes.BIGINT, + float: DataTypes.DOUBLE, + str: DataTypes.STRING, + bytes: DataTypes.BYTES, + bytearray: DataTypes.BYTES, + decimal.Decimal: partial(DataTypes.DECIMAL, 38, 18), + datetime.date: DataTypes.DATE, + datetime.time: DataTypes.TIME, + datetime.datetime: DataTypes.TIMESTAMP, + Any: DataTypes.STRING, +} + @PublicEvolving() class DataType: @@ -39,8 +61,13 @@ class DataType: """ def __init__(self, table_data_type: TableDataType): + if isinstance(table_data_type, NullType) and not table_data_type._nullable: + raise ValueError("NULL data type must be nullable") self._table_data_type = table_data_type + def __repr__(self) -> str: + return f"DataType({self._table_data_type!r})" + @PublicEvolving() def __eq__(self, other: object) -> bool: if not isinstance(other, DataType): @@ -49,7 +76,55 @@ def __eq__(self, other: object) -> bool: @PublicEvolving() def __hash__(self) -> int: - return hash(str(self._table_data_type)) + return hash(repr(self._table_data_type)) + + @PublicEvolving() + def not_null(self) -> "DataType": + """ + Return a non-nullable version of this data type. + + .. versionadded:: 2.4.0 + """ + return DataType(self._table_data_type.not_null()) + + @PublicEvolving() + def nullable(self) -> "DataType": + """ + Return a nullable version of this data type. + + .. versionadded:: 2.4.0 + """ + return DataType(self._table_data_type.nullable()) + + @classmethod + @PublicEvolving() + def int8(cls) -> "DataType": + """ + Create an 8-bit signed integer type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.TINYINT()) + + @classmethod + @PublicEvolving() + def int16(cls) -> "DataType": + """ + Create a 16-bit signed integer type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.SMALLINT()) + + @classmethod + @PublicEvolving() + def int32(cls) -> "DataType": + """ + Create a 32-bit signed integer type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.INT()) @classmethod @PublicEvolving() @@ -66,6 +141,39 @@ def int64(cls) -> "DataType": """ return cls(DataTypes.BIGINT()) + @classmethod + @PublicEvolving() + def float32(cls) -> "DataType": + """ + Create a 32-bit floating point type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.FLOAT()) + + @classmethod + @PublicEvolving() + def float64(cls) -> "DataType": + """ + Create a 64-bit floating point type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.DOUBLE()) + + @classmethod + @PublicEvolving() + def decimal(cls, precision: int, scale: int) -> "DataType": + """ + Create a decimal type with the given precision and scale. + + :param precision: Total number of digits. + :param scale: Number of digits to the right of the decimal point. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.DECIMAL(precision, scale)) + @classmethod @PublicEvolving() def string(cls) -> "DataType": @@ -81,5 +189,280 @@ def string(cls) -> "DataType": """ return cls(DataTypes.STRING()) + @classmethod + @PublicEvolving() + def fixed_size_string(cls, length: int) -> "DataType": + """ + Create a fixed-length character string type. + + :param length: Number of characters. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.CHAR(length)) + + @classmethod + @PublicEvolving() + def binary(cls) -> "DataType": + """ + Create a variable-length binary string type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.BYTES()) + + @classmethod + @PublicEvolving() + def fixed_size_binary(cls, length: int) -> "DataType": + """ + Create a fixed-length binary string type. + + :param length: Number of bytes. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.BINARY(length)) + + @classmethod + @PublicEvolving() + def bool(cls) -> "DataType": + """ + Create a boolean type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.BOOLEAN()) + + @classmethod + @PublicEvolving() + def null(cls) -> "DataType": + """ + Create a null type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.NULL()) + + @classmethod + @PublicEvolving() + def date(cls) -> "DataType": + """ + Create a date type. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.DATE()) + + @classmethod + @PublicEvolving() + def time(cls, precision: int = 0) -> "DataType": + """ + Create a time type. + + :param precision: Number of fractional-second digits. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.TIME(precision)) + + @classmethod + @PublicEvolving() + def timestamp(cls, precision: int = 6) -> "DataType": + """ + Create a timestamp type without a time zone. + + :param precision: Number of fractional-second digits. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.TIMESTAMP(precision)) + + @classmethod + @PublicEvolving() + def timestamp_ltz(cls, precision: int = 6) -> "DataType": + """ + Create a timestamp type with a local time zone. + + :param precision: Number of fractional-second digits. + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.TIMESTAMP_LTZ(precision)) + + @classmethod + @PublicEvolving() + def list(cls, dtype: "DataType") -> "DataType": + """ + Create a list type. + + :param dtype: Type of each list element. + + Example:: + + >>> import pyflink.dataframe as pf + >>> scores_type = pf.DataType.list(pf.DataType.int32()) + + .. versionadded:: 2.4.0 + """ + return cls(DataTypes.ARRAY(dtype._to_table_data_type())) + + @classmethod + @PublicEvolving() + def map(cls, key_type: "DataType", value_type: "DataType") -> "DataType": + """ + Create a map type. + + :param key_type: Type of each map key. + :param value_type: Type of each map value. + + Example:: + + >>> import pyflink.dataframe as pf + >>> config_type = pf.DataType.map( + ... pf.DataType.string(), + ... pf.DataType.int64(), + ... ) + + .. versionadded:: 2.4.0 + """ + return cls( + DataTypes.MAP( + key_type._to_table_data_type(), + value_type._to_table_data_type(), + ) + ) + + @classmethod + @PublicEvolving() + def struct( + cls, + fields: Union[ + Dict[str, "DataType"], + List[Tuple[str, "DataType"]], + ], + ) -> "DataType": + """ + Create a struct type with named fields. + + ``fields`` may be an insertion-ordered dictionary or a list of name and type pairs. + + :param fields: Field names and their data types. + + Example:: + + >>> import pyflink.dataframe as pf + >>> person_type = pf.DataType.struct({ + ... "name": pf.DataType.string(), + ... "age": pf.DataType.int32(), + ... }) + + Fields may also be passed as a list of name and type pairs:: + + >>> person_type = pf.DataType.struct([ + ... ("name", pf.DataType.string()), + ... ("age", pf.DataType.int32()), + ... ]) + + .. versionadded:: 2.4.0 + """ + field_items = fields.items() if isinstance(fields, dict) else fields + + return cls( + DataTypes.ROW( + [ + DataTypes.FIELD(name, data_type._to_table_data_type()) + for name, data_type in field_items + ] + ) + ) + + @classmethod + def _from_type_hint(cls, type_hint: Any) -> "DataType": + def infer_union_type(hint: Any, arguments: Tuple[Any, ...]) -> "DataType": + non_none_types = [ + argument for argument in arguments if argument is not type(None) + ] + if len(non_none_types) == 1: + return infer(non_none_types[0]).nullable() + + raise TypeError( + f"Cannot infer DataType from type hint '{hint}'. " + "Please specify the data type explicitly." + ) + + def infer_basic_type(hint: Any) -> Optional["DataType"]: + factory = _BASIC_TYPE_HINT_FACTORIES.get(hint) + return cls(factory()) if factory is not None else None + + def infer(hint: Any) -> "DataType": + origin = get_origin(hint) + arguments = get_args(hint) + + if origin is Union or ( + _PEP_604_UNION_TYPE is not None + and origin is _PEP_604_UNION_TYPE + ): + return infer_union_type(hint, arguments) + + if origin is list: + if not arguments: + raise TypeError( + "Cannot infer DataType from list without type argument. " + "Use list[T], for example list[int]." + ) + return cls.list(infer(arguments[0])) + + if origin is dict: + if len(arguments) != 2: + raise TypeError( + "Cannot infer DataType from dict without key and value type arguments. " + "Use dict[K, V], for example dict[str, int]." + ) + return cls.map( + infer(arguments[0]), + infer(arguments[1]), + ) + + data_type = infer_basic_type(hint) + if data_type is not None: + return data_type + + raise TypeError( + f"Cannot infer DataType from type hint '{hint}'. " + "Please specify the data type explicitly." + ) + + return infer(type_hint) + + @classmethod + def _from_sql(cls, sql_type: str) -> "DataType": + """ + Create a data type from its SQL representation. + + :param sql_type: SQL data type string, such as ``INT`` or ``ARRAY``. + :raises ValueError: If the SQL data type cannot be parsed. + """ + from py4j.protocol import Py4JJavaError + + from pyflink.java_gateway import get_gateway + from pyflink.table.types import _from_java_data_type + from pyflink.util.exceptions import JavaException + + try: + gateway = get_gateway() + j_logical_type = ( + gateway.jvm.org.apache.flink.table.types.logical.utils.LogicalTypeParser.parse( + sql_type, + gateway.jvm.Thread.currentThread().getContextClassLoader(), + ) + ) + j_data_type = ( + gateway.jvm.org.apache.flink.table.types.utils.TypeConversions + .fromLogicalToDataType(j_logical_type) + ) + return cls(_from_java_data_type(j_data_type)) + except (JavaException, Py4JJavaError) as exc: + raise ValueError(str(exc)) from None + def _to_table_data_type(self) -> TableDataType: return self._table_data_type diff --git a/flink-python/pyflink/dataframe/tests/test_datatype.py b/flink-python/pyflink/dataframe/tests/test_datatype.py index 756bba371ea87..f129f9a5eb6f8 100644 --- a/flink-python/pyflink/dataframe/tests/test_datatype.py +++ b/flink-python/pyflink/dataframe/tests/test_datatype.py @@ -16,26 +16,128 @@ # limitations under the License. ################################################################################ +import datetime +import decimal +import sys import unittest +from typing import Any, List, Optional, Union import pyflink.dataframe as pf from pyflink.table import DataTypes -from pyflink.util.api_stability_decorators import PublicEvolving +from pyflink.testing.test_case_utils import PyFlinkTestCase class DataTypeTests(unittest.TestCase): - def test_public_factory_surface(self): + def test_public_api_surface(self): public_methods = { name for name in dir(pf.DataType) if not name.startswith("_") } - self.assertEqual(public_methods, {"int64", "string"}) + self.assertEqual( + public_methods, + { + "binary", + "bool", + "date", + "decimal", + "fixed_size_binary", + "fixed_size_string", + "float32", + "float64", + "int8", + "int16", + "int32", + "int64", + "list", + "map", + "not_null", + "null", + "nullable", + "string", + "struct", + "time", + "timestamp", + "timestamp_ltz", + }, + ) - def test_int64_maps_to_table_bigint(self): - self.assertEqual(pf.DataType.int64()._to_table_data_type(), DataTypes.BIGINT()) + def test_scalar_factories_map_to_table_types(self): + expected_types = { + "binary": DataTypes.BYTES(), + "bool": DataTypes.BOOLEAN(), + "date": DataTypes.DATE(), + "float32": DataTypes.FLOAT(), + "float64": DataTypes.DOUBLE(), + "int8": DataTypes.TINYINT(), + "int16": DataTypes.SMALLINT(), + "int32": DataTypes.INT(), + "int64": DataTypes.BIGINT(), + "null": DataTypes.NULL(), + "string": DataTypes.STRING(), + } + + for factory_name, expected_type in expected_types.items(): + with self.subTest(factory_name=factory_name): + data_type = getattr(pf.DataType, factory_name)() + self.assertEqual(data_type._to_table_data_type(), expected_type) + + def test_parameterized_factories_map_to_table_types(self): + test_cases = [ + ( + "decimal", + pf.DataType.decimal, + {"precision": 10, "scale": 3}, + DataTypes.DECIMAL(10, 3), + ), + ( + "fixed_size_binary", + pf.DataType.fixed_size_binary, + {"length": 16}, + DataTypes.BINARY(16), + ), + ( + "fixed_size_string", + pf.DataType.fixed_size_string, + {"length": 12}, + DataTypes.CHAR(12), + ), + ("time_default", pf.DataType.time, {}, DataTypes.TIME(0)), + ( + "time_precision", + pf.DataType.time, + {"precision": 3}, + DataTypes.TIME(3), + ), + ( + "timestamp_default", + pf.DataType.timestamp, + {}, + DataTypes.TIMESTAMP(6), + ), + ( + "timestamp_precision", + pf.DataType.timestamp, + {"precision": 3}, + DataTypes.TIMESTAMP(3), + ), + ( + "timestamp_ltz_default", + pf.DataType.timestamp_ltz, + {}, + DataTypes.TIMESTAMP_LTZ(6), + ), + ( + "timestamp_ltz_precision", + pf.DataType.timestamp_ltz, + {"precision": 3}, + DataTypes.TIMESTAMP_LTZ(3), + ), + ] - def test_string_maps_to_table_string(self): - self.assertEqual(pf.DataType.string()._to_table_data_type(), DataTypes.STRING()) + for factory_name, factory, arguments, expected_type in test_cases: + with self.subTest(factory_name=factory_name): + data_type = factory(**arguments) + self.assertEqual(data_type._to_table_data_type(), expected_type) def test_logically_equal_types_compare_and_hash_equally(self): first_int = pf.DataType.int64() @@ -46,19 +148,253 @@ def test_logically_equal_types_compare_and_hash_equally(self): self.assertNotEqual(first_int, string) self.assertEqual(len({first_int, second_int, string}), 2) - def test_equality_and_hash_are_public_evolving(self): - for method in [pf.DataType.__eq__, pf.DataType.__hash__]: - with self.subTest(method=method.__name__): - self.assertIn( - PublicEvolving, - getattr(method, "__stability_decorators", set()), + def test_nullability_participates_in_equality_and_hashing(self): + nullable = pf.DataType.int32() + first_non_nullable = nullable.not_null() + second_non_nullable = pf.DataType.int32().not_null() + + self.assertNotEqual(nullable, first_non_nullable) + self.assertEqual(first_non_nullable, second_non_nullable) + self.assertEqual(hash(first_non_nullable), hash(second_non_nullable)) + self.assertEqual(len({nullable, first_non_nullable}), 2) + + def test_nullability_modifiers_preserve_the_original_type(self): + original = pf.DataType.int32() + + non_nullable = original.not_null() + nullable_again = non_nullable.nullable() + + self.assertEqual(original._to_table_data_type(), DataTypes.INT()) + self.assertEqual( + non_nullable._to_table_data_type(), + DataTypes.INT().not_null(), + ) + self.assertEqual(nullable_again._to_table_data_type(), DataTypes.INT()) + + def test_null_type_cannot_be_made_non_nullable(self): + with self.assertRaisesRegex(ValueError, "NULL"): + pf.DataType.null().not_null() + + def test_list_preserves_its_element_type(self): + list_type = pf.DataType.list(dtype=pf.DataType.int32().not_null()) + + self.assertEqual( + list_type._to_table_data_type(), + DataTypes.ARRAY(DataTypes.INT().not_null()), + ) + + def test_map_preserves_its_key_and_value_types(self): + map_type = pf.DataType.map( + key_type=pf.DataType.string(), + value_type=pf.DataType.int64(), + ) + + self.assertEqual( + map_type._to_table_data_type(), + DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT()), + ) + + def test_struct_preserves_dict_insertion_order(self): + struct_type = pf.DataType.struct( + fields={ + "name": pf.DataType.string(), + "age": pf.DataType.int32().not_null(), + } + ) + + self.assertEqual( + struct_type._to_table_data_type(), + DataTypes.ROW( + [ + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("age", DataTypes.INT().not_null()), + ] + ), + ) + + def test_struct_preserves_list_field_order(self): + struct_type = pf.DataType.struct( + fields=[ + ("age", pf.DataType.int32().not_null()), + ("name", pf.DataType.string()), + ] + ) + + self.assertEqual( + struct_type._to_table_data_type(), + DataTypes.ROW( + [ + DataTypes.FIELD("age", DataTypes.INT().not_null()), + DataTypes.FIELD("name", DataTypes.STRING()), + ] + ), + ) + + def test_from_basic_python_type_hints(self): + expected_types = { + bool: pf.DataType.bool(), + int: pf.DataType.int64(), + float: pf.DataType.float64(), + str: pf.DataType.string(), + bytes: pf.DataType.binary(), + bytearray: pf.DataType.binary(), + decimal.Decimal: pf.DataType.decimal(38, 18), + datetime.date: pf.DataType.date(), + datetime.time: pf.DataType.time(), + datetime.datetime: pf.DataType.timestamp(), + Any: pf.DataType.string(), + } + + for python_type, expected_type in expected_types.items(): + with self.subTest(python_type=python_type): + self.assertEqual( + pf.DataType._from_type_hint(python_type), + expected_type, + ) + + def test_from_optional_type_hint(self): + self.assertEqual( + pf.DataType._from_type_hint(Optional[int]), + pf.DataType.int64(), + ) + + @unittest.skipIf( + sys.version_info < (3, 10), + "PEP 604 union types require Python 3.10 or later", + ) + def test_from_pep_604_union_type_hint(self): + self.assertEqual( + pf.DataType._from_type_hint(int | None), + pf.DataType.int64(), + ) + self.assertEqual( + pf.DataType._from_type_hint(list[int | None]), + pf.DataType.list(dtype=pf.DataType.int64()), + ) + with self.assertRaises(TypeError): + pf.DataType._from_type_hint(int | str) + + def test_from_list_type_hint(self): + self.assertEqual( + pf.DataType._from_type_hint(list[int]), + pf.DataType.list(dtype=pf.DataType.int64()), + ) + + def test_from_dict_type_hint(self): + self.assertEqual( + pf.DataType._from_type_hint(dict[str, float]), + pf.DataType.map( + key_type=pf.DataType.string(), + value_type=pf.DataType.float64(), + ), + ) + + def test_from_type_hint_rejects_ambiguous_or_incomplete_hints(self): + invalid_hints = [ + List, + complex, + ] + + for type_hint in invalid_hints: + with self.subTest(type_hint=type_hint): + with self.assertRaises(TypeError): + pf.DataType._from_type_hint(type_hint) + + def test_from_type_hint_reports_ambiguous_union_error(self): + with self.assertRaises(TypeError) as context: + pf.DataType._from_type_hint(Union[int, str]) + + self.assertEqual( + "Cannot infer DataType from type hint 'typing.Union[int, str]'. " + "Please specify the data type explicitly.", + str(context.exception), + ) + + def test_repr_preserves_type_parameters_and_nested_nullability(self): + self.assertEqual( + repr( + pf.DataType.list( + pf.DataType.struct( + [ + ( + "amount", + pf.DataType.decimal(10, 2).not_null(), + ) + ] + ).not_null() + ) + ), + "DataType(ArrayType(" + "RowType(RowField(amount, DecimalType(10, 2, false), ...), false), " + "true))", + ) + + +class DataTypeExpressionTests(PyFlinkTestCase): + def test_expression_casts_accept_dataframe_data_types(self): + test_cases = [ + ("cast", "cast(value, DOUBLE)"), + ("try_cast", "TRY_CAST(value, DOUBLE)"), + ] + + for operation, expected_expression in test_cases: + with self.subTest(operation=operation): + expression = getattr(pf.col("value"), operation)( + pf.DataType.float64() ) - def test_nullability_modifiers_are_not_exposed(self): - for data_type in [pf.DataType.int64(), pf.DataType.string()]: - with self.subTest(data_type=data_type): - self.assertFalse(hasattr(data_type, "not_null")) - self.assertFalse(hasattr(data_type, "nullable")) + self.assertEqual(str(expression), expected_expression) + + +class DataTypeSqlTests(PyFlinkTestCase): + def test_from_sql_parses_scalar_and_nested_types(self): + test_cases = [ + ("INT", pf.DataType.int32()), + ( + "DECIMAL(10, 3) NOT NULL", + pf.DataType.decimal(10, 3).not_null(), + ), + ( + "ROW>", + pf.DataType.struct( + fields=[ + ("name", pf.DataType.string()), + ( + "scores", + pf.DataType.list( + dtype=pf.DataType.float64().not_null() + ), + ), + ] + ), + ), + ] + + for sql_type, expected_type in test_cases: + with self.subTest(sql_type=sql_type): + self.assertEqual(pf.DataType._from_sql(sql_type), expected_type) + + def test_from_sql_preserves_timestamp_precision(self): + self.assertEqual( + pf.DataType._from_sql("TIMESTAMP(9)"), + pf.DataType.timestamp(precision=9), + ) + + def test_from_sql_preserves_timestamp_ltz_precision(self): + self.assertEqual( + pf.DataType._from_sql("TIMESTAMP_LTZ(3)"), + pf.DataType.timestamp_ltz(precision=3), + ) + + def test_from_sql_supports_null_type(self): + self.assertEqual( + pf.DataType._from_sql("NULL"), + pf.DataType.null(), + ) + + def test_from_sql_reports_parser_errors_as_value_errors(self): + with self.assertRaises(ValueError): + pf.DataType._from_sql("VARCHAR(test)") if __name__ == "__main__": diff --git a/flink-python/pyflink/table/expression.py b/flink-python/pyflink/table/expression.py index 82c49f2edc63d..aa3b952f215f2 100644 --- a/flink-python/pyflink/table/expression.py +++ b/flink-python/pyflink/table/expression.py @@ -20,7 +20,12 @@ from pyflink import add_version_doc from pyflink.java_gateway import get_gateway -from pyflink.table.types import DataType, DataTypes, _to_java_data_type +from pyflink.table.types import ( + DataType, + DataTypes, + _TableDataTypeLike, + _to_java_data_type, +) from pyflink.util.api_stability_decorators import PublicEvolving from pyflink.util.java_utils import to_jarray @@ -886,7 +891,7 @@ def as_argument(self, name: str) -> 'Expression': """ return _binary_op("asArgument")(self, name) - def cast(self, data_type: DataType) -> 'Expression': + def cast(self, data_type: _TableDataTypeLike) -> 'Expression': """ Returns a new value being cast to type type. A cast error throws an exception and fails the job. @@ -900,7 +905,7 @@ def cast(self, data_type: DataType) -> 'Expression': """ return _binary_op("cast")(self, _to_java_data_type(data_type)) - def try_cast(self, data_type: DataType) -> 'Expression': + def try_cast(self, data_type: _TableDataTypeLike) -> 'Expression': """ Like cast, but in case of error, returns NULL rather than failing the job. diff --git a/flink-python/pyflink/table/tests/test_types.py b/flink-python/pyflink/table/tests/test_types.py index d3bd37c6ffd8f..f8124ba5e8cdf 100644 --- a/flink-python/pyflink/table/tests/test_types.py +++ b/flink-python/pyflink/table/tests/test_types.py @@ -128,6 +128,18 @@ def dst(self, dt): class TypesTests(PyFlinkTestCase): + def test_row_type_repr_includes_nullability(self): + row_type = RowType([RowField("id", BigIntType())]) + + self.assertEqual( + "RowType(RowField(id, BigIntType(true), ...), true)", + repr(row_type), + ) + self.assertEqual( + "RowType(RowField(id, BigIntType(true), ...), false)", + repr(row_type.not_null()), + ) + def test_infer_schema(self): from decimal import Decimal @@ -170,14 +182,14 @@ def __init__(self): 'DoubleType(true)', "ArrayType(DoubleType(false), true)", "ArrayType(BigIntType(true), true)", - 'RowType(RowField(_1, BigIntType(true), ...))', - 'RowType(RowField(x, DoubleType(true), ...),RowField(y, DoubleType(true), ...))', + 'RowType(RowField(_1, BigIntType(true), ...), true)', + 'RowType(RowField(x, DoubleType(true), ...),RowField(y, DoubleType(true), ...), true)', 'MapType(VarCharType(2147483647, false), BigIntType(true), true)', 'VarBinaryType(2147483647, true)', 'DecimalType(38, 18, true)', - 'RowType(RowField(a, BigIntType(true), ...))', - 'RowType(RowField(a, BigIntType(true), ...))', - 'RowType(RowField(a, BigIntType(true), ...))', + 'RowType(RowField(a, BigIntType(true), ...), true)', + 'RowType(RowField(a, BigIntType(true), ...), true)', + 'RowType(RowField(a, BigIntType(true), ...), true)', ] schema = _infer_schema_from_data([data]) diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index b62b55af9e1f3..964097f25e272 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -29,7 +29,7 @@ from threading import RLock from py4j.java_gateway import get_java_class -from typing import List, Union +from typing import List, Protocol, Union from pyflink.common.types import _create_row from pyflink.util.api_stability_decorators import PublicEvolving @@ -123,6 +123,14 @@ def from_sql_type(self, obj): return obj +class _SupportsToTableDataType(Protocol): + def _to_table_data_type(self) -> DataType: + ... + + +_TableDataTypeLike = Union[DataType, _SupportsToTableDataType] + + class AtomicType(DataType): """ An internal type used to represent everything that is not @@ -1216,7 +1224,8 @@ def __getitem__(self, key): raise TypeError('RowType keys should be strings, integers or slices') def __repr__(self): - return "RowType(%s)" % ",".join(repr(field) for field in self) + fields = ",".join(repr(field) for field in self) + return f"RowType({fields}, {str(self._nullable).lower()})" def field_names(self): """ @@ -1709,7 +1718,8 @@ def _from_java_data_type(j_data_type): elif is_instance_of(logical_type, gateway.jvm.TimeType): data_type = DataTypes.TIME(logical_type.getPrecision(), logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.TimestampType): - data_type = DataTypes.TIMESTAMP(precision=3, nullable=logical_type.isNullable()) + data_type = DataTypes.TIMESTAMP( + precision=logical_type.getPrecision(), nullable=logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.BooleanType): data_type = DataTypes.BOOLEAN(logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.TinyIntType): @@ -1729,7 +1739,8 @@ def _from_java_data_type(j_data_type): TypeError("Unsupported type: %s, ZonedTimestampType is not supported yet." % j_data_type) elif is_instance_of(logical_type, gateway.jvm.LocalZonedTimestampType): - data_type = DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(nullable=logical_type.isNullable()) + data_type = DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE( + precision=logical_type.getPrecision(), nullable=logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.DayTimeIntervalType) or \ is_instance_of(logical_type, gateway.jvm.YearMonthIntervalType): data_type = _from_java_interval_type(logical_type) @@ -1759,6 +1770,8 @@ def _from_java_data_type(j_data_type): % type_info) elif is_instance_of(logical_type, gateway.jvm.RawType): data_type = RawType() + elif is_instance_of(logical_type, gateway.jvm.NullType): + data_type = DataTypes.NULL() else: raise TypeError("Unsupported type: %s, it is not supported yet in current python type" " system" % j_data_type) @@ -1821,10 +1834,15 @@ def _from_java_data_type(j_data_type): TypeError("Unsupported data type: %s" % j_data_type) -def _to_java_data_type(data_type: DataType): +def _to_java_data_type(data_type: _TableDataTypeLike): """ Converts the specified Python DataType to Java DataType. """ + if not isinstance(data_type, DataType): + to_table_data_type = getattr(data_type, "_to_table_data_type", None) + if callable(to_table_data_type): + data_type = to_table_data_type() + gateway = get_gateway() JDataTypes = gateway.jvm.org.apache.flink.table.api.DataTypes From 0b66c7ee9940dc93463b8a146220d92fac1529e6 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 10:08:03 +0200 Subject: [PATCH 02/34] [FLINK-40286][table] Adapt keyless upsert sink should fall back to retract to `MiscSemanticTests` --- .../table/planner/plan/nodes/exec/common/CalcTestPrograms.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CalcTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CalcTestPrograms.java index 04c867637572a..77a69a6b4e672 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CalcTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/common/CalcTestPrograms.java @@ -328,7 +328,7 @@ public class CalcTestPrograms { .setupTableSink( SinkTestStep.newBuilder("coalesce_sink") .addSchema("order_id_str STRING") - .consumedValues("+I[1]", "+I[2]") + .consumedValues("+I[1]", "-D[1]", "+I[1]", "+I[2]") .build()) .runSql( "INSERT INTO coalesce_sink " From 77c47c84a4d27fcb72ac008e26d64a983de7e400 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 10:08:52 +0200 Subject: [PATCH 03/34] [FLINK-40283][tests] Rename `CorrelateITCase2` to `Correlate2ITCase` to make executing while CI/local tests --- .../sql/{CorrelateITCase2.scala => Correlate2ITCase.scala} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/{CorrelateITCase2.scala => Correlate2ITCase.scala} (99%) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CorrelateITCase2.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/Correlate2ITCase.scala similarity index 99% rename from flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CorrelateITCase2.scala rename to flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/Correlate2ITCase.scala index c9f85316776db..98b3614619a1c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/CorrelateITCase2.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/Correlate2ITCase.scala @@ -25,7 +25,7 @@ import org.apache.flink.table.planner.runtime.utils.TestData._ import org.junit.jupiter.api.{BeforeEach, Test} -class CorrelateITCase2 extends BatchTestBase { +class Correlate2ITCase extends BatchTestBase { @BeforeEach override def before(): Unit = { From 77a017a3970931286e11a2b29ab10326b55c525b Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 10:09:56 +0200 Subject: [PATCH 04/34] [FLINK-40288][table] Use correct type in `ExprCodeGenerator#visitFieldAccess` --- .../flink/table/planner/codegen/ExprCodeGenerator.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala index 9427dace35da5..147fc1bdf36ae 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala @@ -389,7 +389,8 @@ class ExprCodeGenerator( val index = rexFieldAccess.getField.getIndex val fieldAccessExpr = generateFieldAccess(ctx, refExpr.resultType, refExpr.resultTerm, index) - val resultType = fieldAccessExpr.resultType + // Use the field access' own (planner-derived) type: a field of a nullable parent row is nullable even if declared NOT NULL. + val resultType = FlinkTypeFactory.toLogicalType(rexFieldAccess.getType) val resultTypeTerm = primitiveTypeTermForType(resultType) val defaultValue = primitiveDefaultValue(resultType) @@ -410,7 +411,7 @@ class ExprCodeGenerator( |} |""".stripMargin - GeneratedExpression(resultTerm, nullTerm, resultCode, fieldAccessExpr.resultType) + GeneratedExpression(resultTerm, nullTerm, resultCode, resultType) } override def visitLiteral(literal: RexLiteral): GeneratedExpression = { From 6e57cc57b3119a68ce2edc45095521fa1b1d485e Mon Sep 17 00:00:00 2001 From: Purushottam Sinha Date: Wed, 29 Jul 2026 22:43:31 +0530 Subject: [PATCH 05/34] [FLINK-40256][docs] Include sub-package config options in the configuration reference ConfigurationOptionLocator discovers ConfigOptions from a hard-coded list of packages and reads each with Files.newDirectoryStream, which does not recurse into sub-packages. Options outside that list are dropped from the generated configuration reference without any error, and ConfigOptionsDocsCompletenessITCase cannot detect it because it derives its expectations from the same list. All seven state.backend.rocksdb.manual-compaction.* options were affected: they carry @Documentation.Section(EXPERT_ROCKSDB) but live in org.apache.flink.state.rocksdb.sstmerge, a sub-package of a searched package, so the feature shipped in 1.20 had no documented configuration. Add a location for the sub-package and regenerate the affected tables. RocksDBManualCompactionOptions needs a stability annotation because becoming discoverable also subjects it to ConfigOptionsDocGenerator#verifyClassAnnotation; @PublicEvolving matches RocksDBOptions and RocksDBConfigurableOptions in the same module. Two option descriptions were missing a space between concatenated sentences, which is now user-visible, so fix those too. Add ConfigurationOptionLocatorTest to prevent recurrence: it scans the source tree and fails when a @Documentation.Section option sits in a package the locator does not search. Repo-wide it needs no exclusions. Generated-by: Claude Code (claude-opus-5) --- .../generated/expert_rocksdb_section.html | 42 +++++ ...cksdb_manual_compaction_configuration.html | 54 +++++++ .../docs/util/ConfigurationOptionLocator.java | 8 + .../util/ConfigurationOptionLocatorTest.java | 146 ++++++++++++++++++ .../RocksDBManualCompactionOptions.java | 6 +- 5 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html create mode 100644 flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java diff --git a/docs/layouts/shortcodes/generated/expert_rocksdb_section.html b/docs/layouts/shortcodes/generated/expert_rocksdb_section.html index eac1d574e26d9..e34bf48952e99 100644 --- a/docs/layouts/shortcodes/generated/expert_rocksdb_section.html +++ b/docs/layouts/shortcodes/generated/expert_rocksdb_section.html @@ -20,6 +20,48 @@ String The local directory (on the TaskManager) where RocksDB puts its files. Per default, it will be <WORKING_DIR>/tmp. See process.taskmanager.working-dir for more details. + +
state.backend.rocksdb.manual-compaction.max-auto-compactions
+ 30 + Integer + The maximum number of automatic compactions running for manual compaction to start. If the actual number is higher, manual compaction won't be started to avoid delaying automatic ones. + + +
state.backend.rocksdb.manual-compaction.max-file-size-to-compact
+ 50 kb + MemorySize + The maximum size of individual input files + + +
state.backend.rocksdb.manual-compaction.max-files-to-compact
+ 30 + Integer + The maximum number of input files to compact together in a single compaction run + + +
state.backend.rocksdb.manual-compaction.max-output-file-size
+ 64 mb + MemorySize + The maximum output file size + + +
state.backend.rocksdb.manual-compaction.max-parallel-compactions
+ 5 + Integer + The maximum number of manual compactions to start. Note that only one of them can run at a time as of v8.10.0; all the others will be waiting + + +
state.backend.rocksdb.manual-compaction.min-files-to-compact
+ 5 + Integer + The minimum number of input files to compact together in a single compaction run + + +
state.backend.rocksdb.manual-compaction.min-interval
+ 0 ms + Duration + The minimum interval between manual compactions. Zero disables manual compactions +
state.backend.rocksdb.options-factory
(none) diff --git a/docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html b/docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html new file mode 100644 index 0000000000000..fce0dd16f6a03 --- /dev/null +++ b/docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultTypeDescription
state.backend.rocksdb.manual-compaction.max-auto-compactions
30IntegerThe maximum number of automatic compactions running for manual compaction to start. If the actual number is higher, manual compaction won't be started to avoid delaying automatic ones.
state.backend.rocksdb.manual-compaction.max-file-size-to-compact
50 kbMemorySizeThe maximum size of individual input files
state.backend.rocksdb.manual-compaction.max-files-to-compact
30IntegerThe maximum number of input files to compact together in a single compaction run
state.backend.rocksdb.manual-compaction.max-output-file-size
64 mbMemorySizeThe maximum output file size
state.backend.rocksdb.manual-compaction.max-parallel-compactions
5IntegerThe maximum number of manual compactions to start. Note that only one of them can run at a time as of v8.10.0; all the others will be waiting
state.backend.rocksdb.manual-compaction.min-files-to-compact
5IntegerThe minimum number of input files to compact together in a single compaction run
state.backend.rocksdb.manual-compaction.min-interval
0 msDurationThe minimum interval between manual compactions. Zero disables manual compactions
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java b/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java index ed2d987be504a..7001b86e48239 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java @@ -68,6 +68,9 @@ public class ConfigurationOptionLocator { new OptionsClassLocation( "flink-state-backends/flink-statebackend-rocksdb", "org.apache.flink.state.rocksdb"), + new OptionsClassLocation( + "flink-state-backends/flink-statebackend-rocksdb", + "org.apache.flink.state.rocksdb.sstmerge"), new OptionsClassLocation( "flink-state-backends/flink-statebackend-forst", "org.apache.flink.state.forst"), @@ -132,6 +135,11 @@ public ConfigurationOptionLocator(OptionsClassLocation[] locations, String pathP this.pathPrefix = pathPrefix; } + @VisibleForTesting + static OptionsClassLocation[] getLocations() { + return LOCATIONS; + } + public void discoverOptionsAndApply( Path rootDir, BiConsumerWithException, Collection, ? extends Exception> diff --git a/flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java b/flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java new file mode 100644 index 0000000000000..d12e6daccb989 --- /dev/null +++ b/flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.docs.util; + +import org.apache.flink.annotation.docs.Documentation; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link ConfigurationOptionLocator}. */ +class ConfigurationOptionLocatorTest { + + private static final String SOURCE_ROOT = "src/main/java"; + + private static final String SECTION_ANNOTATION = "@Documentation.Section"; + + private static final Set PRUNED_DIRECTORIES = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("target", "node_modules", ".git"))); + + /** Mirrors the file names {@link ConfigurationOptionLocator} recognizes. */ + private static final Pattern OPTIONS_CLASS_FILE_NAME = + Pattern.compile("[a-zA-Z]*(?:Options|Config|Parameters)\\.java"); + + /** + * Verifies that every option annotated with {@link Documentation.Section} sits in a package + * that {@link ConfigurationOptionLocator} actually searches. + * + *

The annotation is an explicit statement that the option belongs in the generated + * configuration reference, but discovery is driven by a hard-coded list of packages and does + * not recurse into sub-packages. An option outside that list is therefore dropped from the + * reference without any error, and {@code ConfigOptionsDocsCompletenessITCase} cannot catch it + * because it derives its expectations from the same list. + */ + @Test + void testSectionAnnotatedOptionsAreAllDiscoverable() throws IOException { + final Path rootDir = Paths.get(Utils.getProjectRootDir()).toAbsolutePath().normalize(); + + final Set searchedPackages = + Arrays.stream(ConfigurationOptionLocator.getLocations()) + .map( + location -> + location.getModule() + + '/' + + location.getPackage().replace('.', '/')) + .collect(Collectors.toSet()); + + final List undiscoverable = new ArrayList<>(); + for (Path optionsClass : findSectionAnnotatedOptionClasses(rootDir)) { + final String relativePath = toUnixPath(rootDir.relativize(optionsClass)); + final String modulePath = relativePath.substring(0, relativePath.indexOf(SOURCE_ROOT)); + final String packagePath = + relativePath.substring( + modulePath.length() + SOURCE_ROOT.length() + 1, + relativePath.lastIndexOf('/')); + + if (!searchedPackages.contains(modulePath + packagePath)) { + undiscoverable.add(relativePath); + } + } + + assertThat(undiscoverable) + .as( + "The options in these classes are annotated with @Documentation.Section but " + + "cannot be found by %s, so they are silently missing from the " + + "generated configuration reference. Add an %s entry for the " + + "containing package to %s#LOCATIONS.", + ConfigurationOptionLocator.class.getSimpleName(), + OptionsClassLocation.class.getSimpleName(), + ConfigurationOptionLocator.class.getSimpleName()) + .isEmpty(); + } + + private static List findSectionAnnotatedOptionClasses(Path rootDir) throws IOException { + final List optionClasses = new ArrayList<>(); + + Files.walkFileTree( + rootDir, + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory( + Path dir, BasicFileAttributes attributes) { + return PRUNED_DIRECTORIES.contains(dir.getFileName().toString()) + ? FileVisitResult.SKIP_SUBTREE + : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) + throws IOException { + if (OPTIONS_CLASS_FILE_NAME.matcher(file.getFileName().toString()).matches() + && toUnixPath(file).contains('/' + SOURCE_ROOT + '/') + && isSectionAnnotated(file)) { + optionClasses.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + + return optionClasses; + } + + private static boolean isSectionAnnotated(Path file) throws IOException { + try (Stream lines = Files.lines(file)) { + return lines.anyMatch(line -> line.contains(SECTION_ANNOTATION)); + } + } + + private static String toUnixPath(Path path) { + return path.toString().replace(path.getFileSystem().getSeparator(), "/"); + } +} diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java index c328dc5a589a4..81c7a7e77f025 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java @@ -18,6 +18,7 @@ package org.apache.flink.state.rocksdb.sstmerge; +import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.annotation.docs.Documentation; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; @@ -26,6 +27,7 @@ import java.time.Duration; /** Configuration options for manual compaction for the RocksDB backend. */ +@PublicEvolving public class RocksDBManualCompactionOptions { @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB) @@ -42,7 +44,7 @@ public class RocksDBManualCompactionOptions { .intType() .defaultValue(5) .withDescription( - "The maximum number of manual compactions to start." + "The maximum number of manual compactions to start. " + "Note that only one of them can run at a time as of v8.10.0; all the others will be waiting"); @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB) @@ -81,6 +83,6 @@ public class RocksDBManualCompactionOptions { .intType() .defaultValue(30) .withDescription( - "The maximum number of automatic compactions running for manual compaction to start." + "The maximum number of automatic compactions running for manual compaction to start. " + "If the actual number is higher, manual compaction won't be started to avoid delaying automatic ones."); } From 1b1258e3d8992846f07b9401251ecd8263884013 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:43:37 +0800 Subject: [PATCH 06/34] [hotfix][python] Bump soupsieve from 2.8.3 to 2.8.4 (#28812) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- flink-python/docs/uv.lock | 76 +++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/flink-python/docs/uv.lock b/flink-python/docs/uv.lock index 52702f9460da4..5c0333bbb98fa 100644 --- a/flink-python/docs/uv.lock +++ b/flink-python/docs/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ [[package]] name = "alabaster" version = "0.7.16" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c9/3e/13dd8e5ed9094e734ac430b5d0eb4f2bb001708a8b7856cbf8e084e001ba/alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65", size = 23776, upload-time = "2024-01-10T00:56:10.189Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/32/34/d4e1c02d3bee589efb5dfa17f88ea08bdb3e3eac12bc475462aec52ed223/alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92", size = 13511, upload-time = "2024-01-10T00:56:08.388Z" }, @@ -18,7 +18,7 @@ wheels = [ [[package]] name = "babel" version = "2.18.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, @@ -27,7 +27,7 @@ wheels = [ [[package]] name = "beautifulsoup4" version = "4.14.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "soupsieve" }, { name = "typing-extensions" }, @@ -40,7 +40,7 @@ wheels = [ [[package]] name = "certifi" version = "2026.1.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, @@ -49,7 +49,7 @@ wheels = [ [[package]] name = "charset-normalizer" version = "3.4.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1f/b8/6d51fc1d52cbd52cd4ccedd5b5b2f0f6a11bbf6765c782298b0f3e808541/charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d", size = 209709, upload-time = "2025-10-14T04:40:11.385Z" }, @@ -154,7 +154,7 @@ wheels = [ [[package]] name = "click" version = "8.1.8" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.10'", ] @@ -169,7 +169,7 @@ wheels = [ [[package]] name = "click" version = "8.3.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.10'", ] @@ -184,7 +184,7 @@ wheels = [ [[package]] name = "colorama" version = "0.4.6" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, @@ -193,7 +193,7 @@ wheels = [ [[package]] name = "docutils" version = "0.17.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/4c/17/559b4d020f4b46e0287a2eddf2d8ebf76318fd3bd495f1625414b052fdc9/docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125", size = 2016138, upload-time = "2021-04-17T14:13:28.434Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4c/5e/6003a0d1f37725ec2ebd4046b657abb9372202655f96e76795dca8c0063c/docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61", size = 575533, upload-time = "2021-04-17T14:13:24.796Z" }, @@ -202,7 +202,7 @@ wheels = [ [[package]] name = "idna" version = "3.11" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, @@ -211,7 +211,7 @@ wheels = [ [[package]] name = "imagesize" version = "1.4.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a", size = 1280026, upload-time = "2022-07-01T12:21:05.687Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, @@ -220,7 +220,7 @@ wheels = [ [[package]] name = "importlib-metadata" version = "4.4.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] @@ -232,7 +232,7 @@ wheels = [ [[package]] name = "jinja2" version = "3.0.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markupsafe" }, ] @@ -244,7 +244,7 @@ wheels = [ [[package]] name = "markupsafe" version = "3.0.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, @@ -340,7 +340,7 @@ wheels = [ [[package]] name = "mistune" version = "2.0.5" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/fb/6b/d8013058fcdb0088b4130164fc961e15c50d30302f60a349c16bdfda9770/mistune-2.0.5.tar.gz", hash = "sha256:0246113cb2492db875c6be56974a7c893333bf26cd92891c85f63151cee09d34", size = 75854, upload-time = "2023-02-07T05:42:06.739Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9f/e5/780d22d19543f339aad583304f58002975b586757aa590cbe7bea5cc6f13/mistune-2.0.5-py2.py3-none-any.whl", hash = "sha256:bad7f5d431886fcbaf5f758118ecff70d31f75231b34024a1341120340a65ce8", size = 24549, upload-time = "2023-02-07T05:42:05.089Z" }, @@ -349,7 +349,7 @@ wheels = [ [[package]] name = "packaging" version = "26.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, @@ -358,7 +358,7 @@ wheels = [ [[package]] name = "pydata-sphinx-theme" version = "0.11.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "beautifulsoup4" }, { name = "docutils" }, @@ -413,7 +413,7 @@ dev = [ [[package]] name = "pygments" version = "2.19.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, @@ -422,7 +422,7 @@ wheels = [ [[package]] name = "requests" version = "2.32.5" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, @@ -437,7 +437,7 @@ wheels = [ [[package]] name = "snowballstemmer" version = "3.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, @@ -445,17 +445,17 @@ wheels = [ [[package]] name = "soupsieve" -version = "2.8.3" -source = { registry = "https://pypi.org/simple/" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } +version = "2.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, + { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] [[package]] name = "sphinx" version = "4.5.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alabaster" }, { name = "babel" }, @@ -483,11 +483,11 @@ wheels = [ [[package]] name = "sphinx-intl" version = "2.3.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "babel" }, - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple/" }, marker = "python_full_version < '3.10'" }, - { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple/" }, marker = "python_full_version >= '3.10'" }, + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "sphinx" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/21/eb12016ecb0b52861762b0d227dff75622988f238776a5ee4c75bade507e/sphinx_intl-2.3.2.tar.gz", hash = "sha256:04b0d8ea04d111a7ba278b17b7b3fe9625c58b6f8ffb78bb8a1dd1288d88c1c7", size = 27921, upload-time = "2025-08-02T04:53:01.891Z" } @@ -498,7 +498,7 @@ wheels = [ [[package]] name = "sphinx-mdinclude" version = "0.5.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docutils" }, { name = "mistune" }, @@ -512,7 +512,7 @@ wheels = [ [[package]] name = "sphinxcontrib-applehelp" version = "1.0.4" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/32/df/45e827f4d7e7fcc84e853bcef1d836effd762d63ccb86f43ede4e98b478c/sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e", size = 24766, upload-time = "2023-01-23T09:41:54.435Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/06/c1/5e2cafbd03105ce50d8500f9b4e8a6e8d02e22d0475b574c3b3e9451a15f/sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228", size = 120601, upload-time = "2023-01-23T09:41:52.364Z" }, @@ -521,7 +521,7 @@ wheels = [ [[package]] name = "sphinxcontrib-devhelp" version = "1.0.2" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/98/33/dc28393f16385f722c893cb55539c641c9aaec8d1bc1c15b69ce0ac2dbb3/sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4", size = 17398, upload-time = "2020-02-29T04:14:43.378Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c5/09/5de5ed43a521387f18bdf5f5af31d099605c992fd25372b2b9b825ce48ee/sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", size = 84690, upload-time = "2020-02-29T04:14:40.765Z" }, @@ -530,7 +530,7 @@ wheels = [ [[package]] name = "sphinxcontrib-htmlhelp" version = "2.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b3/47/64cff68ea3aa450c373301e5bebfbb9fce0a3e70aca245fcadd4af06cd75/sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff", size = 27967, upload-time = "2023-01-31T17:29:20.935Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6e/ee/a1f5e39046cbb5f8bc8fba87d1ddf1c6643fbc9194e58d26e606de4b9074/sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903", size = 99833, upload-time = "2023-01-31T17:29:18.489Z" }, @@ -539,7 +539,7 @@ wheels = [ [[package]] name = "sphinxcontrib-jsmath" version = "1.0.1" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, @@ -548,7 +548,7 @@ wheels = [ [[package]] name = "sphinxcontrib-qthelp" version = "1.0.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b1/8e/c4846e59f38a5f2b4a0e3b27af38f2fcf904d4bfd82095bf92de0b114ebd/sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", size = 21658, upload-time = "2020-02-29T04:19:10.026Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2b/14/05f9206cf4e9cfca1afb5fd224c7cd434dcc3a433d6d9e4e0264d29c6cdb/sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6", size = 90609, upload-time = "2020-02-29T04:19:08.451Z" }, @@ -557,7 +557,7 @@ wheels = [ [[package]] name = "sphinxcontrib-serializinghtml" version = "1.1.5" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b5/72/835d6fadb9e5d02304cf39b18f93d227cd93abd3c41ebf58e6853eeb1455/sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952", size = 21019, upload-time = "2021-05-22T16:07:43.043Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c6/77/5464ec50dd0f1c1037e3c93249b040c8fc8078fdda97530eeb02424b6eea/sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", size = 94021, upload-time = "2021-05-22T16:07:41.627Z" }, @@ -566,7 +566,7 @@ wheels = [ [[package]] name = "typing-extensions" version = "4.15.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, @@ -575,7 +575,7 @@ wheels = [ [[package]] name = "urllib3" version = "2.6.3" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, @@ -584,7 +584,7 @@ wheels = [ [[package]] name = "zipp" version = "3.23.0" -source = { registry = "https://pypi.org/simple/" } +source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, From ea801695739a4238cf2edfec1ff855100cb1e58b Mon Sep 17 00:00:00 2001 From: Nikolaus Schuetz Date: Mon, 3 Aug 2026 02:06:33 -0700 Subject: [PATCH 07/34] [FLINK-40236][python] Fix _infer_type inferring array element type with a leading None (#28819) _infer_type inferred a list's element type from obj[0] rather than the first non-None element the loop scans for, so a leading None collapsed the array element type to NULL. Infer from the scanned element v instead, matching the dict branch above. --- flink-python/pyflink/table/tests/test_types.py | 5 +++++ flink-python/pyflink/table/types.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-python/pyflink/table/tests/test_types.py b/flink-python/pyflink/table/tests/test_types.py index f8124ba5e8cdf..9c7a47556cc95 100644 --- a/flink-python/pyflink/table/tests/test_types.py +++ b/flink-python/pyflink/table/tests/test_types.py @@ -221,6 +221,11 @@ def test_infer_schema_nulltype(self): # third column is varchar self.assertTrue(isinstance(schema.fields[2].data_type, VarCharType)) + def test_infer_array_type_with_leading_none(self): + data_type = _infer_type([None, 1]) + self.assertTrue(isinstance(data_type, ArrayType)) + self.assertTrue(isinstance(data_type.element_type, BigIntType)) + def test_infer_schema_not_enough_names(self): schema = _infer_schema_from_data([["a", "b"]], ["col1"]) self.assertTrue(schema.names, ['col1', '_2']) diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index 964097f25e272..408dddfa6f7fd 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -1500,7 +1500,7 @@ def _infer_type(obj): elif isinstance(obj, list): for v in obj: if v is not None: - return ArrayType(_infer_type(obj[0])) + return ArrayType(_infer_type(v)) else: return ArrayType(NullType()) elif isinstance(obj, array): From 12197ea92a5667073bc0c6810e526a39979d835c Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 13:03:12 +0200 Subject: [PATCH 08/34] [FLINK-40285][table] `MLPredictSemanticTests` fails because of `ON CONFLICT` --- .../exec/stream/MLPredictTestPrograms.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/MLPredictTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/MLPredictTestPrograms.java index 26d4903a3e41d..205aa672e25d6 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/MLPredictTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/MLPredictTestPrograms.java @@ -20,6 +20,7 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Expressions; +import org.apache.flink.table.api.InsertConflictStrategy; import org.apache.flink.table.api.ModelDescriptor; import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.config.ExecutionConfigOptions; @@ -182,7 +183,8 @@ public class MLPredictTestPrograms { env.from("features").asArgument("INPUT"), env.fromModel("chatgpt").asArgument("MODEL"), descriptor("feature").asArgument("ARGS")), - "sink") + "sink", + InsertConflictStrategy.deduplicate()) .build(); public static final TableTestProgram ASYNC_ML_PREDICT_TABLE_API = @@ -209,7 +211,8 @@ public class MLPredictTestPrograms { DataTypes.STRING()) .notNull()) .asArgument("CONFIG")), - "sink") + "sink", + InsertConflictStrategy.deduplicate()) .build(); public static final TableTestProgram ASYNC_ML_PREDICT_TABLE_API_MAP_EXPRESSION_CONFIG = @@ -235,7 +238,8 @@ public class MLPredictTestPrograms { "max-concurrent-operations", "10") .asArgument("CONFIG")), - "sink") + "sink", + InsertConflictStrategy.deduplicate()) .build(); public static final TableTestProgram ML_PREDICT_MODEL_API = @@ -248,7 +252,8 @@ public class MLPredictTestPrograms { env.fromModel("chatgpt") .predict( env.from("features"), ColumnList.of("feature")), - "sink") + "sink", + InsertConflictStrategy.deduplicate()) .build(); public static final TableTestProgram ASYNC_ML_PREDICT_MODEL_API = @@ -270,7 +275,8 @@ public class MLPredictTestPrograms { "true", "max-concurrent-operations", "10")), - "sink") + "sink", + InsertConflictStrategy.deduplicate()) .build(); public static final TableTestProgram ML_PREDICT_ANON_MODEL_API = @@ -304,6 +310,7 @@ public class MLPredictTestPrograms { .build()) .predict( env.from("features"), ColumnList.of("feature")), - "sink") + "sink", + InsertConflictStrategy.deduplicate()) .build(); } From ae9449e6bb502587429cc554b489f89f2a64b189 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 20:13:56 +0200 Subject: [PATCH 09/34] [FLINK-40317][tests] Make DeletesByKeySemanticTests more stable --- .../exec/stream/DeletesByKeyPrograms.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeletesByKeyPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeletesByKeyPrograms.java index 6599f64995055..a66837d1c4542 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeletesByKeyPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/DeletesByKeyPrograms.java @@ -237,13 +237,8 @@ public final class DeletesByKeyPrograms { "`value` INT") .addOption("changelog-mode", "I,UA,D") .addOption("sink.supports-delete-by-key", "false") - .consumedValues( - "+I[1, Alice, 10]", - "+I[2, Bob, 20]", - "+I[3, Emily, 30]", - "-D[1, Alice, 10]", - "+U[3, Emily, 40]", - "+U[2, BOB, 20]") + .testMaterializedData() + .consumedValues("+I[3, Emily, 40]", "+I[2, BOB, 20]") .build()) .runSql( "INSERT INTO sink_t SELECT l.id, r.name, l.`value` FROM left_t l JOIN right_t r ON l.id = r.id") @@ -291,13 +286,8 @@ public final class DeletesByKeyPrograms { "`value` INT") .addOption("changelog-mode", "I,UA,D") .addOption("sink.supports-delete-by-key", "true") - .consumedValues( - "+I[1, Alice, 10]", - "+I[2, Bob, 20]", - "+I[3, Emily, 30]", - "-D[1, Alice, null]", - "+U[3, Emily, 40]", - "+U[2, BOB, 20]") + .testMaterializedData() + .consumedValues("+I[2, BOB, 20]", "+I[3, Emily, 40]") .build()) .runSql( "INSERT INTO sink_t SELECT l.id, r.name, l.`value` FROM left_t l JOIN right_t r ON l.id = r.id") From 17c70e8f7ba969c63075b15fed4a07c6b6b4839b Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 22:25:00 +0200 Subject: [PATCH 10/34] [hotfix][ci] Add ignore pattern in case of Azure --- tools/azure-pipelines/build_properties.sh | 18 ++++++++++++------ tools/azure-pipelines/e2e-template.yml | 14 +++++++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/tools/azure-pipelines/build_properties.sh b/tools/azure-pipelines/build_properties.sh index 3ab34d3daa0e0..72ec892c37766 100755 --- a/tools/azure-pipelines/build_properties.sh +++ b/tools/azure-pipelines/build_properties.sh @@ -45,18 +45,24 @@ function github_num_commits() { return $GITHUB_NUM_COMMITS } -# -# Returns 0 if the change is a documentation-only pull request -# -function is_docs_only_pullrequest() { +# Returns 0 if the change only touches ignored paths (IGNORE_PATHS pipeline variable, newline-separated pathspec globs), e.g. a documentation-only pull request. +function is_ignored_pattern_pullrequest() { github_num_commits GITHUB_NUM_COMMITS=$? if [[ $GITHUB_NUM_COMMITS == 0 ]]; then return 1 fi - if [[ $(git diff --name-only HEAD..HEAD~$GITHUB_NUM_COMMITS | grep -v "docs/") == "" ]] ; then - echo "INFO: This is a docs only change. Changed files:" + # A diff that lists nothing after the git excludes means only ignored paths changed. + local exclude_pathspecs=() + local pattern + while IFS= read -r pattern; do + [[ -z "$pattern" ]] && continue + exclude_pathspecs+=(":(exclude,glob)$pattern") + done <<< "$IGNORE_PATHS" + + if [[ $(git diff --name-only HEAD..HEAD~$GITHUB_NUM_COMMITS -- . "${exclude_pathspecs[@]}") == "" ]] ; then + echo "INFO: Only ignored paths changed ($IGNORE_PATHS). Changed files:" git diff --name-only HEAD..HEAD~$GITHUB_NUM_COMMITS return 0 fi diff --git a/tools/azure-pipelines/e2e-template.yml b/tools/azure-pipelines/e2e-template.yml index 8045525156c6c..dcffa9b36a072 100644 --- a/tools/azure-pipelines/e2e-template.yml +++ b/tools/azure-pipelines/e2e-template.yml @@ -31,11 +31,23 @@ jobs: cancelTimeoutInMinutes: 1 workspace: clean: all + variables: + # Newline-separated git pathspec globs (GitHub Actions "paths-ignore" style); PRs touching only these skip e2e. + IGNORE_PATHS: | + docs/** + **/*.md + .idea/** + .asf.yaml + .editorconfig + .git-blame-ignore-revs + .gitignore + .gitattributes + .github/** steps: # Skip e2e test execution if this is a documentation only pull request (master / release builds will still be checked regularly) - bash: | source ./tools/azure-pipelines/build_properties.sh - is_docs_only_pullrequest + is_ignored_pattern_pullrequest if [[ "$?" == 0 ]] ; then echo "##[debug]This is a documentation-only change. Skipping e2e execution." echo "##vso[task.setvariable variable=skip;]1" From bd1becc5177482c17ce5ddbdb9fa2335f7cad853 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 12:23:38 +0200 Subject: [PATCH 11/34] [FLINK-40311][tests] Rename `EndiannessAccessChecks` to `EndiannessAccessChecksTest` --- ...iannessAccessChecks.java => EndiannessAccessChecksTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename flink-core/src/test/java/org/apache/flink/core/memory/{EndiannessAccessChecks.java => EndiannessAccessChecksTest.java} (99%) diff --git a/flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecks.java b/flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecksTest.java similarity index 99% rename from flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecks.java rename to flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecksTest.java index 053c895cac9bb..06b3fe78fb7c0 100644 --- a/flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecks.java +++ b/flink-core/src/test/java/org/apache/flink/core/memory/EndiannessAccessChecksTest.java @@ -29,7 +29,7 @@ * Verifies correct accesses with regards to endianness in {@link MemorySegment} (in both heap and * off-heap modes). */ -class EndiannessAccessChecks { +class EndiannessAccessChecksTest { @Test void testOnHeapSegment() { From e700705ac7b84a954362c7f6bb1fe5e62973de06 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 12:25:05 +0200 Subject: [PATCH 12/34] [FLINK-40310][tests] Rename `CsvBulkWriterIT` to `CsvBulkWriterTest` --- ...vBulkWriterIT.java => CsvBulkWriterITCase.java} | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) rename flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/{CsvBulkWriterIT.java => CsvBulkWriterITCase.java} (92%) diff --git a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterIT.java b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterITCase.java similarity index 92% rename from flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterIT.java rename to flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterITCase.java index aa070b22abad5..349c7c9461da4 100644 --- a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterIT.java +++ b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvBulkWriterITCase.java @@ -27,10 +27,13 @@ import org.apache.flink.connector.datagen.source.TestDataGenerators; import org.apache.flink.connector.file.sink.FileSink; import org.apache.flink.core.fs.FSDataOutputStream; +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.test.junit5.MiniClusterExtension; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; import java.io.File; @@ -44,7 +47,14 @@ import static org.assertj.core.api.Assertions.assertThat; -public class CsvBulkWriterIT { +class CsvBulkWriterITCase { + + @RegisterExtension + static final MiniClusterExtension MINI_CLUSTER = + new MiniClusterExtension( + new MiniClusterResourceConfiguration.Builder() + .setNumberTaskManagers(2) + .build()); @TempDir File outDir; @@ -53,7 +63,7 @@ public class CsvBulkWriterIT { * flush signal from Flink. */ @Test - public void testNoDataIsWrittenBeforeFlinkFlush() throws Exception { + void testNoDataIsWrittenBeforeFlinkFlush() throws Exception { Configuration config = new Configuration(); config.set( From a357106be6c4daa976f93e60fbc7dea147bd265c Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 12:39:58 +0200 Subject: [PATCH 13/34] [FLINK-40307][table] `RestoreTestCompleteness` was never executed in CI --- .../batch/SortAggregateBatchRestoreTest.java | 2 +- .../stream/AsyncCorrelateRestoreTest.java | 2 +- .../ProcessTableFunctionRestoreTests.java | 2 +- .../stream/WatermarkAssignerRestoreTest.java | 2 +- ....java => RestoreTestCompletenessTest.java} | 78 ++++++++++++------ .../plan/async-correlate-catalog-func.json | 0 .../savepoint/_metadata | Bin .../plan/async-correlate-exception.json | 0 .../savepoint/_metadata | Bin .../plan/async-correlate-join-filter.json | 0 .../savepoint/_metadata | Bin .../plan/async-correlate-left-join.json | 0 .../savepoint/_metadata | Bin .../plan/async-correlate-system-func.json | 0 .../savepoint/_metadata | Bin 15 files changed, 57 insertions(+), 29 deletions(-) rename flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/{RestoreTestCompleteness.java => RestoreTestCompletenessTest.java} (65%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-catalog-func/plan/async-correlate-catalog-func.json (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-catalog-func/savepoint/_metadata (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-exception/plan/async-correlate-exception.json (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-exception/savepoint/_metadata (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-join-filter/plan/async-correlate-join-filter.json (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-join-filter/savepoint/_metadata (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-left-join/plan/async-correlate-left-join.json (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-left-join/savepoint/_metadata (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-system-func/plan/async-correlate-system-func.json (100%) rename flink-table/flink-table-planner/src/test/resources/restore-tests/{stream-exec-correlate_1 => stream-exec-async-correlate_1}/async-correlate-system-func/savepoint/_metadata (100%) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/SortAggregateBatchRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/SortAggregateBatchRestoreTest.java index d329a2d04ecb1..a4581a32cea8e 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/SortAggregateBatchRestoreTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/SortAggregateBatchRestoreTest.java @@ -24,7 +24,7 @@ import java.util.List; /** Batch Compiled Plan tests for {@link BatchExecSortAggregate}. */ -class SortAggregateBatchRestoreTest extends BatchRestoreTestBase { +public class SortAggregateBatchRestoreTest extends BatchRestoreTestBase { public SortAggregateBatchRestoreTest() { super(BatchExecSortAggregate.class); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/AsyncCorrelateRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/AsyncCorrelateRestoreTest.java index 2d52118b22526..6d8214e1caa8c 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/AsyncCorrelateRestoreTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/AsyncCorrelateRestoreTest.java @@ -28,7 +28,7 @@ public class AsyncCorrelateRestoreTest extends RestoreTestBase { public AsyncCorrelateRestoreTest() { - super(StreamExecCorrelate.class); + super(StreamExecAsyncCorrelate.class); } @Override diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionRestoreTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionRestoreTests.java index 03ef42de0e142..caccbfc02190d 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionRestoreTests.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionRestoreTests.java @@ -26,7 +26,7 @@ /** Restore tests for {@link StreamExecProcessTableFunction}. */ public class ProcessTableFunctionRestoreTests extends RestoreTestBase { - protected ProcessTableFunctionRestoreTests() { + public ProcessTableFunctionRestoreTests() { super(StreamExecProcessTableFunction.class); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WatermarkAssignerRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WatermarkAssignerRestoreTest.java index 0a684e4c13360..3d63a1523e14e 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WatermarkAssignerRestoreTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WatermarkAssignerRestoreTest.java @@ -24,7 +24,7 @@ import java.util.List; /** Restore tests for {@link StreamExecWatermarkAssigner}. */ -class WatermarkAssignerRestoreTest extends RestoreTestBase { +public class WatermarkAssignerRestoreTest extends RestoreTestBase { public WatermarkAssignerRestoreTest() { super(StreamExecWatermarkAssigner.class); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompletenessTest.java similarity index 65% rename from flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java rename to flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompletenessTest.java index ea183bf3dbad3..ec8cff2bddf0f 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompletenessTest.java @@ -19,6 +19,10 @@ package org.apache.flink.table.planner.plan.nodes.exec.testutils; import org.apache.flink.table.planner.plan.nodes.exec.ExecNode; +import org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecHashAggregate; +import org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecNestedLoopJoin; +import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecGlobalWindowAggregate; +import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLocalWindowAggregate; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonAsyncCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCorrelate; @@ -31,7 +35,6 @@ import org.apache.flink.shaded.guava33.com.google.common.reflect.ClassPath; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -43,22 +46,30 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.fail; + /** Validate restore tests exists for Exec Nodes. */ -public class RestoreTestCompleteness { +class RestoreTestCompletenessTest { private static final Set>> SKIP_EXEC_NODES = - new HashSet>>() { - { - /** Ignoring python based exec nodes temporarily. */ - add(StreamExecPythonCalc.class); - add(StreamExecPythonCorrelate.class); - add(StreamExecPythonOverAggregate.class); - add(StreamExecPythonGroupAggregate.class); - add(StreamExecPythonGroupTableAggregate.class); - add(StreamExecPythonGroupWindowAggregate.class); - add(StreamExecPythonAsyncCalc.class); - } - }; + Set.of( + /* Ignoring python based exec nodes temporarily. */ + StreamExecPythonCalc.class, + StreamExecPythonCorrelate.class, + StreamExecPythonOverAggregate.class, + StreamExecPythonGroupAggregate.class, + StreamExecPythonGroupTableAggregate.class, + StreamExecPythonGroupWindowAggregate.class, + StreamExecPythonAsyncCalc.class, + + // Covered by tests in WindowAggregateEventTimeRestoreTest + StreamExecLocalWindowAggregate.class, + StreamExecGlobalWindowAggregate.class, + + // There is jira for these 2 batch tests + // https://issues.apache.org/jira/browse/FLINK-40306 + BatchExecHashAggregate.class, + BatchExecNestedLoopJoin.class); private Class> getExecNode(Class restoreTest) throws NoSuchMethodException, @@ -87,7 +98,7 @@ private List>> getChildExecNodes(Class restoreTes } @Test - public void testMissingRestoreTest() + void testMissingRestoreTest() throws IOException, NoSuchMethodException, InstantiationException, @@ -97,12 +108,14 @@ public void testMissingRestoreTest() ExecNodeMetadataUtil.getVersionedExecNodes(); Set classesInPackage = - ClassPath.from(this.getClass().getClassLoader()) - .getTopLevelClassesRecursive( - "org.apache.flink.table.planner.plan.nodes.exec.stream") - .stream() - .filter(x -> RestoreTestBase.class.isAssignableFrom(x.load())) - .collect(Collectors.toSet()); + new HashSet<>( + gatherClasses( + RestoreTestBase.class, + "org.apache.flink.table.planner.plan.nodes.exec.stream")); + classesInPackage.addAll( + gatherClasses( + BatchRestoreTestBase.class, + "org.apache.flink.table.planner.plan.nodes.exec.batch")); Set>> execNodesWithRestoreTests = new HashSet<>(); @@ -118,18 +131,33 @@ public void testMissingRestoreTest() } } + Set>> productionExecNodes = ExecNodeMetadataUtil.execNodes(); for (Map.Entry>> entry : versionedExecNodes.entrySet()) { ExecNodeNameVersion execNodeNameVersion = entry.getKey(); Class> execNode = entry.getValue(); - if (!SKIP_EXEC_NODES.contains(execNode)) { - final String msg = + // Ignore test-only nodes that other tests leak into the shared LOOKUP_MAP via + // addTestNode(). + if (!productionExecNodes.contains(execNode)) { + continue; + } + if (!SKIP_EXEC_NODES.contains(execNode) + && !execNodesWithRestoreTests.contains(execNode)) { + fail( "Missing restore test for " + execNodeNameVersion + "\nPlease add a restore test for " - + execNode.toString(); - Assertions.assertTrue(execNodesWithRestoreTests.contains(execNode), msg); + + execNode.toString()); } } } + + private Set gatherClasses(Class clazz, String packageName) + throws IOException { + return ClassPath.from(this.getClass().getClassLoader()) + .getTopLevelClassesRecursive(packageName) + .stream() + .filter(x -> clazz.isAssignableFrom(x.load())) + .collect(Collectors.toSet()); + } } diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-catalog-func/plan/async-correlate-catalog-func.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-catalog-func/plan/async-correlate-catalog-func.json similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-catalog-func/plan/async-correlate-catalog-func.json rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-catalog-func/plan/async-correlate-catalog-func.json diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-catalog-func/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-catalog-func/savepoint/_metadata similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-catalog-func/savepoint/_metadata rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-catalog-func/savepoint/_metadata diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-exception/plan/async-correlate-exception.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-exception/plan/async-correlate-exception.json similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-exception/plan/async-correlate-exception.json rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-exception/plan/async-correlate-exception.json diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-exception/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-exception/savepoint/_metadata similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-exception/savepoint/_metadata rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-exception/savepoint/_metadata diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-join-filter/plan/async-correlate-join-filter.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-join-filter/plan/async-correlate-join-filter.json similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-join-filter/plan/async-correlate-join-filter.json rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-join-filter/plan/async-correlate-join-filter.json diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-join-filter/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-join-filter/savepoint/_metadata similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-join-filter/savepoint/_metadata rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-join-filter/savepoint/_metadata diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-left-join/plan/async-correlate-left-join.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-left-join/plan/async-correlate-left-join.json similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-left-join/plan/async-correlate-left-join.json rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-left-join/plan/async-correlate-left-join.json diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-left-join/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-left-join/savepoint/_metadata similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-left-join/savepoint/_metadata rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-left-join/savepoint/_metadata diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-system-func/plan/async-correlate-system-func.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-system-func/plan/async-correlate-system-func.json similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-system-func/plan/async-correlate-system-func.json rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-system-func/plan/async-correlate-system-func.json diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-system-func/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-system-func/savepoint/_metadata similarity index 100% rename from flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-correlate_1/async-correlate-system-func/savepoint/_metadata rename to flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-async-correlate_1/async-correlate-system-func/savepoint/_metadata From 2111b41ef1895c2d817d2a073f1159b4c638b599 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 12:46:46 +0200 Subject: [PATCH 14/34] [FLINK-40284][tests] Make tests ending with `Tests` executing in CI --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 82e5475e92a35..b67166326c562 100644 --- a/pom.xml +++ b/pom.xml @@ -221,7 +221,7 @@ under the License. 256 1.0 - **/*Test.* + **/*Test.*,**/*Tests.* 1.1.10.7 3.18.0 8.10.0-ververica-1.0 From 6d8ce98dd3ec1f1eeaa211a673d64f65439a86e7 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Mon, 3 Aug 2026 12:35:20 +0200 Subject: [PATCH 15/34] [FLINK-40284][tests] Archunit should fail in case of tests not matching name requirements This closes #28887. --- .../TestCodeArchitectureTestBase.java | 3 + .../architecture/rules/TestNamingRules.java | 86 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/rules/TestNamingRules.java diff --git a/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/TestCodeArchitectureTestBase.java b/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/TestCodeArchitectureTestBase.java index a33ad9b6d6653..9b7984d002b75 100644 --- a/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/TestCodeArchitectureTestBase.java +++ b/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/TestCodeArchitectureTestBase.java @@ -19,6 +19,7 @@ package org.apache.flink.architecture; import org.apache.flink.architecture.rules.ITCaseRules; +import org.apache.flink.architecture.rules.TestNamingRules; import com.tngtech.archunit.junit.ArchTest; import com.tngtech.archunit.junit.ArchTests; @@ -33,4 +34,6 @@ public class TestCodeArchitectureTestBase { @ArchTest public static final ArchTests ITCASE = ArchTests.in(ITCaseRules.class); + + @ArchTest public static final ArchTests TEST_NAMING = ArchTests.in(TestNamingRules.class); } diff --git a/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/rules/TestNamingRules.java b/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/rules/TestNamingRules.java new file mode 100644 index 0000000000000..79810fa043f72 --- /dev/null +++ b/flink-architecture-tests/flink-architecture-tests-test/src/main/java/org/apache/flink/architecture/rules/TestNamingRules.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.architecture.rules; + +import com.tngtech.archunit.base.DescribedPredicate; +import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.junit.ArchTest; +import com.tngtech.archunit.lang.ArchRule; + +import java.util.Arrays; +import java.util.List; + +import static com.tngtech.archunit.core.domain.JavaModifier.ABSTRACT; +import static org.apache.flink.architecture.common.GivenJavaClasses.javaClassesThat; + +/** + * Rules ensuring executable test classes are named so the build actually runs them. + * + *

Surefire only runs the unit include pattern {@code **}{@code /*Test.*} in the {@code test} + * phase; integration tests follow the {@code *ITCase} convention. A concrete class that carries (or + * inherits) JUnit test methods but is named otherwise (e.g. {@code *Tests}) is silently skipped by + * the unit run. This rule flags such classes so they are renamed to {@code *Test} or {@code + * *ITCase}. + */ +public class TestNamingRules { + + /** JUnit 5 and (for modules still mid-migration) JUnit 4 test method annotations. */ + private static final List TEST_METHOD_ANNOTATIONS = + Arrays.asList( + "org.junit.jupiter.api.Test", + "org.junit.jupiter.api.TestTemplate", + "org.junit.jupiter.api.RepeatedTest", + "org.junit.jupiter.api.TestFactory", + "org.junit.jupiter.params.ParameterizedTest", + "org.junit.Test"); + + /** + * A class JUnit would execute: it declares or inherits a test method. {@code getAllMethods()} + * covers inherited {@code @TestTemplate} methods, e.g. semantic-test suites that only extend a + * base and add no annotation themselves. + */ + private static final DescribedPredicate ARE_EXECUTABLE_TEST_CLASSES = + DescribedPredicate.describe( + "are executable JUnit test classes", + clazz -> + clazz.getAllMethods().stream() + .anyMatch( + method -> + TEST_METHOD_ANNOTATIONS.stream() + .anyMatch(method::isAnnotatedWith))); + + @ArchTest + public static final ArchRule TEST_CLASSES_SHOULD_BE_NAMED_TEST_OR_ITCASE = + javaClassesThat() + .areTopLevelClasses() + .and() + .doNotHaveModifier(ABSTRACT) + .and(ARE_EXECUTABLE_TEST_CLASSES) + .should() + .haveSimpleNameEndingWith("Test") + .orShould() + .haveSimpleNameEndingWith("Tests") + .orShould() + .haveSimpleNameEndingWith("ITCase") + // not every module has such classes + .allowEmptyShould(true) + .as( + "Executable test classes must be named *Test[s] or *ITCase so the surefire " + + "include pattern runs them"); +} From 3c8740f3f24aaa407112b5b576e5bfbf9cf2d24a Mon Sep 17 00:00:00 2001 From: Roman Khachatryan Date: Fri, 31 Jul 2026 20:33:31 +0200 Subject: [PATCH 16/34] [hotfix][docs] Add code review guidelines to AGENTS.md --- AGENTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index dac317aae91e3..5a9b127f925e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -305,6 +305,15 @@ This section maps common types of Flink changes to the modules they touch and th - You must be able to explain the design, code, and tests, debug them, and respond to review feedback substantively - Reviewer-ready quality bar: the author owns PR quality. PRs that look AI-generated without author refinement (walls of unreviewed prose, scaffolding without behaviour, tests that do not exercise the change, padded commit messages) will be closed without review +## Code Review Guidelines + +When reviewing a PR or diff against this repo: + +- Look for opportunities to simplify the code, scoped to the diff itself (not pre-existing code outside the change). +- Flag comments that are obvious (restate what the code already says) or overly verbose. +- In test code, look for potential flakiness — e.g. `Thread.sleep` used outside a retry/poll loop, or similar timing-dependent, non-deterministic patterns. Where applicable, suggest clock injection (e.g. a manually-advanced `Clock`/`ManualClock`) instead of relying on wall-clock time, or waiting for the actual condition in a loop with a timeout, for deterministic tests. +- Check that each commit message conforms to Flink conventions: it must start with `[FLINK-XXXX]` or `[hotfix]`, and must specify a subsystem/component (e.g. `[FLINK-XXXX][runtime] Description`). + ## Boundaries ### Ask first From c1133c338d0a94431ff7d2e3747a8c22ade05075 Mon Sep 17 00:00:00 2001 From: Nikolaus Schuetz Date: Wed, 5 Aug 2026 00:07:27 -0700 Subject: [PATCH 17/34] [FLINK-40322][python] Fix Array/Multiset from_sql_type to decode elements (#28912) --- flink-python/pyflink/table/tests/test_types.py | 12 ++++++++++++ flink-python/pyflink/table/types.py | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/flink-python/pyflink/table/tests/test_types.py b/flink-python/pyflink/table/tests/test_types.py index 9c7a47556cc95..400a3729ee921 100644 --- a/flink-python/pyflink/table/tests/test_types.py +++ b/flink-python/pyflink/table/tests/test_types.py @@ -549,6 +549,18 @@ def test_datetype_equal_zero(self): dt = DataTypes.DATE() self.assertEqual(dt.from_sql_type(0), datetime.date(1970, 1, 1)) + def test_array_from_sql_type_converts_elements(self): + at = DataTypes.ARRAY(DataTypes.DATE()) + self.assertEqual( + at.from_sql_type([0, 1]), + [datetime.date(1970, 1, 1), datetime.date(1970, 1, 2)]) + + def test_multiset_from_sql_type_converts_elements(self): + mst = DataTypes.MULTISET(DataTypes.DATE()) + self.assertEqual( + mst.from_sql_type([0, 1]), + [datetime.date(1970, 1, 1), datetime.date(1970, 1, 2)]) + @unittest.skipIf(on_windows(), "Windows x64 system only support the datetime not larger " "than time.ctime(32536799999), so this test can't run " "under Windows platform") diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index 408dddfa6f7fd..316bb44b55e23 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -931,7 +931,7 @@ def to_sql_type(self, obj): def from_sql_type(self, obj): if not self.need_conversion(): return obj - return obj and [self.element_type.to_sql_type(v) for v in obj] + return obj and [self.element_type.from_sql_type(v) for v in obj] class ListViewType(DataType): @@ -1059,7 +1059,7 @@ def to_sql_type(self, obj): def from_sql_type(self, obj): if not self.need_conversion(): return obj - return obj and [self.element_type.to_sql_type(v) for v in obj] + return obj and [self.element_type.from_sql_type(v) for v in obj] class RowField(object): From 5d91e07c8f2cfc3b4d0e779748e08311ea306bd3 Mon Sep 17 00:00:00 2001 From: Gustavo de Morais Date: Wed, 5 Aug 2026 10:54:36 +0200 Subject: [PATCH 18/34] [FLINK-40324][table] Append-only should stay unmaterialized without an ON CONFLICT clause This closes #28918 --- .../FlinkChangelogModeInferenceProgram.scala | 60 +++++++++++-------- .../planner/plan/stream/sql/TableSinkTest.xml | 37 ++++++++++++ .../plan/stream/sql/TableSinkTest.scala | 41 +++++++++++++ 3 files changed, 113 insertions(+), 25 deletions(-) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala index 1c91e960a8a18..5d578e2b076b8 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.planner.plan.optimize.program import org.apache.flink.legacy.table.sinks.{AppendStreamTableSink, RetractStreamTableSink, StreamTableSink, UpsertStreamTableSink} -import org.apache.flink.table.api.{TableException, ValidationException} +import org.apache.flink.table.api.{TableConfig, TableException, ValidationException} import org.apache.flink.table.api.InsertConflictStrategy.ConflictBehavior import org.apache.flink.table.api.config.ExecutionConfigOptions import org.apache.flink.table.api.config.ExecutionConfigOptions.UpsertMaterialize @@ -1148,7 +1148,8 @@ class FlinkChangelogModeInferenceProgram extends FlinkOptimizeProgram[StreamOpti * Analyze whether to enable upsertMaterialize or not. In these case will return true: * 1. when `TABLE_EXEC_SINK_UPSERT_MATERIALIZE` set to FORCE and sink's primary key nonempty. * 2. when `TABLE_EXEC_SINK_UPSERT_MATERIALIZE` set to AUTO and sink's primary key doesn't - * contain upsertKeys of the input update stream. + * contain upsertKeys of the input update stream, unless the input is insert only and the + * effective conflict strategy is DEDUPLICATE. * * Also validates that ON CONFLICT clause is specified when upsert key differs from primary key. */ @@ -1197,41 +1198,50 @@ class FlinkChangelogModeInferenceProgram extends FlinkOptimizeProgram[StreamOpti return false } - // For a DEDUPLICATE strategy and INSERT only input, we simply let the inserts be handled - // as UPSERT_AFTER and overwrite previous value - if (inputIsAppend && sink.isDeduplicateConflictStrategy) { - return false - } - // if input has updates and primary key != upsert key we should enable upsertMaterialize. // // An optimize is: do not enable upsertMaterialize when sink pk(s) contains input // changeLogUpsertKeys val upsertKeyDiffersFromPk = !sink.primaryKeysContainsUpsertKey + validateOnConflictSpecifiedIfRequired(sink, tableConfig, upsertKeyDiffersFromPk) - // Validate that ON CONFLICT is specified when upsert key differs from primary key - val requireOnConflict = - tableConfig.get(ExecutionConfigOptions.TABLE_EXEC_SINK_REQUIRE_ON_CONFLICT) - if (requireOnConflict && upsertKeyDiffersFromPk && sink.conflictStrategy == null) { - val pkNames = sink.getPrimaryKeyNames - val upsertKeyNames = sink.getUpsertKeyNames - throw new ValidationException( - "The query has an upsert key that differs from the primary key of the sink table " + - s"'${sink.contextResolvedTable.getIdentifier.asSummaryString}'. " + - s"Primary key: $pkNames, upsert key: $upsertKeyNames. " + - "This can lead to non-deterministic results when multiple records with different " + - "upsert keys map to the same primary key. " + - "Please specify an ON CONFLICT clause to define how conflicts should be handled: " + - "ON CONFLICT DO DEDUPLICATE (update to the latest record, state intensive, since we" + - " need to keep the entire history), or " + - "ON CONFLICT DO ERROR (fail on conflict), or " + - "ON CONFLICT DO NOTHING (keep first record).") + // Once enforcement above has passed, an absent clause leaves DEDUPLICATE as the strategy. + val deduplicatesOnConflict = + sink.conflictStrategy == null || sink.isDeduplicateConflictStrategy + + // For a DEDUPLICATE strategy and INSERT only input, we simply let the inserts be handled + // as UPDATE_AFTER and overwrite previous value + if (deduplicatesOnConflict && inputIsAppend) { + return false } upsertKeyDiffersFromPk } } + private def validateOnConflictSpecifiedIfRequired( + sink: StreamPhysicalSink, + tableConfig: TableConfig, + upsertKeyDiffersFromPk: Boolean): Unit = { + val requireOnConflict = + tableConfig.get(ExecutionConfigOptions.TABLE_EXEC_SINK_REQUIRE_ON_CONFLICT) + if (requireOnConflict && upsertKeyDiffersFromPk && sink.conflictStrategy == null) { + val pkNames = sink.getPrimaryKeyNames + val upsertKeyNames = sink.getUpsertKeyNames + throw new ValidationException( + "The query has an upsert key that differs from the primary key of the sink table " + + s"'${sink.contextResolvedTable.getIdentifier.asSummaryString}'. " + + s"Primary key: $pkNames, upsert key: $upsertKeyNames. " + + "This can lead to non-deterministic results when multiple records with different " + + "upsert keys map to the same primary key. " + + "Please specify an ON CONFLICT clause to define how conflicts should be handled: " + + "ON CONFLICT DO DEDUPLICATE (update to the latest record, state intensive, since we" + + " need to keep the entire history), or " + + "ON CONFLICT DO ERROR (fail on conflict), or " + + "ON CONFLICT DO NOTHING (keep first record).") + } + } + private def validateSourcesHaveWatermarks(sink: StreamPhysicalSink): Unit = { val sourcesWithoutWatermarks = new java.util.ArrayList[String]() collectSourcesWithoutWatermarks(sink.getInput, sourcesWithoutWatermarks) diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.xml index 1ae3035de31d8..90d4c28486542 100644 --- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.xml +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.xml @@ -16,6 +16,22 @@ See the License for the specific language governing permissions and limitations under the License. --> + + + + + + + + + + + + + + + + diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala index 31f0d6ca6fa65..d3e46cfc7bdc6 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/TableSinkTest.scala @@ -864,6 +864,47 @@ class TableSinkTest extends TableTestBase { util.verifyRelPlan(stmtSet, ExplainDetail.CHANGELOG_MODE) } + @Test + def testAppendOnlyInputWithoutOnConflict(): Unit = { + util.tableEnv.getConfig + .set(ExecutionConfigOptions.TABLE_EXEC_SINK_REQUIRE_ON_CONFLICT, Boolean.box(false)) + util.addTable(s""" + |CREATE TABLE sinkWithPk ( + | `a` INT, + | `b` BIGINT, + | PRIMARY KEY (a) NOT ENFORCED + |) WITH ( + | 'connector' = 'values', + | 'sink-insert-only' = 'false' + |) + |""".stripMargin) + val stmtSet = util.tableEnv.createStatementSet() + stmtSet.addInsertSql("INSERT INTO sinkWithPk SELECT a, b FROM MyTable") + util.verifyRelPlan(stmtSet, ExplainDetail.CHANGELOG_MODE) + } + + @Test + def testUpdatingInputWithoutOnConflict(): Unit = { + util.tableEnv.getConfig + .set(ExecutionConfigOptions.TABLE_EXEC_SINK_REQUIRE_ON_CONFLICT, Boolean.box(false)) + util.addTable(s""" + |CREATE TABLE updatingSinkWithPk ( + | `id` INT, + | `cnt` BIGINT, + | PRIMARY KEY (id) NOT ENFORCED + |) WITH ( + | 'connector' = 'values', + | 'sink-insert-only' = 'false' + |) + |""".stripMargin) + val stmtSet = util.tableEnv.createStatementSet() + // The upsert key is the grouping key c, which is not written to the sink, so it can never + // match the primary key. + stmtSet.addInsertSql( + "INSERT INTO updatingSinkWithPk SELECT MAX(a), COUNT(*) FROM MyTable GROUP BY c") + util.verifyRelPlan(stmtSet, ExplainDetail.CHANGELOG_MODE) + } + @Test def testInjectiveCastPreservesUpsertKey(): Unit = { // Aggregation produces upsert stream with key (a). From e095557d5dbbae33e6011d0b2893af0251f5bb57 Mon Sep 17 00:00:00 2001 From: Sergey Nuyanzin Date: Wed, 5 Aug 2026 11:42:07 +0200 Subject: [PATCH 19/34] [FLINK-18476][python] `PythonEnvUtils#testStartPythonProcess` might fail --- .../java/org/apache/flink/client/python/PythonEnvUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flink-python/src/main/java/org/apache/flink/client/python/PythonEnvUtils.java b/flink-python/src/main/java/org/apache/flink/client/python/PythonEnvUtils.java index bd816b6424abf..4e4a511430eae 100644 --- a/flink-python/src/main/java/org/apache/flink/client/python/PythonEnvUtils.java +++ b/flink-python/src/main/java/org/apache/flink/client/python/PythonEnvUtils.java @@ -383,7 +383,8 @@ static Process startPythonProcess( .collect(Collectors.joining(", ")), String.join(" ", commands)); Process process = pythonProcessBuilder.start(); - if (!process.isAlive()) { + // Only a non-zero exit is a start failure; a fast process may already have finished. + if (!process.isAlive() && process.exitValue() != 0) { throw new RuntimeException("Failed to start Python process. "); } From 33a198ae41ca1f11a8a8f313a5295551d1d591df Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Wed, 5 Aug 2026 12:27:54 +0200 Subject: [PATCH 20/34] [FLINK-40305][core] Decode `VARIANT` strings and object keys as UTF-8 `BinaryVariantUtil` decoded string values and object field names with `new String(byte[], int, int)`, which uses the JVM default charset, while `BinaryVariantInternalBuilder` writes both as UTF-8. The two only agree on Java 18+, where JEP 400 made UTF-8 the default charset. On Java 11 and 17 a non-UTF-8 platform charset corrupts any non-ASCII text. Corrupted field names are the worse half of this. `getField(name)` silently returns null, and `getFieldNames()` and `toJson()` return mangled keys. Both call sites now pass `StandardCharsets.UTF_8` explicitly, matching Spark's `VariantUtil`. --- .../types/variant/BinaryVariantUtil.java | 6 ++- .../BinaryVariantInternalBuilderTest.java | 12 +++++ .../types/variant/BinaryVariantTest.java | 49 +++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java index a3aed62cc130c..c3ab8d29be811 100644 --- a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java +++ b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantUtil.java @@ -23,6 +23,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.util.Arrays; @@ -528,7 +529,7 @@ public static String getString(byte[] value, int pos) { length = readUnsigned(value, pos + 1, U32_SIZE); } checkIndex(start + length - 1, value.length); - return new String(value, start, length); + return new String(value, start, length, StandardCharsets.UTF_8); } throw unexpectedType(Type.STRING); } @@ -625,6 +626,7 @@ public static String getMetadataKey(byte[] metadata, int id) { throw malformedVariant(); } checkIndex(stringStart + nextOffset - 1, metadata.length); - return new String(metadata, stringStart + offset, nextOffset - offset); + return new String( + metadata, stringStart + offset, nextOffset - offset, StandardCharsets.UTF_8); } } diff --git a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java index cec12149ceb5f..3ce271896a2f8 100644 --- a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java +++ b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java @@ -122,6 +122,18 @@ void testParseJsonObject() throws IOException { assertThat(variant.getField("k2").getDecimal()).isEqualTo(BigDecimal.valueOf(1.5)); } + @Test + void testParseJsonWithNonAsciiStringsAndKeys() throws IOException { + String json = "{\"schlüssel\":\"Grüße, 世界 🚀\",\"キー\":[\"äöü\"]}"; + + BinaryVariant variant = BinaryVariantInternalBuilder.parseJson(json, false); + + assertThat(variant.getFieldNames()).containsExactlyInAnyOrder("schlüssel", "キー"); + assertThat(variant.getField("schlüssel").getString()).isEqualTo("Grüße, 世界 🚀"); + assertThat(variant.getField("キー").getElement(0).getString()).isEqualTo("äöü"); + assertThat(variant.toJson()).isEqualTo(json); + } + @ParameterizedTest @ValueSource(strings = {"NaN", "Infinity", "-Infinity", "1e400", "-1e400"}) void testParseJsonRejectsNonFiniteNumbers(final String nonFiniteNumber) { diff --git a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java index 77235e968eebc..4468fcbe8d8e1 100644 --- a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java +++ b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java @@ -24,10 +24,12 @@ import org.junit.jupiter.params.provider.ValueSource; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -254,6 +256,53 @@ void testToJsonRejectsNonFiniteFloat(final float nonFinite) { .hasMessageContaining("cannot be serialized to JSON"); } + @Test + void testNonAsciiStringsAndFieldNames() { + // Multi-byte code points make the UTF-8 byte length differ from the character count, so a + // charset mismatch between writing and reading mangles the text instead of preserving it. + final String nestedKey = "キー"; + final String shortValue = "Grüße, 世界 🚀"; + final String longValue = String.join("", Collections.nCopies(20, "äö🚀")); + + assertThat(longValue.getBytes(StandardCharsets.UTF_8).length) + .as("long string must not fit into the short string encoding") + .isGreaterThan(BinaryVariantUtil.MAX_SHORT_STR_SIZE); + + final BinaryVariant variant = + (BinaryVariant) + builder.object() + .add("schlüssel", builder.of(shortValue)) + .add( + nestedKey, + builder.object() + .add("schlüssel", builder.of(longValue)) + .build()) + .build(); + + // Reading through the raw binaries is what happens once a variant has been serialized, and + // it is the only path that decodes the field names from the metadata. + final BinaryVariant decoded = new BinaryVariant(variant.getValue(), variant.getMetadata()); + + assertThat(decoded.getFieldNames()).containsExactlyInAnyOrder("schlüssel", nestedKey); + assertThat(decoded.getField("schlüssel").getString()).isEqualTo(shortValue); + assertThat(decoded.getField(nestedKey).getFieldNames()).containsExactly("schlüssel"); + assertThat(decoded.getField(nestedKey).getField("schlüssel").getString()) + .isEqualTo(longValue); + assertThat(decoded.toJson()) + .isEqualTo( + "{\"" + + "schlüssel" + + "\":\"" + + shortValue + + "\",\"" + + nestedKey + + "\":{\"" + + "schlüssel" + + "\":\"" + + longValue + + "\"}}"); + } + @Test void testVariantException() { assertThatThrownBy(() -> new BinaryVariant(new byte[0], new byte[0])) From cf2df298f22f18e9d104e80597ca69b05006bc4c Mon Sep 17 00:00:00 2001 From: Gabor Somogyi Date: Wed, 5 Aug 2026 17:11:09 +0200 Subject: [PATCH 21/34] [FLINK-40332][table] Prevent path traversal in FileCatalogStore catalog names --- .../flink/table/catalog/FileCatalogStore.java | 23 ++++++++++++- .../table/catalog/FileCatalogStoreTest.java | 32 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FileCatalogStore.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FileCatalogStore.java index 50763a828ead3..0e1fb69ee3803 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FileCatalogStore.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FileCatalogStore.java @@ -260,6 +260,27 @@ public boolean contains(String catalogName) throws CatalogException { } private Path getCatalogPath(String catalogName) { - return new Path(catalogStorePath, catalogName + FILE_EXTENSION); + Path catalogPath; + try { + catalogPath = new Path(catalogStorePath, catalogName + FILE_EXTENSION); + } catch (Exception e) { + // e.g. catalogName embeds its own scheme-qualified URI (like "file:///etc/passwd"), + // which Path may reject outright while merging it against catalogStorePath. + throw new CatalogException(String.format("Invalid catalog name '%s'.", catalogName), e); + } + + // catalogName is caller-supplied and may try to escape catalogStorePath, e.g. via ".." + // segments. Path's own resolution above already fully normalizes the result (RFC 3986 + // dot-segment removal), so checking that the *resolved* path's parent is still + // catalogStorePath is sufficient to reject every variant of escape, without needing to + // inspect catalogName itself. + if (!catalogStorePath.equals(catalogPath.getParent())) { + throw new CatalogException( + String.format( + "Invalid catalog name '%s'. It resolves to '%s', which is outside of " + + "the catalog store directory '%s'.", + catalogName, catalogPath, catalogStorePath)); + } + return catalogPath; } } diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/FileCatalogStoreTest.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/FileCatalogStoreTest.java index 011ce881bb84f..e22f3c7d89c88 100644 --- a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/FileCatalogStoreTest.java +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/FileCatalogStoreTest.java @@ -106,6 +106,38 @@ void testStore() { assertThat(storedCatalogs.contains(DUMMY)).isTrue(); } + @Test + void testStoreCatalogRejectsPathTraversal() throws Exception { + CatalogStore catalogStore = initCatalogStore(); + catalogStore.open(); + + // A malicious catalog name that, if not validated, resolves outside of the + // catalog store directory: tempDir/dummy-catalog-store/../escaped.yaml -> + // tempDir/escaped.yaml. + String maliciousName = "../escaped"; + File escapedFile = tempDir.resolve("escaped" + FileCatalogStore.FILE_EXTENSION).toFile(); + + assertThatThrownBy(() -> catalogStore.storeCatalog(maliciousName, DUMMY_CATALOG)) + .isInstanceOf(CatalogException.class); + assertThat(escapedFile).doesNotExist(); + } + + @Test + void testStoreCatalogRejectsAbsoluteSchemeOverride() throws Exception { + CatalogStore catalogStore = initCatalogStore(); + catalogStore.open(); + + // A catalog name that embeds its own absolute file:// URI. Per RFC 3986 §5.3, resolving + // an absolute reference against a base URI discards the base entirely, so if this isn't + // rejected, the catalog store directory is bypassed altogether. + File escapedFile = tempDir.resolve("escaped" + FileCatalogStore.FILE_EXTENSION).toFile(); + String maliciousName = "file://" + escapedFile.getAbsolutePath().replace(".yaml", ""); + + assertThatThrownBy(() -> catalogStore.storeCatalog(maliciousName, DUMMY_CATALOG)) + .isInstanceOf(CatalogException.class); + assertThat(escapedFile).doesNotExist(); + } + @Test void testRemoveExisting() { CatalogStore catalogStore = initCatalogStore(); From 5eb36d438796ff7da2ef0a4608c216f9aad0694c Mon Sep 17 00:00:00 2001 From: Aleksandr Savonin Date: Fri, 31 Jul 2026 15:29:27 +0200 Subject: [PATCH 22/34] [FLINK-40270][connector-base][runtime] Make source threads job-attributable via MDC propagation and thread names Source split-fetcher threads previously carried no job identity, so on a shared TaskManager their logs and thread dumps could not be traced back to the job that owned them. This adds the job id into each fetcher pool thread's MDC and appends a truncated job-name/job-id suffix to the fetcher thread name, making both logs and thread dumps attributable per job. --- ...SingleThreadMultiplexSourceReaderBase.java | 21 ++- .../fetcher/SingleThreadFetcherManager.java | 22 +++ .../reader/fetcher/SplitFetcherManager.java | 51 ++++++- ...leThreadMultiplexSourceReaderBaseTest.java | 129 ++++++++++++++++++ .../fetcher/SplitFetcherManagerTest.java | 108 +++++++++++++++ .../java/org/apache/flink/util/MdcUtils.java | 57 ++++++++ .../org/apache/flink/util/MdcUtilsTest.java | 69 +++++++++- .../coordinator/SourceCoordinatorContext.java | 7 +- .../SourceCoordinatorProvider.java | 18 ++- .../SourceCoordinatorContextTest.java | 42 ++++++ .../SourceCoordinatorProviderTest.java | 37 +++++ 11 files changed, 552 insertions(+), 9 deletions(-) create mode 100644 flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBaseTest.java diff --git a/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBase.java b/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBase.java index 023a7d0c50db2..36f2762cc4a52 100644 --- a/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBase.java +++ b/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBase.java @@ -19,6 +19,7 @@ package org.apache.flink.connector.base.source.reader; import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.JobInfo; import org.apache.flink.api.connector.source.SourceReader; import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.api.connector.source.SourceSplit; @@ -75,7 +76,8 @@ public SingleThreadMultiplexSourceReaderBase( Configuration config, SourceReaderContext context) { super( - new SingleThreadFetcherManager<>(splitReaderSupplier, config), + new SingleThreadFetcherManager<>( + splitReaderSupplier, config, (ignore) -> {}, getJobInfoOrNull(context)), recordEmitter, config, context); @@ -93,7 +95,8 @@ public SingleThreadMultiplexSourceReaderBase( SourceReaderContext context, @Nullable RateLimiterStrategy rateLimiterStrategy) { super( - new SingleThreadFetcherManager<>(splitReaderSupplier, config), + new SingleThreadFetcherManager<>( + splitReaderSupplier, config, (ignore) -> {}, getJobInfoOrNull(context)), recordEmitter, null, config, @@ -149,4 +152,18 @@ public SingleThreadMultiplexSourceReaderBase( context, rateLimiterStrategy); } + + /** + * Returns the {@link JobInfo} of the given context, or {@code null} if the context (e.g. an + * older runtime or a test double) does not implement {@link SourceReaderContext#getJobInfo()}, + * which throws {@link UnsupportedOperationException} by default. + */ + @Nullable + private static JobInfo getJobInfoOrNull(SourceReaderContext context) { + try { + return context.getJobInfo(); + } catch (UnsupportedOperationException e) { + return null; + } + } } diff --git a/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SingleThreadFetcherManager.java b/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SingleThreadFetcherManager.java index 642d4ae495d13..adb903cbcd6b7 100644 --- a/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SingleThreadFetcherManager.java +++ b/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SingleThreadFetcherManager.java @@ -19,10 +19,13 @@ package org.apache.flink.connector.base.source.reader.fetcher; import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.JobInfo; import org.apache.flink.api.connector.source.SourceSplit; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import javax.annotation.Nullable; + import java.util.Collection; import java.util.List; import java.util.function.Consumer; @@ -78,6 +81,25 @@ public SingleThreadFetcherManager( super(splitReaderSupplier, configuration, splitFinishedHook); } + /** + * Creates a new SplitFetcherManager with a single I/O thread. + * + * @param splitReaderSupplier The factory for the split reader that connects to the source + * system. + * @param configuration The configuration to create the fetcher manager. + * @param splitFinishedHook Hook for handling finished splits in split fetchers + * @param jobInfo The job this fetcher manager belongs to, or {@code null} if unknown. See + * {@link SplitFetcherManager#SplitFetcherManager(Supplier, Configuration, Consumer, + * JobInfo)}. + */ + public SingleThreadFetcherManager( + Supplier> splitReaderSupplier, + Configuration configuration, + Consumer> splitFinishedHook, + @Nullable JobInfo jobInfo) { + super(splitReaderSupplier, configuration, splitFinishedHook, jobInfo); + } + @Override public void addSplits(List splitsToAdd) { SplitFetcher fetcher = getRunningFetcher(); diff --git a/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManager.java b/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManager.java index e101bc4861c65..6ea31d2d53bae 100644 --- a/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManager.java +++ b/flink-connectors/flink-connector-base/src/main/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManager.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.JobInfo; import org.apache.flink.api.connector.source.SourceSplit; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; @@ -28,10 +29,13 @@ import org.apache.flink.connector.base.source.reader.SourceReaderOptions; import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; +import org.apache.flink.util.MdcUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -126,6 +130,24 @@ public SplitFetcherManager( Supplier> splitReaderFactory, Configuration configuration, Consumer> splitFinishedHook) { + this(splitReaderFactory, configuration, splitFinishedHook, null); + } + + /** + * Create a split fetcher manager. + * + * @param splitReaderFactory a supplier that could be used to create split readers. + * @param configuration the configuration of this fetcher manager. + * @param splitFinishedHook Hook for handling finished splits in split fetchers. + * @param jobInfo the job this fetcher manager belongs to, or {@code null} if unknown. When + * provided, fetcher threads carry the job id in their MDC ({@value MdcUtils#JOB_ID}) and + * thread names, making their logs and thread dumps attributable to the job. + */ + public SplitFetcherManager( + Supplier> splitReaderFactory, + Configuration configuration, + Consumer> splitFinishedHook, + @Nullable JobInfo jobInfo) { this.elementsQueue = new FutureCompletingBlockingQueue<>( configuration.get(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY)); @@ -153,12 +175,35 @@ public void accept(Throwable t) { // Create the executor with a thread factory that fails the source reader if one of // the fetcher thread exits abnormally. final String taskThreadName = Thread.currentThread().getName(); - this.executors = - Executors.newCachedThreadPool( - r -> new Thread(r, THREAD_NAME_PREFIX + taskThreadName)); + final String fetcherThreadName = createFetcherThreadName(taskThreadName, jobInfo); + if (jobInfo != null) { + // MDC is thread-local and not inherited, so seed the job id into each pool thread. + final Map jobMdcContext = MdcUtils.asContextData(jobInfo.getJobId()); + this.executors = + Executors.newCachedThreadPool( + r -> + new Thread( + MdcUtils.wrapRunnable(jobMdcContext, r), + fetcherThreadName)); + } else { + this.executors = Executors.newCachedThreadPool(r -> new Thread(r, fetcherThreadName)); + } this.closed = false; } + /** + * Builds the name shared by all fetcher threads of this manager. When the job is known, a + * {@link MdcUtils#jobThreadNameSuffix(JobInfo) job suffix} is appended so fetcher threads of + * different jobs are distinguishable on a shared TaskManager. + */ + private static String createFetcherThreadName( + String taskThreadName, @Nullable JobInfo jobInfo) { + if (jobInfo == null) { + return THREAD_NAME_PREFIX + taskThreadName; + } + return THREAD_NAME_PREFIX + taskThreadName + MdcUtils.jobThreadNameSuffix(jobInfo); + } + public abstract void addSplits(List splitsToAdd); public abstract void removeSplits(List splitsToRemove); diff --git a/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBaseTest.java b/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBaseTest.java new file mode 100644 index 0000000000000..d59433684e882 --- /dev/null +++ b/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SingleThreadMultiplexSourceReaderBaseTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.base.source.reader; + +import org.apache.flink.api.common.JobInfo; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.api.connector.source.mocks.MockSourceSplit; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.mocks.MockSourceReader; +import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import org.apache.flink.connector.base.source.reader.splitreader.SplitsChange; +import org.apache.flink.connector.testutils.source.reader.TestingReaderContext; +import org.apache.flink.core.testutils.OneShotLatch; +import org.apache.flink.util.MdcUtils; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests the job attribution that {@link SingleThreadMultiplexSourceReaderBase} gives the fetcher + * threads it creates. + */ +class SingleThreadMultiplexSourceReaderBaseTest { + + @Test + void testFetcherThreadNameIdentifiesJob() throws Exception { + final TestingReaderContext context = new TestingReaderContext(); + + assertThat(fetcherThreadNameOf(context)) + .endsWith(MdcUtils.jobThreadNameSuffix(context.getJobInfo())); + } + + /** + * A context that leaves {@link SourceReaderContext#getJobInfo()} at its throwing default must + * still yield a working reader, only one without a job suffix: attribution is diagnostic only. + */ + @Test + void testContextWithoutJobInfoYieldsUnattributedFetcherThread() throws Exception { + final SourceReaderContext context = + new TestingReaderContext() { + @Override + public JobInfo getJobInfo() { + throw new UnsupportedOperationException(); + } + }; + + assertThat(fetcherThreadNameOf(context)) + .as("an unattributable reader must not gain a job suffix") + .doesNotContain(" (job: "); + } + + /** + * Builds a reader over the given context and assigns it a split, so that the fetcher thread + * starts and can report its own name. + */ + private static String fetcherThreadNameOf(SourceReaderContext context) throws Exception { + final CompletableFuture fetcherThreadName = new CompletableFuture<>(); + try (MockSourceReader reader = + new MockSourceReader( + () -> new ThreadNameReportingSplitReader(fetcherThreadName), + new Configuration(), + context)) { + reader.start(); + reader.addSplits(Collections.singletonList(new MockSourceSplit(0, 0, 1))); + assertThat(fetcherThreadName) + .as("The fetcher thread should have started fetching.") + .succeedsWithin(Duration.ofSeconds(60)); + return fetcherThreadName.get(); + } + } + + /** Reports the thread it is driven on, which is the fetcher thread under test. */ + private static final class ThreadNameReportingSplitReader + implements SplitReader { + + private final CompletableFuture threadName; + private final OneShotLatch fetchBlocker = new OneShotLatch(); + + private ThreadNameReportingSplitReader(CompletableFuture threadName) { + this.threadName = threadName; + } + + @Override + public RecordsWithSplitIds fetch() { + threadName.complete(Thread.currentThread().getName()); + // Stay inside fetch() until woken up, so the fetcher does not spin on empty fetches. + try { + fetchBlocker.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); + } + + @Override + public void handleSplitsChanges(SplitsChange splitsChanges) {} + + @Override + public void wakeUp() { + fetchBlocker.trigger(); + } + + @Override + public void close() { + fetchBlocker.trigger(); + } + } +} diff --git a/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManagerTest.java b/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManagerTest.java index 742571e37ddf0..cf0fce5d42cf6 100644 --- a/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManagerTest.java +++ b/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/fetcher/SplitFetcherManagerTest.java @@ -18,6 +18,9 @@ package org.apache.flink.connector.base.source.reader.fetcher; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobInfo; +import org.apache.flink.api.common.JobInfoImpl; import org.apache.flink.api.connector.source.SourceSplit; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.base.source.reader.RecordsBySplits; @@ -30,9 +33,13 @@ import org.apache.flink.connector.base.source.reader.splitreader.SplitsChange; import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; import org.apache.flink.core.testutils.OneShotLatch; +import org.apache.flink.util.MdcUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.slf4j.MDC; + +import javax.annotation.Nullable; import java.io.IOException; import java.time.Duration; @@ -42,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Queue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import static org.apache.flink.test.util.TestUtils.waitUntil; @@ -75,6 +83,36 @@ void testCloseFetcherWithException() throws Exception { .hasRootCauseMessage("Artificial exception on closing the split reader."); } + /** + * The exact suffix format is covered by {@code MdcUtilsTest#testJobThreadNameSuffix}; this only + * has to show that a fetcher thread is seeded with the job MDC and named after the job at all. + */ + @Test + void testFetcherThreadCarriesJobIdInMdcAndThreadName() throws Exception { + final JobID jobId = new JobID(); + final JobInfo jobInfo = new JobInfoImpl(jobId, "my-test-job"); + final String taskThreadName = Thread.currentThread().getName(); + + final FetcherThreadInfo fetcherThread = captureFetcherThread(jobInfo); + + assertThat(fetcherThread.mdcJobId).isEqualTo(jobId.toHexString()); + assertThat(fetcherThread.threadName) + .startsWith(SplitFetcherManager.THREAD_NAME_PREFIX) + .contains(taskThreadName) + .endsWith(MdcUtils.jobThreadNameSuffix(jobInfo)); + } + + @Test + void testFetcherThreadWithoutJobInfoKeepsHistoricalNameAndNoJobIdInMdc() throws Exception { + final FetcherThreadInfo fetcherThread = captureFetcherThread(null); + + // Fetcher threads are named after the thread creating the manager, i.e. this test thread. + assertThat(fetcherThread.threadName) + .isEqualTo( + SplitFetcherManager.THREAD_NAME_PREFIX + Thread.currentThread().getName()); + assertThat(fetcherThread.mdcJobId).isNull(); + } + @Test @Timeout(value = 30000, unit = TimeUnit.MILLISECONDS) void testCloseCleansUpPreviouslyClosedFetcher() throws Exception { @@ -243,6 +281,31 @@ private static SplitFetcherManager createFetcher( return fetcher; } + /** + * Runs a fetcher for the given job identity ({@code null} exercising the constructors without + * {@link JobInfo}) and returns what its fetcher thread saw. The fetcher manager is closed again + * before returning. + */ + private static FetcherThreadInfo captureFetcherThread(@Nullable JobInfo jobInfo) + throws Exception { + final ThreadInfoCapturingSplitReader reader = + new ThreadInfoCapturingSplitReader<>(); + final SingleThreadFetcherManager fetcherManager = + jobInfo == null + ? new SingleThreadFetcherManager<>(() -> reader, new Configuration()) + : new SingleThreadFetcherManager<>( + () -> reader, new Configuration(), (ignore) -> {}, jobInfo); + try { + fetcherManager.addSplits(Collections.singletonList(new TestingSourceSplit("split-0"))); + assertThat(reader.threadInfo) + .as("The fetcher thread should have started fetching.") + .succeedsWithin(Duration.ofSeconds(60)); + return reader.threadInfo.get(); + } finally { + fetcherManager.close(30_000L); + } + } + private static void drainQueue(FutureCompletingBlockingQueue queue) { //noinspection StatementWithEmptyBody while (queue.poll() != null) {} @@ -262,6 +325,51 @@ private static List findThread(String keyword) { // test mocks // ------------------------------------------------------------------------ + /** Thread name and {@value MdcUtils#JOB_ID} MDC value observed on a fetcher thread. */ + private static final class FetcherThreadInfo { + + private final String threadName; + @Nullable private final String mdcJobId; + + private FetcherThreadInfo(String threadName, @Nullable String mdcJobId) { + this.threadName = threadName; + this.mdcJobId = mdcJobId; + } + } + + /** A {@link SplitReader} that reports the fetcher thread it is executed on. */ + private static final class ThreadInfoCapturingSplitReader + implements SplitReader { + + private final CompletableFuture threadInfo = new CompletableFuture<>(); + private final OneShotLatch fetchBlocker = new OneShotLatch(); + + @Override + public RecordsWithSplitIds fetch() { + threadInfo.complete( + new FetcherThreadInfo( + Thread.currentThread().getName(), MDC.get(MdcUtils.JOB_ID))); + // Stay inside fetch() until woken up, so the fetcher does not spin on empty fetches. + try { + fetchBlocker.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return new RecordsBySplits<>(Collections.emptyMap(), Collections.emptySet()); + } + + @Override + public void handleSplitsChanges(SplitsChange splitsChanges) {} + + @Override + public void wakeUp() { + fetchBlocker.trigger(); + } + + @Override + public void close() {} + } + private static final class AwaitingReader implements SplitReader { diff --git a/flink-core/src/main/java/org/apache/flink/util/MdcUtils.java b/flink-core/src/main/java/org/apache/flink/util/MdcUtils.java index 935dfa7950526..979b8a2c94310 100644 --- a/flink-core/src/main/java/org/apache/flink/util/MdcUtils.java +++ b/flink-core/src/main/java/org/apache/flink/util/MdcUtils.java @@ -19,11 +19,14 @@ package org.apache.flink.util; import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MdcOptions; import org.slf4j.MDC; +import javax.annotation.Nonnull; + import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -39,6 +42,27 @@ public class MdcUtils { public static final String JOB_ID = "flink-job-id"; + /** + * Longest job name embedded in a thread name; longer ones, such as generated SQL job names, are + * truncated. Matches the length of the hex {@link JobID} that follows it. + */ + private static final int MAX_JOB_NAME_IN_THREAD_NAME = 32; + + /** + * Number of trailing job name characters kept when a job name is truncated. Generated job names + * often share a long prefix and differ only near the end (e.g. {@code ...-v1} / {@code + * ...-v2}), so dropping the tail would make distinct jobs indistinguishable in a thread dump. + */ + private static final int TRUNCATED_JOB_NAME_TAIL_LENGTH = 9; + + private static final String TRUNCATION_MARKER = "..."; + + /** Chosen so that a truncated name is exactly {@link #MAX_JOB_NAME_IN_THREAD_NAME} long. */ + private static final int TRUNCATED_JOB_NAME_HEAD_LENGTH = + MAX_JOB_NAME_IN_THREAD_NAME + - TRUNCATION_MARKER.length() + - TRUNCATED_JOB_NAME_TAIL_LENGTH; + /** * Replace MDC contents with the provided one and return a closeable object that can be used to * restore the original MDC. @@ -149,4 +173,37 @@ public static Map asContextData( context.put(JOB_ID, jobID.toHexString()); return Collections.unmodifiableMap(context); } + + /** + * Build a thread-name suffix identifying the job, e.g. {@code " (job: my-job / + * 2f4b0e4a9cbb223e924f1e5d9e6a7c11)"}. The job name may be truncated; the hex job id never is, + * so it always matches the {@link #JOB_ID} MDC value and a thread dump can be lined up with the + * logs. + * + * @param jobInfo the job meta information + * @return a suffix to append to a thread name + */ + public static String jobThreadNameSuffix(@Nonnull JobInfo jobInfo) { + final String hexJobId = jobInfo.getJobId().toHexString(); + final String rawJobName = jobInfo.getJobName(); + final String jobName = rawJobName == null ? "" : rawJobName.strip(); + if (jobName.isEmpty()) { + return " (job: " + hexJobId + ")"; + } + return " (job: " + truncateJobName(jobName) + " / " + hexJobId + ")"; + } + + /** + * Shorten a job name to {@link #MAX_JOB_NAME_IN_THREAD_NAME} characters by eliding its middle + * and keeping the last {@link #TRUNCATED_JOB_NAME_TAIL_LENGTH}, for the reason given on that + * constant. + */ + private static String truncateJobName(String jobName) { + if (jobName.length() <= MAX_JOB_NAME_IN_THREAD_NAME) { + return jobName; + } + return jobName.substring(0, TRUNCATED_JOB_NAME_HEAD_LENGTH) + + TRUNCATION_MARKER + + jobName.substring(jobName.length() - TRUNCATED_JOB_NAME_TAIL_LENGTH); + } } diff --git a/flink-core/src/test/java/org/apache/flink/util/MdcUtilsTest.java b/flink-core/src/test/java/org/apache/flink/util/MdcUtilsTest.java index 117a51b74bcf4..8075bffd9a104 100644 --- a/flink-core/src/test/java/org/apache/flink/util/MdcUtilsTest.java +++ b/flink-core/src/test/java/org/apache/flink/util/MdcUtilsTest.java @@ -19,6 +19,8 @@ package org.apache.flink.util; import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobInfo; +import org.apache.flink.api.common.JobInfoImpl; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MdcOptions; import org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService; @@ -128,7 +130,72 @@ private static Stream wrappingMechanisms() { void testJobIdLoggedByWrappingMechanism( final String scenario, final ThrowingConsumer action) throws Exception { - assertJobIDLogged(scenario, jobID -> action.accept(jobID)); + assertJobIDLogged(scenario, action); + } + + /** + * Expected strings are literals rather than values computed from the truncation constants: + * computing them would make the test agree with the production formula even when that formula + * is wrong. Job names use a distinct character per position so that the head and tail + * boundaries in each literal can be checked by eye against the input. + */ + private static Stream jobThreadNameSuffixCases() { + final JobID jobID = new JobID(); + final String hexJobId = jobID.toHexString(); + final String nameAtCap = "0123456789abcdefghijABCDEFGHIJxy"; + return Stream.of( + Arguments.of( + "short name kept verbatim", + new JobInfoImpl(jobID, "my-job"), + " (job: my-job / " + hexJobId + ")"), + Arguments.of( + "padded name stripped", + new JobInfoImpl(jobID, " my-job "), + " (job: my-job / " + hexJobId + ")"), + Arguments.of( + "name at the cap kept verbatim", + new JobInfoImpl(jobID, nameAtCap), + " (job: " + nameAtCap + " / " + hexJobId + ")"), + Arguments.of( + "one character over the cap is elided in the middle", + new JobInfoImpl(jobID, nameAtCap + "z"), + " (job: 0123456789abcdefghij...EFGHIJxyz / " + hexJobId + ")"), + Arguments.of( + "long name keeps head and tail (v1)", + new JobInfoImpl(jobID, "0123456789abcdefghijABCDEFGHIJ-job-v1"), + " (job: 0123456789abcdefghij...IJ-job-v1 / " + hexJobId + ")"), + Arguments.of( + "empty name omitted", + new JobInfoImpl(jobID, ""), + " (job: " + hexJobId + ")"), + Arguments.of( + "blank name omitted", + new JobInfoImpl(jobID, " "), + " (job: " + hexJobId + ")"), + Arguments.of( + "null name omitted", nullNameJobInfo(jobID), " (job: " + hexJobId + ")")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("jobThreadNameSuffixCases") + void testJobThreadNameSuffix( + final String scenario, final JobInfo jobInfo, final String expectedSuffix) { + assertThat(MdcUtils.jobThreadNameSuffix(jobInfo)).isEqualTo(expectedSuffix); + } + + /** {@link JobInfoImpl} rejects a null name, so null handling needs a hand-written one. */ + private static JobInfo nullNameJobInfo(JobID jobID) { + return new JobInfo() { + @Override + public JobID getJobId() { + return jobID; + } + + @Override + public String getJobName() { + return null; + } + }; } @Test diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java index bb2d6a300502b..67e20fac4b990 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java @@ -154,7 +154,7 @@ public SourceCoordinatorContext( SimpleVersionedSerializer splitSerializer, SplitAssignmentTracker splitAssignmentTracker, boolean supportsConcurrentExecutionAttempts) { - this.workerExecutor = workerExecutor; + this.workerExecutor = MdcUtils.scopeToJob(jobID, workerExecutor); this.coordinatorExecutor = MdcUtils.scopeToJob(jobID, coordinatorExecutor); this.coordinatorThreadFactory = coordinatorThreadFactory; this.operatorCoordinatorContext = operatorCoordinatorContext; @@ -170,7 +170,10 @@ public SourceCoordinatorContext( new ThrowableCatchingRunnable( this::handleUncaughtExceptionFromAsyncCall, runnable)); - this.notifier = new ExecutorNotifier(workerExecutor, errorHandlingCoordinatorExecutor); + // Must be the field, not the constructor parameter: the field is the scopeToJob-wrapped + // executor, so the callables ExecutorNotifier schedules on it, one-shot and periodic alike, + // log with the job id rather than an empty MDC. + this.notifier = new ExecutorNotifier(this.workerExecutor, errorHandlingCoordinatorExecutor); } boolean isConcurrentExecutionAttemptsSupported() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java index 742a0bcb56308..6c4b672266314 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java @@ -25,6 +25,7 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.operators.coordination.OperatorCoordinator; import org.apache.flink.runtime.operators.coordination.RecreateOnResetOperatorCoordinator; +import org.apache.flink.util.MdcUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +76,7 @@ public SourceCoordinatorProvider( @Override public OperatorCoordinator getCoordinator(OperatorCoordinator.Context context) { - final String coordinatorThreadName = "SourceCoordinator-" + operatorName; + final String coordinatorThreadName = createCoordinatorThreadName(context); CoordinatorExecutorThreadFactory coordinatorThreadFactory = new CoordinatorExecutorThreadFactory(coordinatorThreadName, context); @@ -98,6 +99,21 @@ public OperatorCoordinator getCoordinator(OperatorCoordinator.Context context) { coordinatorListeningID); } + /** + * Builds the coordinator thread name. The operator name alone is ambiguous on a shared cluster + * because two jobs may use identically named sources, so the job identity is appended to make + * the coordinator thread (and its derived {@code -worker} pool) attributable to a job. + */ + private String createCoordinatorThreadName(OperatorCoordinator.Context context) { + final String base = "SourceCoordinator-" + operatorName; + try { + return base + MdcUtils.jobThreadNameSuffix(context.getJobInfo()); + } catch (UnsupportedOperationException e) { + // A custom Context may not expose job identity - fall back to the operator-only name. + return base; + } + } + /** * A thread factory class that provides some helper methods. Because it is used to check the * current thread, it is a one-off, do not use this ThreadFactory to create multiple threads. diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java index f6db6e5bb8ea2..aa94c861f3232 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java @@ -29,8 +29,10 @@ import org.apache.flink.runtime.source.event.AddSplitEvent; import org.apache.flink.runtime.source.event.IsProcessingBacklogEvent; import org.apache.flink.runtime.source.event.ReaderRegistrationEvent; +import org.apache.flink.util.MdcUtils; import org.junit.jupiter.api.Test; +import org.slf4j.MDC; import java.util.Arrays; import java.util.Collections; @@ -252,6 +254,46 @@ void testCallableInterruptedDuringShutdownDoNotFailJob() throws InterruptedExcep assertThat(operatorCoordinatorContext.isJobFailed()).isFalse(); } + @Test + void testCallAsyncCallableRunsWithJobIdInMdc() throws Exception { + final JobID jobId = new JobID(); + final AtomicReference mdcJobIdInCallable = new AtomicReference<>(); + + ManuallyTriggeredScheduledExecutorService manualWorkerExecutor = + new ManuallyTriggeredScheduledExecutorService(); + ManuallyTriggeredScheduledExecutorService manualCoordinatorExecutor = + new ManuallyTriggeredScheduledExecutorService(); + + SourceCoordinatorContext testingContext = + new SourceCoordinatorContext<>( + jobId, + manualCoordinatorExecutor, + manualWorkerExecutor, + new SourceCoordinatorProvider.CoordinatorExecutorThreadFactory( + TEST_OPERATOR_ID.toHexString(), operatorCoordinatorContext), + operatorCoordinatorContext, + new MockSourceSplitSerializer(), + splitSplitAssignmentTracker, + false); + + try { + // The callable runs on the worker executor, which must be job-scoped. + testingContext.callAsync( + () -> { + mdcJobIdInCallable.set(MDC.get(MdcUtils.JOB_ID)); + return null; + }, + (ignored, e) -> {}); + + // triggerAll() runs the queued callable synchronously on this thread. + manualWorkerExecutor.triggerAll(); + + assertThat(mdcJobIdInCallable.get()).isEqualTo(jobId.toHexString()); + } finally { + testingContext.close(); + } + } + @Test void testSupportsIntermediateNoMoreSplits() throws Exception { sourceReady(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProviderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProviderTest.java index 11fefa5b981a4..8be7d66559b0f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProviderTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProviderTest.java @@ -18,6 +18,7 @@ Licensed to the Apache Software Foundation (ASF) under one package org.apache.flink.runtime.source.coordinator; +import org.apache.flink.api.common.JobInfo; import org.apache.flink.api.common.eventtime.WatermarkAlignmentParams; import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.api.connector.source.mocks.MockSource; @@ -123,6 +124,42 @@ void testCallAsyncExceptionFailsJob() throws Exception { "The job did not fail before timeout."); } + @Test + void testCoordinatorThreadNameContainsJobIdentity() throws Exception { + final MockOperatorCoordinatorContext context = + new MockOperatorCoordinatorContext(OPERATOR_ID, NUM_SPLITS); + final RecreateOnResetOperatorCoordinator coordinator = + (RecreateOnResetOperatorCoordinator) provider.create(context); + final JobInfo jobInfo = context.getJobInfo(); + try { + // Starting the coordinator creates the (lazily initialized) coordinator thread. + coordinator.start(); + CommonTestUtils.waitUtil( + () -> findCoordinatorThread(jobInfo) != null, + Duration.ofMinutes(5L), + "The coordinator thread carrying the job identity was not found."); + + final Thread coordinatorThread = findCoordinatorThread(jobInfo); + assertThat(coordinatorThread).isNotNull(); + assertThat(coordinatorThread.getName()) + .startsWith("SourceCoordinator-SourceCoordinatorProviderTest") + .contains(jobInfo.getJobName()) + .contains(jobInfo.getJobId().toHexString()); + } finally { + coordinator.close(); + } + } + + private static Thread findCoordinatorThread(JobInfo jobInfo) { + for (Thread t : Thread.getAllStackTraces().keySet()) { + if (t.getName().startsWith("SourceCoordinator-") + && t.getName().contains(jobInfo.getJobId().toHexString())) { + return t; + } + } + return null; + } + @Test void testCoordinatorExecutorThreadFactoryNewMultipleThread() { SourceCoordinatorProvider.CoordinatorExecutorThreadFactory From 90dcc69fd4e4191f825a1f69b929c567de4ee890 Mon Sep 17 00:00:00 2001 From: Vasudev Kelappassery <150450112+VasShabu@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:32:34 +0100 Subject: [PATCH 23/34] [FLINK-40089][table] Add new JSON_LENGTH built-in function This closes #28688 --- docs/data/sql_functions.yml | 44 ++++ docs/data/sql_functions_zh.yml | 44 ++++ .../reference/pyflink.table/expressions.rst | 1 + flink-python/pyflink/table/expression.py | 62 +++++ .../table/api/internal/BaseExpressions.java | 88 +++++++ .../functions/BuiltInFunctionDefinitions.java | 22 ++ .../codegen/calls/JsonLengthCallGen.java | 136 +++++++++++ .../planner/codegen/ExprCodeGenerator.scala | 3 + .../codegen/calls/BuiltInMethods.scala | 9 + .../planner/codegen/JsonParseReuseTest.java | 35 +++ .../functions/JsonFunctionsITCase.java | 228 +++++++++++++++++- .../table/runtime/functions/SqlJsonUtils.java | 80 ++++++ 12 files changed, 751 insertions(+), 1 deletion(-) create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/codegen/calls/JsonLengthCallGen.java diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml index c6b683c70c9e3..75d8554305101 100644 --- a/docs/data/sql_functions.yml +++ b/docs/data/sql_functions.yml @@ -1223,6 +1223,50 @@ json: -- [{"nested_json":{"value":42}}] JSON_ARRAY(JSON('{"nested_json": {"value": 42}}')) ``` + - sql: JSON_LENGTH(json_doc[, path]) + table: jsonLength(jsonObject[, path]) + description: | + Returns the number of elements in a JSON document, or the length of the value at the specified path if one is provided. + + The input can be a JSON STRING or a VARIANT. Returns NULL if the argument is NULL, the json is invalid, or the path is empty, malformed or does not locate a value. + The path must be a plain path literal such as '$.a.b'. A path carrying a 'lax'/'strict' path mode prefix raises an error. + eg. + -- 2 + JSON_LENGTH('{"1": "hello", "2": "bye bye"}') + + -- 5 + JSON_LENGTH('[1,2,3,4,5]') + + -- 1 + JSON_LENGTH('"hello"') + + -- 1 + JSON_LENGTH('{"1": "hello", "2": "bye bye"}', '$.2') + + -- NULL + JSON_LENGTH('{"1": "hello", "2": "BAD SYNTAX ->"', '$.2') + + -- NULL + JSON_LENGTH('{"1": "hello", "2": "bye bye"}', '$.[') + + -- error: JSON_LENGTH does not support the 'lax'/'strict' path mode prefix + JSON_LENGTH('{"1": "hello", "2": "bye bye"}', 'strict $.1') + + The length is determined as follows: + + - Scalar values (number, string, boolean): has length 1. + - Array: has a length equal to the number of its elements. + - Object: has a length equal to the number of its key-value pairs. + + A wildcard path that matches 2 or more nodes returns NULL. + A NULL result is ambiguous - it means invalid JSON, no match, or a multi-match wildcard. + + Pair JSON_LENGTH with a helper function to handle these cases explicitly: + -- IS JSON separates invalid input from a real result + SELECT CASE WHEN json_doc IS JSON THEN JSON_LENGTH(json_doc) END; + + -- JSON_EXISTS separates an absent path from a present one + SELECT JSON_EXISTS(json_doc, '$.items[*]'), JSON_LENGTH(json_doc, '$.items[*]'); variant: - sql: PARSE_JSON(json_string[, allow_duplicate_keys]) diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml index 998f46d413487..5b332fca088cd 100644 --- a/docs/data/sql_functions_zh.yml +++ b/docs/data/sql_functions_zh.yml @@ -1309,6 +1309,50 @@ json: -- '[[1]]' JSON_ARRAY(JSON_ARRAY(1)) ``` + - sql: JSON_LENGTH(json_doc[, path]) + table: jsonLength(jsonObject[, path]) + description: | + Returns the number of elements in a JSON document, or the length of the value at the specified path if one is provided. + + The input can be a JSON STRING or a VARIANT. Returns NULL if the argument is NULL, the json is invalid, or the path is empty, malformed or does not locate a value. + The path must be a plain path literal such as '$.a.b'. A path carrying a 'lax'/'strict' path mode prefix raises an error. + eg. + -- 2 + JSON_LENGTH('{"1": "hello", "2": "bye bye"}') + + -- 5 + JSON_LENGTH('[1,2,3,4,5]') + + -- 1 + JSON_LENGTH('"hello"') + + -- 1 + JSON_LENGTH('{"1": "hello", "2": "bye bye"}', '$.2') + + -- NULL + JSON_LENGTH('{"1": "hello", "2": "BAD SYNTAX ->"', '$.2') + + -- NULL + JSON_LENGTH('{"1": "hello", "2": "bye bye"}', '$.[') + + -- error: JSON_LENGTH does not support the 'lax'/'strict' path mode prefix + JSON_LENGTH('{"1": "hello", "2": "bye bye"}', 'strict $.1') + + The length is determined as follows: + + - Scalar values (number, string, boolean): has length 1. + - Array: has a length equal to the number of its elements. + - Object: has a length equal to the number of its key-value pairs. + + A wildcard path that matches 2 or more nodes returns NULL. + A NULL result is ambiguous - it means invalid JSON, no match, or a multi-match wildcard. + + Pair JSON_LENGTH with a helper function to handle these cases explicitly: + -- IS JSON separates invalid input from a real result + SELECT CASE WHEN json_doc IS JSON THEN JSON_LENGTH(json_doc) END; + + -- JSON_EXISTS separates an absent path from a present one + SELECT JSON_EXISTS(json_doc, '$.items[*]'), JSON_LENGTH(json_doc, '$.items[*]'); variant: - sql: PARSE_JSON(json_string[, allow_duplicate_keys]) diff --git a/flink-python/docs/reference/pyflink.table/expressions.rst b/flink-python/docs/reference/pyflink.table/expressions.rst index e2aeb34f41107..17a475516b98c 100644 --- a/flink-python/docs/reference/pyflink.table/expressions.rst +++ b/flink-python/docs/reference/pyflink.table/expressions.rst @@ -326,6 +326,7 @@ JSON functions Expression.json_query Expression.json_quote Expression.json_unquote + Expression.json_length value modification functions ---------------------------- diff --git a/flink-python/pyflink/table/expression.py b/flink-python/pyflink/table/expression.py index aa3b952f215f2..d4a9186882f63 100644 --- a/flink-python/pyflink/table/expression.py +++ b/flink-python/pyflink/table/expression.py @@ -2262,6 +2262,68 @@ def json_unquote(self) -> 'Expression': """ return _unary_op("jsonUnquote")(self) + def json_length(self, path=None) -> 'Expression': + """ + Returns the number of elements in a JSON document, or the length of the value at the + specified path if one is provided. + + The input can be a JSON STRING or a VARIANT. Returns None if the argument is None, + the json is invalid, or the path is empty, malformed or does not locate a value. + + The path must be a plain path literal such as '$.a.b'. A path carrying a + 'lax'/'strict' path mode prefix raises an error. + + The length is determined as follows: + + - Scalar values (number, string, boolean) have length 1. + - Arrays have a length equal to the number of their elements. + - Objects have a length equal to the number of their key-value pairs. + + When provided with a path that uses a wildcard and resolves to 2 or more paths, + 'json_length' resolves to None. + + json_length also supports input of the VARIANT type; you can pass the output of + PARSE_JSON into json_length. + + Because a None result can mean several different things (the input is not valid + JSON, the path does not match anything, or a wildcard path matched 2 or more + nodes), it is recommended to pair json_length with a helper function so invalid + input is handled explicitly rather than silently returning None: + + - Without a path, guard the call with is_json to separate malformed input from a + real result. + - With a path, use json_exists to tell "the path is absent" apart from "the path + matched but was ambiguous / matched 2 or more nodes". + + :: + + # returns the length only for valid JSON, otherwise None means "invalid input" + >>> lit("[1,2]").is_json().then(lit("[1,2]").json_length(), null_of(DataTypes.INT())) + + # path is present even when json_length is None due to a multi-match wildcard + >>> lit("{}").json_exists("$.items[*]") + >>> lit("{}").json_length("$.items[*]") + + Examples: + :: + + >>> lit('{"1": "hello", "2": "bye bye"}').json_length() # 2 + >>> lit('[1,2,3,4,5]').json_length() # 5 + >>> lit('"hello"').json_length() # 1 + + >>> lit('{"1": "hello", "2": "bye bye"}').json_length('$.1') # 1 + >>> lit('{"1": [1,2,3], "2": "bye bye"}').json_length('$.1') # 3 + >>> lit('[1,2,3,4,5]').json_length('$[3]') # 1 + + >>> lit('[1,2,3,4,5]').json_length('$[7]') # None + >>> lit('{"1": "bad", "2": "syntax here ->"').json_length('$.1') # None + >>> lit('[1,2,3,4,5]').json_length('$.[') # None + """ + if path is None: + return _unary_op("jsonLength")(self) + else: + return _binary_op("jsonLength")(self, path) + # ---------------------------- value modification functions ----------------------------- def object_update(self, *kv) -> "Expression": diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java index 72a63e8bd082f..53c05ff3c291b 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java @@ -140,6 +140,7 @@ import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.IS_TRUE; import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.IS_VALID_UTF8; import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.JSON_EXISTS; +import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.JSON_LENGTH; import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.JSON_QUERY; import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.JSON_QUOTE; import static org.apache.flink.table.functions.BuiltInFunctionDefinitions.JSON_UNQUOTE; @@ -2525,6 +2526,93 @@ public OutType jsonQuery(String path, JsonQueryWrapper wrappingBehavior) { path, wrappingBehavior, JsonQueryOnEmptyOrError.NULL, JsonQueryOnEmptyOrError.NULL); } + /** + * Returns the number of elements in a JSON document. + * + *

The result is returned as a {@link DataTypes#INT()}. + * + *

See also {@link #jsonLength(String)} for determining the length of the value at a given + * path. + * + *

Examples: + * + * 

{@code
+     * lit("{\"1\": \"hello\", \"2\": \"bye bye\"}").jsonLength() // 2
+     * lit("[1,2,3,4,5]").jsonLength() // 5
+     * lit("\"hello\"").jsonLength() // 1
+     * nullOf(DataTypes.STRING()).jsonLength() // NULL
+     * lit("invalid").jsonLength() // NULL
+     * }
+ * + * @return The number of elements in the JSON document. + */ + public OutType jsonLength() { + return toApiSpecificExpression(unresolvedCall(JSON_LENGTH, toExpr())); + } + + /** + * Returns the number of elements in a JSON document, or the length of the value at the + * specified path if one is provided. + * + *

The input can be a JSON STRING or a VARIANT. Returns {@code NULL} if the argument is + * {@code NULL}, the json is invalid, or the path is empty, malformed or does not locate a + * value. + * + *

The path must be a plain path literal such as {@code '$.a.b'}. A path carrying a {@code + * 'lax'}/{@code 'strict'} path mode prefix raises an error. + * + *

The length is determined as follows: + * + *

    + *
  • Scalar values (number, string, boolean) have length 1. + *
  • Arrays have a length equal to the number of their elements. + *
  • Objects have a length equal to the number of their key-value pairs. + *
+ * + *

When provided with a path that uses a wildcard and resolves to 2 or more paths, {@code + * JSON_LENGTH} resolves to {@code NULL}. + * + *

JSON_LENGTH also supports input of the VARIANT type; you can pass the output of PARSE_JSON + * into JSON_LENGTH. + * + *

Because a {@code NULL} result can mean several different things (the input is not valid + * JSON, the path does not match anything, or a wildcard path matched 2 or more nodes), it is + * recommended to pair {@code JSON_LENGTH} with a helper function so invalid input is handled + * explicitly rather than silently returning {@code NULL}: + * + *

    + *
  • Without a path, guard the call with {@code IS JSON} to separate malformed input from a + * real result. + *
  • With a path, use {@code JSON_EXISTS} to tell "the path is absent" apart from "the path + * matched but was ambiguous / matched 2 or more nodes". + *
+ * + * 
{@code
+     * // returns the length only for valid JSON, otherwise NULL means "invalid input"
+     * lit("[1,2,3]").isJson().then(lit("[1,2,3]").jsonLength(), nullOf(DataTypes.INT()))
+     *
+     * // pathPresent is true even when jsonLength is NULL because of a multi-match wildcard
+     * lit("{}").jsonExists("$.items[*]")
+     * lit("{}").jsonLength("$.items[*]")
+     * }
+ * + *

Examples: + * + * 

{@code
+     * lit("{\"1\": \"hello\", \"2\": \"bye bye\"}").jsonLength("$.1") // 1
+     * lit("{\"1\": [1,2,3], \"2\": \"bye bye\"}").jsonLength("$.1") // 3
+     * lit("[1,2,3,4,5]").jsonLength("$[3]") // 1
+     *
+     * lit("[1,2,3,4,5]").jsonLength("$[7]") // NULL
+     * }
+ * + * @param path JSON path to search for. + * @return The number of elements in the value located at the given path. + */ + public OutType jsonLength(String path) { + return toApiSpecificExpression(unresolvedCall(JSON_LENGTH, toExpr(), valueLiteral(path))); + } + /** * Extracts JSON values from a JSON string. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index dae5f976da5fd..a69cbcbf0c612 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -3082,6 +3082,28 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) .runtimeProvided() .build(); + public static final BuiltInFunctionDefinition JSON_LENGTH = + BuiltInFunctionDefinition.newBuilder() + .name("JSON_LENGTH") + .kind(SCALAR) + .inputTypeStrategy( + or( + sequence(logical(LogicalTypeFamily.CHARACTER_STRING)), + sequence(logical(LogicalTypeRoot.VARIANT)), + sequence( + logical(LogicalTypeFamily.CHARACTER_STRING), + and( + logical(LogicalTypeFamily.CHARACTER_STRING), + LITERAL)), + sequence( + logical(LogicalTypeRoot.VARIANT), + and( + logical(LogicalTypeFamily.CHARACTER_STRING), + LITERAL)))) + .outputTypeStrategy(explicit(DataTypes.INT().nullable())) + .runtimeProvided() + .build(); + // -------------------------------------------------------------------------------------------- // Variant functions // -------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/codegen/calls/JsonLengthCallGen.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/codegen/calls/JsonLengthCallGen.java new file mode 100644 index 0000000000000..fffe3e80d3f74 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/codegen/calls/JsonLengthCallGen.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.codegen.calls; + +import org.apache.flink.table.planner.codegen.CodeGenUtils; +import org.apache.flink.table.planner.codegen.CodeGeneratorContext; +import org.apache.flink.table.planner.codegen.GeneratedExpression; +import org.apache.flink.table.runtime.functions.SqlJsonUtils; +import org.apache.flink.table.types.logical.LogicalType; + +import scala.Option; +import scala.collection.Seq; + +/** + * {@link CallGenerator} for {@code JSON_LENGTH}. + * + *

The JSON input is parsed into a reusable {@link SqlJsonUtils.JsonValueContext} that is shared + * with other JSON functions operating on the same input, so the parse statement is emitted only + * once. When a path argument is present the path-aware {@link BuiltInMethods#JSON_LENGTH_PATH} + * overload is used, otherwise the whole-document {@link BuiltInMethods#JSON_LENGTH} overload. + * + *

The result is nullable: besides propagating a {@code NULL} argument, {@code JSON_LENGTH} + * itself returns {@code NULL} for invalid JSON, a path that matches nothing, or a wildcard path + * that matches two or more nodes. + */ +public class JsonLengthCallGen implements CallGenerator { + + @Override + public GeneratedExpression generate( + CodeGeneratorContext ctx, Seq operands, LogicalType returnType) { + + String inputTerm = operands.apply(0).resultTerm() + ".toString()"; + + // Parse the JSON input into a reusable context. When multiple JSON functions share the + // same input expression the parse statement is emitted only once and reused. + Option existing = + ctx.getReusableInputUnboxingExprs(inputTerm, Integer.MIN_VALUE); + final String parsedVar; + final String parseCode; + if (existing.isDefined()) { + parsedVar = existing.get().resultTerm(); + parseCode = ""; + } else { + parsedVar = CodeGenUtils.newName(ctx, "jsonParsed"); + String typeName = SqlJsonUtils.JsonValueContext.class.getName(); + ctx.addReusableMember(typeName + " " + parsedVar + ";"); + ctx.addReusableInputUnboxingExprs( + inputTerm, + Integer.MIN_VALUE, + new GeneratedExpression(parsedVar, "false", "", null, Option.empty())); + parseCode = + parsedVar + + " = " + + CodeGenUtils.qualifyMethod(BuiltInMethods.JSON_PARSE()) + + "(" + + inputTerm + + ");"; + } + + final String lengthCall; + if (operands.length() > 1) { + lengthCall = + CodeGenUtils.qualifyMethod(BuiltInMethods.JSON_LENGTH_PATH()) + + "(" + + parsedVar + + ", " + + operands.apply(1).resultTerm() + + ".toString())"; + } else { + lengthCall = + CodeGenUtils.qualifyMethod(BuiltInMethods.JSON_LENGTH()) + + "(" + + parsedVar + + ")"; + } + + String resultTypeTerm = CodeGenUtils.boxedTypeTermForType(returnType); + String defaultValue = CodeGenUtils.primitiveDefaultValue(returnType); + String nullTerm = ctx.addReusableLocalVariable("boolean", "isNull"); + String resultTerm = ctx.addReusableLocalVariable(resultTypeTerm, "result"); + + StringBuilder argsNull = new StringBuilder(); + StringBuilder argsCode = new StringBuilder(); + for (int i = 0; i < operands.length(); i++) { + GeneratedExpression operand = operands.apply(i); + if (i > 0) { + argsNull.append(" || "); + } + argsNull.append(operand.nullTerm()); + argsCode.append(operand.code()).append("\n"); + } + + String code = + argsCode + + nullTerm + + " = " + + argsNull + + ";\n" + + resultTerm + + " = " + + defaultValue + + ";\n" + + "if (!" + + nullTerm + + ") {\n" + + parseCode + + "\n" + + resultTerm + + " = " + + lengthCall + + ";\n" + + nullTerm + + " = (" + + resultTerm + + " == null);\n" + + "}\n"; + + return new GeneratedExpression(resultTerm, nullTerm, code, returnType, Option.empty()); + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala index 147fc1bdf36ae..024229f256336 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala @@ -936,6 +936,9 @@ class ExprCodeGenerator( case BuiltInFunctionDefinitions.JSON_STRING => new JsonStringCallGen(call, rexProgram).generate(ctx, operands, resultType) + case BuiltInFunctionDefinitions.JSON_LENGTH => + new JsonLengthCallGen().generate(ctx, operands, resultType) + case BuiltInFunctionDefinitions.INTERNAL_HASHCODE => new HashCodeCallGen().generate(ctx, operands, resultType) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/calls/BuiltInMethods.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/calls/BuiltInMethods.scala index c13f9f21771cd..6d0d695b39ac7 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/calls/BuiltInMethods.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/calls/BuiltInMethods.scala @@ -490,6 +490,15 @@ object BuiltInMethods { classOf[Any] ) + val JSON_LENGTH = + Types.lookupMethod(classOf[SqlJsonUtils], "jsonLength", classOf[SqlJsonUtils.JsonValueContext]) + + val JSON_LENGTH_PATH = Types.lookupMethod( + classOf[SqlJsonUtils], + "jsonLength", + classOf[SqlJsonUtils.JsonValueContext], + classOf[String]) + val JSON_QUERY = Types.lookupMethod( classOf[SqlJsonUtils], "jsonQuery", diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/JsonParseReuseTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/JsonParseReuseTest.java index 61acdd86eea20..9d646ade3aac7 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/JsonParseReuseTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/JsonParseReuseTest.java @@ -370,4 +370,39 @@ void testReuseIsResetPerRowInBatchFusion() { bEnv.executeSql(sql).collect().forEachRemaining(rows::add); assertThat(rows).containsExactlyInAnyOrder(Row.of("account", "42"), Row.of("admin", "30")); } + + @Test + void testTwoJsonLengthCalls() { + final String sql = + "SELECT JSON_LENGTH(json_data), JSON_LENGTH(json_data, '$.address') FROM json_src"; + final List rows = collect(sql); + assertThat(rows).containsExactlyInAnyOrder(Row.of(4, 1), Row.of(4, 1)); + assertThat(countJsonParse(extractGeneratedCode(sql))) + .as("Two JSON_LENGTH calls on the same input should parse once") + .isOne(); + } + + @Test + void testJsonLengthAndJsonValueMixed() { + final String sql = + "SELECT JSON_LENGTH(json_data), JSON_VALUE(json_data, '$.type') FROM json_src"; + final List rows = collect(sql); + assertThat(rows).containsExactlyInAnyOrder(Row.of(4, "account"), Row.of(4, "admin")); + assertThat(countJsonParse(extractGeneratedCode(sql))) + .as("JSON_LENGTH + JSON_VALUE on the same input should parse once") + .isOne(); + } + + @Test + void testJsonLengthAndJsonQueryMixed() { + final String sql = + "SELECT JSON_LENGTH(json_data), JSON_QUERY(json_data, '$.address') FROM json_src"; + final List rows = collect(sql); + assertThat(rows) + .containsExactlyInAnyOrder( + Row.of(4, "{\"city\":\"Munich\"}"), Row.of(4, "{\"city\":\"Berlin\"}")); + assertThat(countJsonParse(extractGeneratedCode(sql))) + .as("JSON_LENGTH + JSON_QUERY on the same input should parse once") + .isOne(); + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java index fb18458fec600..b1fc7ee891a81 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java @@ -84,6 +84,7 @@ Stream getTestSetSpecs() { final List testCases = new ArrayList<>(); testCases.add(jsonExistsSpec()); testCases.add(jsonValueSpec()); + testCases.add(jsonLengthSpec()); testCases.addAll(isJsonSpec()); testCases.addAll(jsonQuerySpec()); testCases.addAll(jsonStringSpec()); @@ -98,6 +99,229 @@ Stream getTestSetSpecs() { return testCases.stream(); } + private static TestSetSpec jsonLengthSpec() { + final String jsonValue = getJsonFromResource("/json/json-exists.json"); + + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_LENGTH) + .onFieldsWithData( + jsonValue, + "{\"a\":1,\"b\":2}", + "[1,2,3]", + "\"abc\"", + "null", + "{", + ((String) null), + "$", + "{\"a\":[true, false, null]}", + "{}", + "[]") + .andDataTypes( + STRING(), STRING(), STRING(), STRING(), STRING(), STRING(), STRING(), + STRING(), STRING(), STRING(), STRING()) + // path exists but resolves to a JSON null literal -> scalar, length 1 + .testResult( + $("f8").jsonLength("$.a[2]"), + "JSON_LENGTH(f8, '$.a[2]')", + 1, + INT().nullable()) + // missing paths on the same document -> NULL + .testResult( + $("f8").jsonLength("$.a[9]"), + "JSON_LENGTH(f8, '$.a[9]')", + null, + INT().nullable()) + .testResult( + $("f8").jsonLength("$.b"), "JSON_LENGTH(f8, '$.b')", null, INT().nullable()) + + // whole document is a JSON null literal: the root path matches it as a scalar, + // anything else does not exist + .testResult($("f4").jsonLength("$"), "JSON_LENGTH(f4, '$')", 1, INT().nullable()) + .testResult( + $("f4").jsonLength("$.a"), "JSON_LENGTH(f4, '$.a')", null, INT().nullable()) + .testResult( + $("f4").jsonLength("$[0]"), + "JSON_LENGTH(f4, '$[0]')", + null, + INT().nullable()) + .testResult( + $("f4").jsonLength("$.*"), "JSON_LENGTH(f4, '$.*')", null, INT().nullable()) + + // malformed, blank and empty paths -> NULL + .testResult( + $("f8").jsonLength("$["), "JSON_LENGTH(f8, '$[')", null, INT().nullable()) + .testResult( + $("f8").jsonLength("$.[]"), + "JSON_LENGTH(f8, '$.[]')", + null, + INT().nullable()) + .testResult( + $("f8").jsonLength(" "), "JSON_LENGTH(f8, ' ')", null, INT().nullable()) + .testResult($("f8").jsonLength(""), "JSON_LENGTH(f8, '')", null, INT().nullable()) + + // the root path on a scalar document behaves like the no-path overload + .testResult($("f3").jsonLength("$"), "JSON_LENGTH(f3, '$')", 1, INT().nullable()) + + // SQL NULL input + .testResult($("f6").jsonLength(), "JSON_LENGTH(f6)", null, INT().nullable()) + + // whole-document length from the existing resource: + .testResult($("f0").jsonLength(), "JSON_LENGTH(f0)", 3, INT().nullable()) + + // basic shapes + .testResult($("f1").jsonLength(), "JSON_LENGTH(f1)", 2, INT().nullable()) + .testResult($("f2").jsonLength(), "JSON_LENGTH(f2)", 3, INT().nullable()) + .testResult($("f3").jsonLength(), "JSON_LENGTH(f3)", 1, INT().nullable()) + + // empty containers -> 0 + .testResult($("f9").jsonLength(), "JSON_LENGTH(f9)", 0, INT().nullable()) + .testResult($("f10").jsonLength(), "JSON_LENGTH(f10)", 0, INT().nullable()) + .testResult($("f9").jsonLength("$"), "JSON_LENGTH(f9, '$')", 0, INT().nullable()) + .testResult($("f10").jsonLength("$"), "JSON_LENGTH(f10, '$')", 0, INT().nullable()) + .testResult($("f4").jsonLength(), "JSON_LENGTH(f4)", 1, INT().nullable()) + + // (valid) paths + .testResult($("f0").jsonLength("$"), "JSON_LENGTH(f0, '$')", 3, INT().nullable()) + .testResult( + $("f0").jsonLength("$.type"), + "JSON_LENGTH(f0, '$.type')", + 1, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.author"), + "JSON_LENGTH(f0, '$.author')", + 2, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.author.address"), + "JSON_LENGTH(f0, '$.author.address')", + 2, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.metadata.tags"), + "JSON_LENGTH(f0, '$.metadata.tags')", + 3, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.metadata.references"), + "JSON_LENGTH(f0, '$.metadata.references')", + 1, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.metadata.references[0]"), + "JSON_LENGTH(f0, '$.metadata.references[0]')", + 2, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.metadata.references[0].url"), + "JSON_LENGTH(f0, '$.metadata.references[0].url')", + 1, + INT().nullable()) + // (invalid) path + .testResult( + $("f0").jsonLength("$.missing"), + "JSON_LENGTH(f0, '$.missing')", + null, + INT().nullable()) + .testResult($("f7").jsonLength(), "JSON_LENGTH(f7)", null, INT().nullable()) + + // invalid JSON -> NULL + .testResult($("f5").jsonLength(), "JSON_LENGTH(f5)", null, INT().nullable()) + + // literal (NOT NULL) arguments must still yield a nullable result + .testResult( + lit("{\"a\":[1,2,3]}").jsonLength("$.b"), + "JSON_LENGTH('{\"a\":[1,2,3]}', '$.b')", + null, + INT().nullable()) + .testResult( + lit("{\"a\":[1,2,3]}").jsonLength("$.a"), + "JSON_LENGTH('{\"a\":[1,2,3]}', '$.a')", + 3, + INT().nullable()) + + // missing path: neither mode throws -> both yield NULL + .testResult( + $("f0").jsonLength("$.author.nope"), + "JSON_LENGTH(f0, '$.author.nope')", + null, + INT().nullable()) + + // WILDCARDS matching MULTIPLE nodes -> NULL + // PARSE_JSON has no Table API equivalent, so this stays SQL-only + .testSqlResult("JSON_LENGTH(PARSE_JSON(f0), '$.*')", null, INT().nullable()) + .testResult( + $("f0").jsonLength("$.*"), "JSON_LENGTH(f0, '$.*')", null, INT().nullable()) + .testResult( + $("f0").jsonLength("$.author.*"), + "JSON_LENGTH(f0, '$.author.*')", + null, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.author.address.*"), + "JSON_LENGTH(f0, '$.author.address.*')", + null, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.metadata.tags[*]"), + "JSON_LENGTH(f0, '$.metadata.tags[*]')", + null, + INT().nullable()) + .testResult( + $("f0").jsonLength("$..name"), + "JSON_LENGTH(f0, '$..name')", + null, + INT().nullable()) + + // deep-scan `$..url` -> single scalar + .testResult( + $("f0").jsonLength("$..url"), + "JSON_LENGTH(f0, '$..url')", + 1, + INT().nullable()) + .testResult( + $("f0").jsonLength("$..address"), + "JSON_LENGTH(f0, '$..address')", + 2, + INT().nullable()) + .testResult( + $("f0").jsonLength("$.metadata.references[*]"), + "JSON_LENGTH(f0, '$.metadata.references[*]')", + 2, + INT().nullable()) + // `$.metadata.references[*].name` -> single scalar) + .testResult( + $("f0").jsonLength("$.metadata.references[*].name"), + "JSON_LENGTH(f0, '$.metadata.references[*].name')", + 1, + INT().nullable()) + // JSON_LENGTH variant support (runtime path, no constant folding) + // PARSE_JSON has no Table API equivalent, so these stay SQL-only + .testSqlResult("JSON_LENGTH(PARSE_JSON(f0))", 3, INT().nullable()) + .testSqlResult("JSON_LENGTH(PARSE_JSON('[1,2,3,4,5]'))", 5, INT().nullable()) + .testSqlResult("JSON_LENGTH(PARSE_JSON('\"hello\"'))", 1, INT().nullable()) + .testSqlResult( + "JSON_LENGTH(PARSE_JSON(f0), '$.metadata.tags')", 3, INT().nullable()) + .testResult( + $("f0").jsonLength("$.items[*]"), + "JSON_LENGTH(f0, '$.items[*]')", + null, + INT().nullable()) + + // lax/strict path modes are not supported and are rejected at runtime + .testSqlRuntimeError( + "JSON_LENGTH(f0, 'strict $.type')", + TableRuntimeException.class, + "JSON_LENGTH does not support the 'lax'/'strict' path mode prefix (got: 'strict $.type').") + .testSqlRuntimeError( + "JSON_LENGTH(f0, 'lax $.type')", + TableRuntimeException.class, + "JSON_LENGTH does not support the 'lax'/'strict' path mode prefix (got: 'lax $.type').") + .testTableApiRuntimeError( + $("f0").jsonLength("strict $.type"), + TableRuntimeException.class, + "JSON_LENGTH does not support the 'lax'/'strict' path mode prefix (got: 'strict $.type')."); + } + private static TestSetSpec jsonExistsSpec() { final String jsonValue = getJsonFromResource("/json/json-exists.json"); return TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_EXISTS) @@ -175,7 +399,9 @@ private static TestSetSpec jsonExistsSpec() { .testTableApiRuntimeError( $("f0").jsonExists("strict $.invalid", JsonExistsOnError.ERROR), TableRuntimeException.class, - "No results for path: $['invalid']"); + "No results for path: $['invalid']") + .testSqlResult("JSON_EXISTS(f0, '$.items[*]')", false, BOOLEAN()) + .testSqlResult("JSON_EXISTS(f0, '$.metadata.tags[*]')", true, BOOLEAN()); } private static TestSetSpec jsonValueSpec() { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/SqlJsonUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/SqlJsonUtils.java index c73e998133996..6b77cc4fa226f 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/SqlJsonUtils.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/SqlJsonUtils.java @@ -32,6 +32,7 @@ import org.apache.flink.shaded.com.jayway.jsonpath.InvalidPathException; import org.apache.flink.shaded.com.jayway.jsonpath.JsonPath; import org.apache.flink.shaded.com.jayway.jsonpath.Option; +import org.apache.flink.shaded.com.jayway.jsonpath.PathNotFoundException; import org.apache.flink.shaded.com.jayway.jsonpath.spi.cache.CacheProvider; import org.apache.flink.shaded.com.jayway.jsonpath.spi.json.JacksonJsonProvider; import org.apache.flink.shaded.com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; @@ -47,6 +48,7 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; +import java.lang.reflect.Array; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -77,6 +79,20 @@ public class SqlJsonUtils { new JacksonJsonProvider(MAPPER); private static final MappingProvider JSON_PATH_MAPPING_PROVIDER = new JacksonMappingProvider(MAPPER); + + /** + * Configuration for JSON_LENGTH, which evaluates plain paths only and therefore does not need + * the 'lax'/'strict' path mode handling of {@link #jsonApiCommonSyntax}. Exceptions are left + * unsuppressed so that a path resolving to a JSON null literal (returns {@code null}) stays + * distinguishable from a path that does not exist (throws {@link PathNotFoundException}). + * {@link Configuration} is immutable, so a single instance is shared across all calls. + */ + private static final Configuration JSON_PATH_LENGTH_CONFIG = + Configuration.builder() + .jsonProvider(JSON_PATH_JSON_PROVIDER) + .mappingProvider(JSON_PATH_MAPPING_PROVIDER) + .build(); + private static final String JSON_QUERY_FUNCTION_NAME = "JSON_QUERY"; private static final String JSON_VALUE_FUNCTION_NAME = "JSON_VALUE"; private static final String JSON_EXISTS_FUNCTION_NAME = "JSON_EXISTS"; @@ -374,6 +390,70 @@ private static Object errorResultForJsonQuery( } } + /** Accepts a pre-parsed context from {@link #jsonParse}. */ + public static Integer jsonLength(final JsonValueContext parsedInput) { + if (parsedInput == null || parsedInput.hasException()) { + return null; + } + + // Whole document: a top-level JSON null literal counts as a scalar (length 1). + return jsonLengthValue(parsedInput.obj); + } + + /** Accepts a pre-parsed context from {@link #jsonParse}. */ + public static Integer jsonLength(final JsonValueContext parsedInput, final String pathSpec) { + // An empty path is ruled out up front because JsonPath rejects it with an + // IllegalArgumentException instead of the InvalidPathException caught below. + if (parsedInput == null || parsedInput.hasException() || pathSpec.isEmpty()) { + return null; + } + + final Matcher matcher = JSON_PATH_BASE.matcher(pathSpec); + final boolean isExplicitLaxStrict = matcher.matches(); + if (isExplicitLaxStrict) { + throw new TableRuntimeException( + String.format( + "JSON_LENGTH does not support the 'lax'/'strict' path mode prefix (got: '%s'). " + + "Use a plain path such as '$.a.b'. To check path existence or handle " + + "invalid input, use JSON_EXISTS or IS JSON.", + pathSpec)); + } + // JsonPath rejects a null root document, so a whole document that is a JSON null literal + // has to be resolved here. Only the root path matches it, as a scalar of length 1. + if (parsedInput.obj == null) { + return "$".equals(pathSpec) ? 1 : null; + } + final Object value; + try { + value = JsonPath.parse(parsedInput.obj, JSON_PATH_LENGTH_CONFIG).read(pathSpec); + } catch (InvalidPathException e) { + // The path does not exist, or is not a valid path at all. + return null; + } + + if (!JsonPath.isPathDefinite(pathSpec)) { + final List matched = (List) value; + return matched.size() == 1 ? jsonLengthValue(matched.get(0)) : null; + } + + // A definite path that read without throwing but produced null matched a JSON null + // literal, which jsonLengthValue counts as a scalar. + return jsonLengthValue(value); + } + + private static int jsonLengthValue(final Object value) { + if (value instanceof Map) { + return ((Map) value).size(); + } else if (value != null && value.getClass().isArray()) { + return Array.getLength(value); + } else if (value instanceof List) { + return ((List) value).size(); + } + + // Scalars, including a JSON null literal, have length 1. + return 1; + } + public static Object json(String input) { try { String trimmed = input.trim(); From 7232ad9c2c4346f92d820d69ef3f192bebb58045 Mon Sep 17 00:00:00 2001 From: xingsuo-zbz Date: Wed, 15 Jul 2026 11:00:42 +0800 Subject: [PATCH 24/34] [FLINK-39984][runtime][webUI] Support LITE/FULL thread dump modes ThreadMXBean.dumpAllThreads(true, true) enters a single JVM-wide safepoint to collect monitor/synchronizer state; on busy JVMs the pause can exceed heartbeat.timeout and cause unnecessary TaskManager failover. - Introduce ThreadDumpMode {LITE, FULL}: LITE = dumpAllThreads(false, false), FULL preserves today's (true, true) behavior. Exposed via an optional query parameter `?mode=lite|full` on the JM/TM thread-dump endpoints. - Add config cluster.thread-dump.default-mode (default FULL to preserve upgrade behavior; LITE recommended for large clusters). - Add a Lite/Full toggle to both Web UI thread-dump pages; selecting a mode does not auto-fetch, the download link tracks the selection. --- .../generated/cluster_configuration.html | 6 ++ .../generated/expert_cluster_section.html | 6 ++ .../generated/rest_v1_dispatcher.html | 20 +++++ docs/static/generated/rest_v1_dispatcher.yml | 22 ++++++ .../flink/configuration/ClusterOptions.java | 18 +++++ .../flink/configuration/ThreadDumpMode.java | 78 +++++++++++++++++++ .../configuration/ClusterOptionsTest.java | 64 +++++++++++++++ .../src/test/resources/rest_api_v1.snapshot | 10 ++- .../job-manager-thread-dump.component.html | 36 +++++++-- .../job-manager-thread-dump.component.less | 22 ++++++ .../job-manager-thread-dump.component.ts | 40 +++++++++- .../task-manager-thread-dump.component.html | 36 +++++++-- .../task-manager-thread-dump.component.less | 25 ++++++ .../task-manager-thread-dump.component.ts | 52 ++++++++++++- .../src/app/services/job-manager.service.ts | 8 +- .../src/app/services/task-manager.service.ts | 18 +++-- .../flink/runtime/dispatcher/Dispatcher.java | 11 ++- .../resourcemanager/ResourceManager.java | 5 +- .../ResourceManagerGateway.java | 7 +- .../cluster/JobManagerThreadDumpHandler.java | 17 +++- .../TaskManagerThreadDumpHandler.java | 17 +++- .../runtime/rest/messages/ThreadDumpInfo.java | 7 +- .../ThreadDumpModeQueryParameter.java | 56 +++++++++++++ .../cluster/JobManagerThreadDumpHeaders.java | 8 +- ...JobManagerThreadDumpMessageParameters.java | 44 +++++++++++ .../TaskManagerThreadDumpHeaders.java | 6 +- ...askManagerThreadDumpMessageParameters.java | 37 +++++++++ .../runtime/taskexecutor/TaskExecutor.java | 15 ++-- .../taskexecutor/TaskExecutorGateway.java | 5 +- .../TaskExecutorGatewayDecoratorBase.java | 6 +- .../apache/flink/runtime/util/JvmUtils.java | 12 +-- .../NonLeaderRetrievalRestfulGateway.java | 4 +- .../runtime/webmonitor/RestfulGateway.java | 5 +- .../DispatcherThreadDumpOffloadTest.java | 3 +- .../utils/TestingResourceManagerGateway.java | 3 +- .../ThreadDumpModeQueryParameterTest.java | 68 ++++++++++++++++ .../TaskExecutorThreadDumpOffloadTest.java | 69 ++++++++++++---- .../TestingTaskExecutorGateway.java | 4 +- .../webmonitor/TestingRestfulGateway.java | 4 +- 39 files changed, 786 insertions(+), 88 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/configuration/ThreadDumpMode.java create mode 100644 flink-core/src/test/java/org/apache/flink/configuration/ClusterOptionsTest.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ThreadDumpModeQueryParameter.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/cluster/JobManagerThreadDumpMessageParameters.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/taskmanager/TaskManagerThreadDumpMessageParameters.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/ThreadDumpModeQueryParameterTest.java diff --git a/docs/layouts/shortcodes/generated/cluster_configuration.html b/docs/layouts/shortcodes/generated/cluster_configuration.html index 1d1dbd87d9cf1..04e605d53e351 100644 --- a/docs/layouts/shortcodes/generated/cluster_configuration.html +++ b/docs/layouts/shortcodes/generated/cluster_configuration.html @@ -62,6 +62,12 @@ Duration The shutdown timeout for cluster services like executors. + +

cluster.thread-dump.default-mode
+ FULL +

Enum

+ Default granularity of the JobManager/TaskManager thread-dump REST endpoint when no explicit mode query parameter is supplied. The default is FULL to preserve historical behavior; operators of large clusters are strongly encouraged to switch to LITE to avoid heartbeat timeouts caused by long safepoint pauses.

Possible values:
  • "LITE": Stack traces only, without lock information. Negligible JVM pause.
  • "FULL": Additionally collects locked monitors and j.u.c. synchronizers, equivalent to jstack -l. Pauses the JVM in a safepoint for a duration that scales with heap size and thread count, which can take seconds on large TaskManagers.
+
cluster.thread-dump.stacktrace-max-depth
50 diff --git a/docs/layouts/shortcodes/generated/expert_cluster_section.html b/docs/layouts/shortcodes/generated/expert_cluster_section.html index f32e45d44d009..879d46790035a 100644 --- a/docs/layouts/shortcodes/generated/expert_cluster_section.html +++ b/docs/layouts/shortcodes/generated/expert_cluster_section.html @@ -26,6 +26,12 @@ Boolean Whether processes should halt on fatal errors instead of performing a graceful shutdown. In some environments (e.g. Java 8 with the G1 garbage collector), a regular graceful shutdown can lead to a JVM deadlock. See FLINK-16510 for details. + +
cluster.thread-dump.default-mode
+ FULL +

Enum

+ Default granularity of the JobManager/TaskManager thread-dump REST endpoint when no explicit mode query parameter is supplied. The default is FULL to preserve historical behavior; operators of large clusters are strongly encouraged to switch to LITE to avoid heartbeat timeouts caused by long safepoint pauses.

Possible values:
  • "LITE": Stack traces only, without lock information. Negligible JVM pause.
  • "FULL": Additionally collects locked monitors and j.u.c. synchronizers, equivalent to jstack -l. Pauses the JVM in a safepoint for a duration that scales with heap size and thread count, which can take seconds on large TaskManagers.
+
cluster.thread-dump.stacktrace-max-depth
50 diff --git a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html index 98c02d7a418fc..26749b00f691b 100644 --- a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html +++ b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html @@ -1383,6 +1383,16 @@ Returns the thread dump of the JobManager. + + Query parameters + + + +
    +
  • mode (optional): Controls how much lock information is collected. Supported values: [LITE, FULL]. When omitted, cluster.thread-dump.default-mode is used.
    • +
+ +