diff --git a/.github/actions/last_workflow_run/action.yml b/.github/actions/last_workflow_run/action.yml new file mode 100644 index 00000000000000..c59bebbe2ed5b6 --- /dev/null +++ b/.github/actions/last_workflow_run/action.yml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: "Finds the most recent run of a workflow on a branch" +description: "Queries the GitHub Actions API for the most recent run of a given workflow on a given branch, optionally filtered by status, and exposes its head SHA and conclusion." +inputs: + workflow_id: + description: "Workflow file name, e.g. ci.yml" + required: true + branch: + description: "Branch name to query" + required: true + status: + description: "Optional run status filter, e.g. success. Leave empty to match any status." + required: false + default: "" +outputs: + sha: + description: "Head SHA of the matched run, or empty string if none found" + value: ${{ steps.resolve.outputs.sha }} + conclusion: + description: "Conclusion of the matched run, or empty string if none found" + value: ${{ steps.resolve.outputs.conclusion }} +runs: + using: "composite" + steps: + - name: "Query workflow runs" + id: resolve + uses: actions/github-script@v7 + with: + script: | + const workflowId = "${{ inputs.workflow_id }}"; + const branch = "${{ inputs.branch }}"; + const status = "${{ inputs.status }}"; + + const params = { + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: workflowId, + branch: branch, + per_page: 1 + }; + if (status) { + params.status = status; + } + + const { data } = await github.rest.actions.listWorkflowRuns(params); + const run = data.workflow_runs[0]; + + core.setOutput('sha', run?.head_sha ?? ''); + core.setOutput('conclusion', run?.conclusion ?? ''); diff --git a/.github/workflows/community-review.yml b/.github/workflows/community-review.yml index 5785f180af229b..2b27405cb0f529 100644 --- a/.github/workflows/community-review.yml +++ b/.github/workflows/community-review.yml @@ -38,7 +38,7 @@ jobs: if: github.repository_owner == 'apache' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 - run: | chmod +x ${{ github.workspace }}/.github/workflows/community-review.sh - name: Run community review script to set labels diff --git a/.github/workflows/docs-legacy.yml b/.github/workflows/docs-legacy.yml index 4a9bd5790dddc4..be138e54966f9c 100644 --- a/.github/workflows/docs-legacy.yml +++ b/.github/workflows/docs-legacy.yml @@ -47,7 +47,7 @@ jobs: if: github.repository == 'apache/flink' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: ref: ${{ inputs.branch }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1bdb6b97ac13ad..a91095ee897f30 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,7 +39,7 @@ jobs: - release-1.20 - release-1.19 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v7 with: ref: ${{ matrix.branch }} diff --git a/.github/workflows/nightly-trigger.yml b/.github/workflows/nightly-trigger.yml index 3605cf8bb67bd8..682b66e42633d7 100644 --- a/.github/workflows/nightly-trigger.yml +++ b/.github/workflows/nightly-trigger.yml @@ -39,6 +39,19 @@ jobs: - release-1.20 runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v7 + with: + sparse-checkout: | + .github/actions/last_workflow_run + + - name: "Resolve last nightly run" + id: last-nightly + uses: "./.github/actions/last_workflow_run" + with: + workflow_id: "nightly.yml" + branch: ${{ matrix.branch }} + - name: Trigger Workflow uses: actions/github-script@v7 with: @@ -55,17 +68,8 @@ jobs: // Compare SHA from last nightly against current // if it is same, then no need to run nightly for the same SHA again. - const { data: runsData } = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: 'nightly.yml', - branch: branch, - per_page: 1 - }); - - const lastRun = runsData.workflow_runs[0]; - const lastBuiltSha = lastRun?.head_sha; - const lastConclusion = lastRun?.conclusion; + const lastBuiltSha = '${{ steps.last-nightly.outputs.sha }}' || undefined; + const lastConclusion = '${{ steps.last-nightly.outputs.conclusion }}' || undefined; // Skip the scheduled run only if there are no new commits AND the // previous nightly was green. If the last run failed/was cancelled, diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a7a56ae9506982..70c6f8d09e606e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -90,7 +90,7 @@ jobs: os_name: macos steps: - name: "Checkout the repository" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/template.flink-ci.yml b/.github/workflows/template.flink-ci.yml index 532263ee7cdd38..38fde30363367c 100644 --- a/.github/workflows/template.flink-ci.yml +++ b/.github/workflows/template.flink-ci.yml @@ -83,7 +83,7 @@ jobs: stringified-workflow-name: ${{ steps.workflow-prep-step.outputs.stringified_value }} steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false @@ -145,7 +145,7 @@ jobs: steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false sparse-checkout: | @@ -220,7 +220,7 @@ jobs: steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false sparse-checkout: | @@ -370,7 +370,7 @@ jobs: steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false sparse-checkout: | diff --git a/.github/workflows/template.pre-compile-checks.yml b/.github/workflows/template.pre-compile-checks.yml index faf37dad246f16..8975f5c7b842fa 100644 --- a/.github/workflows/template.pre-compile-checks.yml +++ b/.github/workflows/template.pre-compile-checks.yml @@ -50,7 +50,7 @@ jobs: steps: - name: "Flink Checkout" - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: persist-credentials: false @@ -59,16 +59,33 @@ jobs: with: jdk_version: ${{ inputs.jdk_version }} - - name: "Checkstyle" - uses: "./.github/actions/run_mvn" + - name: "Resolve last green commit for spotless ratchet" + id: last-green + uses: "./.github/actions/last_workflow_run" with: - maven-parameters: "checkstyle:check -T1C" + workflow_id: "ci.yml" + branch: ${{ github.ref_name }} + status: "success" - - name: "Spotless" - if: (success() || failure()) + - name: "Fetch last green commit" + if: steps.last-green.outputs.sha != '' + shell: bash + run: | + sha="${{ steps.last-green.outputs.sha }}" + if git -c safe.directory='*' fetch --depth=1 origin "${sha}" \ + && git -c safe.directory='*' cat-file -e "${sha}^{commit}"; then + echo "RATCHET_SHA=${sha}" >> "${GITHUB_ENV}" + echo "Ratcheting spotless from ${sha}" + else + echo "Could not fetch ${sha}; running full spotless check." + fi + + - name: "Checkstyle & Spotless" uses: "./.github/actions/run_mvn" with: - maven-parameters: "spotless:check -T1C" + maven-parameters: >- + checkstyle:check spotless:check -T1C -fae + ${{ env.RATCHET_SHA && format('-Dspotless.ratchetFrom={0}', env.RATCHET_SHA) || '' }} - name: "License Headers" if: (success() || failure()) diff --git a/docs/content.zh/docs/deployment/advanced/historyserver.md b/docs/content.zh/docs/deployment/advanced/historyserver.md index 748dbd867fc456..21ef3728c23002 100644 --- a/docs/content.zh/docs/deployment/advanced/historyserver.md +++ b/docs/content.zh/docs/deployment/advanced/historyserver.md @@ -98,6 +98,24 @@ HistoryServer 支持通过 `historyserver.archive.storage.type` 选择本地存 historyserver.archive.storage.type: ROCKSDB ``` +**加载模式** + +HistoryServer 支持两种加载模式，通过 `historyserver.archive.load.mode` 配置： + +* `EAGER`（默认）：通过周期性的后台刷新，自动、同步地将存档下载到本地存储。 +* `LAZY`：存档会立即展示在 Web 界面上，底层数据则在后台异步拉取。如果某个存档在其后台下载完成之前被访问，系统会优先按需拉取该存档。 + +启用懒加载模式示例： + +```yaml +historyserver.archive.load.mode: LAZY +``` + +在 `LAZY` 模式下，按需拉取使用两个线程池： + +* `historyserver.lazy.fetch.executor.common.pool-size` —— 用于常规后台存档拉取的共享线程池的大小。 +* `historyserver.lazy.fetch.executor.individual.pool-size` —— 专用于按需拉取单个存档（例如用户访问某个存档时）的高优先级线程池的大小。 + ## 日志集成 Flink 本身并不提供已完成作业的日志收集功能。 diff --git a/docs/content.zh/docs/sql/functions/built-in-functions.md b/docs/content.zh/docs/sql/functions/built-in-functions.md index 4aaf786e860c97..6165b239fcbbe3 100644 --- a/docs/content.zh/docs/sql/functions/built-in-functions.md +++ b/docs/content.zh/docs/sql/functions/built-in-functions.md @@ -132,6 +132,24 @@ JSON 函数使用符合 ISO/IEC TR 19075-6 SQL标准的 JSON 路径表达式。 {{< sql_functions_zh "bitmapagg" >}} +Table Functions +--------------- + +Table functions take zero, one, or more values as input and return multiple rows (a table) as the result. Most built-in table functions take a table as an input argument. +Table functions can be used in two ways: as stand-alone inputs, where they are invoked just once, or in a `LATERAL` context, where they are invoked for each row of an outer table. + +| Function | Description | +|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `TUMBLE(data => TABLE t, ...)` | Assigns each row of the `data` table to a tumbling window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#tumble) for the full list of arguments, semantics, and usage. | +| `HOP(data => TABLE t, ...)` | Assigns each row of the `data` table to a hopping window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#hop) for the full list of arguments, semantics, and usage. | +| `CUMULATE(data => TABLE t, ...)` | Assigns each row of the `data` table to a cumulating window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#cumulate) for the full list of arguments, semantics, and usage. | +| `SESSION(data => TABLE t, ...)` | Assigns each row of the `data` table to a session window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#session) for the full list of arguments, semantics, and usage. | +| `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See Changelog Conversion for the full list of arguments, semantics, and usage. | +| `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See Changelog Conversion for the full list of arguments, semantics, and usage. | +| `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | + +To implement your own table functions, see [user-defined table functions]({{< ref "docs/dev/table/functions/udfs" >}}#table-functions). + 时间间隔单位和时间点单位标识符 --------------------------------------- diff --git a/docs/content.zh/docs/sql/reference/queries/joins.md b/docs/content.zh/docs/sql/reference/queries/joins.md index a1e1db2caab8be..a97312e910082b 100644 --- a/docs/content.zh/docs/sql/reference/queries/joins.md +++ b/docs/content.zh/docs/sql/reference/queries/joins.md @@ -295,6 +295,113 @@ WHERE - SQL 中可以定义 temporal table DDL，但不能定义 temporal table 函数; - temporal table DDL 和 temporal table function 都支持 temporal join 版本表，但只有 temporal table function 可以 temporal join 任何表/视图的最新版本（即"处理时间 Temporal Join"）。 +LATERAL SNAPSHOT Join +-------------- + +{{< label Streaming >}} {{< label Batch >}} + +A `LATERAL SNAPSHOT` join is a *stream enrichment* join that augments an append-only table with the current state of an updating table. +As in the [temporal joins](#temporal-joins), the enriched (left) input is called the *probe side* and the enriching (right) input is called the *build side*. +Every probe-side row is joined with the build-side state that is current at the time the row is processed. + +For example, the following query enriches an append-only stream of `orders` (the probe side) with the conversion rate from an updating `currency_rates` table (the build side) that is current when the order is processed: + +```sql +-- probe side: append-only stream of orders +-- order_id | currency | amount | order_time +-- ---------+----------+--------+----------- +-- 1 | EUR | 10 | 10:15 +-- 2 | EUR | 10 | 10:31 +-- 3 | USD | 20 | 11:00 + +-- build side: updating currency rates (upsert on currency) +-- currency | rate | update_time +-- ---------+------+------------ +-- EUR | 1.1 | 10:00 +-- USD | 1.0 | 10:00 +-- EUR | 1.2 | 10:30 + +SELECT o.order_id, o.currency, o.amount, r.rate +FROM orders AS o +JOIN LATERAL TABLE(SNAPSHOT(input => TABLE currency_rates)) AS r +ON o.currency = r.currency; + +order_id currency amount rate +======== ======== ====== ==== + 1 EUR 10 1.1 -- rate as of 10:00 + 2 EUR 10 1.2 -- rate as of 10:30 + 3 USD 20 1.0 -- rate as of 10:00 +``` + +*Important:* The `LATERAL SNAPSHOT` join is non-deterministic. The order with `order_id = 2` could also have been joined with the `10:00` version of `EUR`. The result depends on the order in which the operator processes its inputs, which cannot be controlled. + +**When to use it** + +The `LATERAL SNAPSHOT` join is designed for enrichment scenarios where the other temporal joins are a poor fit: + +- **The build side does not receive continuous updates.** An event-time temporal join only emits a joined row once the combined watermark of both inputs has passed the event time of the probe-side row. A build side that does not continuously produce records, and therefore does not advance its watermark, stalls the join and lets probe-side state accumulate. A `LATERAL SNAPSHOT` join keeps making progress even when the build side is idle. +- **Low latency is required.** An event-time temporal join holds back probe-side rows until the watermark catches up, which adds latency. After the initial load phase, the `LATERAL SNAPSHOT` join immediately joins a probe-side row when it arrives. +- **The build side has no primary key.** Event-time and processing-time temporal joins require the build side to have a primary key that appears in the equi-join condition. A `LATERAL SNAPSHOT` join has no such requirement; neither the probe side nor the build side needs a primary key. + +**How the join works** + +A `LATERAL SNAPSHOT` join operates in two phases to avoid joining probe-side rows against an incomplete build side: it first loads the build side up to a well-defined point in time (the *load phase*) before it starts joining (the *join phase*). + +During the *load phase*, the operator accumulates the build-side changes into state until the load-completion condition is met, without emitting any results yet. Probe-side rows that arrive during the load phase are buffered. The load phase completes when one of the following occurs: + +- the build-side watermark reaches a configured `load_completed_time`. This time is either explicitly set by the user (`load_completed_condition => 'user_time'`) or automatically set to the wall-clock time when the query is compiled (`load_completed_condition => 'compile_time'`), or +- as a fallback, the `load_completed_idle_timeout` elapses in processing time without the build-side watermark advancing (which handles build sides that become idle during start-up). + +When the load phase completes, the operator transitions to the *join phase*: all buffered probe-side rows are joined against the current build-side state and emitted. +From then on, each probe-side row is joined and emitted immediately against the build-side state that is current at that moment. +Build-side updates continue to be applied to the state and become visible to subsequent probe-side rows. + +The load phase is what distinguishes a `LATERAL SNAPSHOT` join from the [processing-time temporal join](#processing-time-temporal-join), which has been disabled for Flink SQL. The processing-time temporal join starts joining immediately at query start, so early probe-side rows are joined against whatever build-side data happens to have been loaded so far, producing missing or stale results that depend on the order in which the inputs are read. By first loading the build side, a `LATERAL SNAPSHOT` join avoids this problem. + +**Syntax** + +The build side is wrapped in the `SNAPSHOT` table function inside a `LATERAL TABLE` clause. The outer (probe-side) table must be an append-only table. +Both `INNER JOIN` and `LEFT [OUTER] JOIN` are supported. The join requires at least one conjunctive equality predicate; additional non-equi predicates are allowed in the `ON` clause. + +```sql +SELECT [column_list] +FROM probe_table +[LEFT] JOIN LATERAL TABLE( + SNAPSHOT( + input => TABLE build_table, + [ load_completed_condition => <'compile_time' | 'user_time'>, ] + [ load_completed_time => , ] + [ load_completed_idle_timeout => , ] + [ state_ttl => ])) AS s +ON probe_table.col = s.col +``` + +The `SNAPSHOT` function accepts the following arguments: + +| Argument | Type | Required | Description | +| --- | --- | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `input` | TABLE | yes | The build-side table. It may use any changelog mode (inserts, updates, and deletes). In streaming mode it must declare a [watermark]({{< ref "docs/concepts/sql-table-concepts/time_attributes" >}}#event-time). | +| `load_completed_condition` | STRING | no | Determines when the initial load phase completes. One of `'compile_time'` (default) or `'user_time'`. With `'compile_time'`, the load phase completes once the build-side watermark reaches the wall-clock time at which the query was compiled. With `'user_time'`, it completes once the build-side watermark reaches the explicit `load_completed_time`. | +| `load_completed_time` | TIMESTAMP_LTZ(3) | no | The build-side event time that completes the load phase. Required when `load_completed_condition` is `'user_time'` and must not be set otherwise. | +| `load_completed_idle_timeout` | INTERVAL | no | A processing-time fallback to complete the load phase. The transition to the join phase happens when the build-side watermark does not advance for more than the configured interval. | +| `state_ttl` | INTERVAL | no | Retention time for build-side state. Join keys that are not accessed within this duration become eligible for eviction. Only applied during the join phase. Defaults to the pipeline's [state TTL]({{< ref "docs/dev/table/config" >}}#table-exec-state-ttl). | + +`load_completed_condition`, `load_completed_time`, `load_completed_idle_timeout`, and `state_ttl` only affect streaming execution and are ignored in batch mode (see **Batch mode** below). + +**Result and state characteristics** + +The result is append-only and preserves the probe-side time attributes. A build-side rowtime attribute that is projected into the output is materialized as a regular `TIMESTAMP` and is no longer a time attribute. Probe-side watermarks are forwarded downstream during the join phase; build-side watermarks are consumed internally and are not propagated. + +Because rows are joined against the build-side state that is current at processing time, the result is **not deterministic**. A given probe-side row may be joined with different build-side versions across different runs, depending on the relative timing of the two inputs. Probe and build-side inputs can be configured with watermark alignment to keep the two inputs roughly aligned on event time, so that a probe-side row tends to be joined with build-side changes of a similar event time. This is a best-effort alignment and does not make the result deterministic. + +The build-side state grows with the number of distinct build-side keys, and during the load phase the buffered probe-side rows add to the state footprint until the operator transitions to the join phase. Use `state_ttl` to bound the build-side state for keys that are no longer updated or joined. You can reduce the amount of data that is buffered and processed during the load phase by configuring scan start offsets on the build and probe-side inputs, for example with a `scan.startup.*` [dynamic table option hint]({{< ref "docs/sql/reference/queries/hints" >}}#dynamic-table-options). + +**Batch mode** + +In batch mode, a `LATERAL SNAPSHOT` join is executed as a regular (`INNER` or `LEFT`) join between the probe side and the complete build side. Batch execution reads the entire build side before joining, so there is no load phase and no incremental state build-up. The streaming-specific arguments (`load_completed_condition`, `load_completed_time`, `load_completed_idle_timeout`, and `state_ttl`) are accepted but have no effect, and the build side does not need to declare a watermark. + +Because every probe-side row is joined against the final, complete build side, the batch result is **deterministic**. + Lookup Join -------------- diff --git a/docs/content.zh/release-notes/flink-2.3.md b/docs/content.zh/release-notes/flink-2.3.md index 8ddc99dbd9245f..06635d5f6f9ef3 100644 --- a/docs/content.zh/release-notes/flink-2.3.md +++ b/docs/content.zh/release-notes/flink-2.3.md @@ -90,17 +90,11 @@ CREATE FUNCTION my_func AS 'com.example.MyUdf' Flink 2.3 reworks how `SinkUpsertMaterializer` handles the case where a query's upsert key differs from the sink's primary key. Previously this required maintaining the full history of -records and could blow up state. Two changes address this: +records and could blow up state. An enhancement allows you to manage this situation: - A new `ON CONFLICT` clause with `DO NOTHING`, `DO ERROR` and `DO DEDUPLICATE` strategies makes the behavior on key conflict explicit. By default, planning now fails when the upsert and primary keys differ, requiring the user to choose a conflict strategy. -- Watermark-based record compaction is introduced to fix internal changelog disorder. The - trigger and frequency of compaction are controlled by: - - `table.exec.sink.upserts.compaction-mode` (default: `WATERMARK`) — `WATERMARK` or - `CHECKPOINT`. - - `table.exec.sink.upserts.compaction-interval` — optional fallback interval for emitting - watermarks when none arrive naturally. #### Process Table Function enhancements diff --git a/docs/content/docs/deployment/advanced/historyserver.md b/docs/content/docs/deployment/advanced/historyserver.md index 607f08b826b07e..842b73e3a0b6b6 100644 --- a/docs/content/docs/deployment/advanced/historyserver.md +++ b/docs/content/docs/deployment/advanced/historyserver.md @@ -92,6 +92,24 @@ Example for enabling the RocksDB backend: historyserver.archive.storage.type: ROCKSDB ``` +**Archive Load Mode** + +The HistoryServer supports two modes for loading archives, selected via `historyserver.archive.load.mode`: + +* `EAGER` (default): Archives are automatically and synchronously downloaded to local storage via periodic background refreshes. +* `LAZY`: Archives are displayed on the web interface immediately, while the underlying data is fetched asynchronously in the background. If a specific archive is accessed before its background download completes, the system will prioritize and fetch it on demand. + +Example for enabling the lazy loading: + +```yaml +historyserver.archive.load.mode: LAZY +``` + +In `LAZY` mode, on-demand fetches use two thread pools: + +* `historyserver.lazy.fetch.executor.common.pool-size` — The size of the shared thread pool used for routine, background archive fetching. +* `historyserver.lazy.fetch.executor.individual.pool-size` — The size of the high-priority thread pool dedicated to fetching individual archives on demand (e.g., when a user accesses them). + ## Log Integration Flink does not provide built-in methods for archiving logs of completed jobs. diff --git a/docs/content/docs/sql/functions/built-in-functions.md b/docs/content/docs/sql/functions/built-in-functions.md index 42808103674bb9..3946cd225abee5 100644 --- a/docs/content/docs/sql/functions/built-in-functions.md +++ b/docs/content/docs/sql/functions/built-in-functions.md @@ -135,6 +135,24 @@ The aggregate functions take an expression across all the rows as the input and {{< sql_functions "bitmapagg" >}} +Table Functions +--------------- + +Table functions take zero, one, or more values as input and return multiple rows (a table) as the result. Most built-in table functions take a table as an input argument. +Table functions can be used in two ways: as stand-alone inputs, where they are invoked just once, or in a `LATERAL` context, where they are invoked for each row of an outer table. + +| Function | Description | +|--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `TUMBLE(data => TABLE t, ...)` | Assigns each row of the `data` table to a tumbling window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#tumble) for the full list of arguments, semantics, and usage. | +| `HOP(data => TABLE t, ...)` | Assigns each row of the `data` table to a hopping window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#hop) for the full list of arguments, semantics, and usage. | +| `CUMULATE(data => TABLE t, ...)` | Assigns each row of the `data` table to a cumulating window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#cumulate) for the full list of arguments, semantics, and usage. | +| `SESSION(data => TABLE t, ...)` | Assigns each row of the `data` table to a session window specified by additional window columns (`window_start`, `window_end`, `window_time`). See [Window TVF]({{< ref "docs/sql/reference/queries/window-tvf" >}}#session) for the full list of arguments, semantics, and usage. | +| `FROM_CHANGELOG(input => TABLE t [, ...])` | Converts an append-only table with an explicit operation column into a dynamic table. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#from_changelog) for the full list of arguments, semantics, and usage. | +| `TO_CHANGELOG(input => TABLE t [, ...])` | Converts a dynamic table into an append-only table with an explicit operation column. See [Changelog Conversion]({{< ref "docs/sql/reference/queries/changelog" >}}#to_changelog) for the full list of arguments, semantics, and usage. | +| `SNAPSHOT(input => TABLE t [, ...])` | Returns the current state of a dynamic table `t`. `SNAPSHOT` can only be used in a `LATERAL` context and not as a stand-alone table function. See [LATERAL SNAPSHOT join]({{< ref "docs/sql/reference/queries/joins" >}}#lateral-snapshot-join) for the full list of arguments, the join semantics, and usage. | + +To implement your own table functions, see [user-defined table functions]({{< ref "docs/dev/table/functions/udfs" >}}#table-functions). + Time Interval and Point Unit Specifiers --------------------------------------- diff --git a/docs/content/docs/sql/reference/queries/joins.md b/docs/content/docs/sql/reference/queries/joins.md index d000a46b822ceb..c323554d37ba45 100644 --- a/docs/content/docs/sql/reference/queries/joins.md +++ b/docs/content/docs/sql/reference/queries/joins.md @@ -300,6 +300,117 @@ The main difference between above Temporal Table DDL and Temporal Table Function - The temporal table DDL can be defined in SQL but temporal table function can not; - Both temporal table DDL and temporal table function support temporal join versioned table, but only temporal table function can temporal join the latest version of any table/view. +LATERAL SNAPSHOT Join +-------------- + +{{< label Streaming >}} {{< label Batch >}} + +A `LATERAL SNAPSHOT` join is a *stream enrichment* join that augments an append-only table with the current state of an updating table. +As in the [temporal joins](#temporal-joins), the enriched (left) input is called the *probe side* and the enriching (right) input is called the *build side*. +Every probe-side row is joined with the build-side state that is current at the time the row is processed. + +For example, the following query enriches an append-only stream of `orders` (the probe side) with the conversion rate from an updating `currency_rates` table (the build side) that is current when the order is processed: + +```sql +-- probe side: append-only stream of orders +-- order_id | currency | amount | order_time +-- ---------+----------+--------+----------- +-- 1 | EUR | 10 | 10:15 +-- 2 | EUR | 10 | 10:31 +-- 3 | USD | 20 | 11:00 + +-- build side: updating currency rates (upsert on currency) +-- currency | rate | update_time +-- ---------+------+------------ +-- EUR | 1.1 | 10:00 +-- USD | 1.0 | 10:00 +-- EUR | 1.2 | 10:30 + +SELECT o.order_id, o.currency, o.amount, r.rate +FROM orders AS o +JOIN LATERAL TABLE(SNAPSHOT(input => TABLE currency_rates)) AS r +ON o.currency = r.currency; + +order_id currency amount rate +======== ======== ====== ==== + 1 EUR 10 1.1 -- rate as of 10:00 + 2 EUR 10 1.2 -- rate as of 10:30 + 3 USD 20 1.0 -- rate as of 10:00 +``` + +*Important:* The `LATERAL SNAPSHOT` join is non-deterministic. The order with `order_id = 2` could also have been joined with the `10:00` version of `EUR`. The result depends on the order in which the operator processes its inputs, which cannot be controlled. + +**When to use it** + +The `LATERAL SNAPSHOT` join is designed for enrichment scenarios where the other temporal joins are a poor fit: + +- **The build side does not receive continuous updates.** An event-time temporal join only emits a joined row once the combined watermark of both inputs has passed the event time of the probe-side row. A build side that does not continuously produce records, and therefore does not advance its watermark, stalls the join and lets probe-side state accumulate. A `LATERAL SNAPSHOT` join keeps making progress even when the build side is idle. +- **Low latency is required.** An event-time temporal join holds back probe-side rows until the watermark catches up, which adds latency. After the initial load phase, the `LATERAL SNAPSHOT` join immediately joins a probe-side row when it arrives. +- **The build side has no primary key.** Event-time and processing-time temporal joins require the build side to have a primary key that appears in the equi-join condition. A `LATERAL SNAPSHOT` join has no such requirement; neither the probe side nor the build side needs a primary key. + +**How the join works** + +A `LATERAL SNAPSHOT` join operates in two phases to avoid joining probe-side rows against an incomplete build side: +it first loads the build side up to a well-defined point in time (the *load phase*) before it starts joining (the *join phase*). + +During the *load phase*, the operator accumulates the build-side changes into state until the load-completion condition is met, without emitting any results yet. +Probe-side rows that arrive during the load phase are buffered. The load phase completes when one of the following occurs: + +- the build-side watermark reaches a configured `load_completed_time`. This time is either explicitly set by the user (`load_completed_condition => 'user_time'`) or automatically set to the wall-clock time when the query is compiled (`load_completed_condition => 'compile_time'`), or +- as a fallback, the `load_completed_idle_timeout` elapses in processing time without the build-side watermark advancing (which handles build sides that become idle during start-up). + +When the load phase completes, the operator transitions to the *join phase*: all buffered probe-side rows are joined against the current build-side state and emitted. +From then on, each probe-side row is joined and emitted immediately against the build-side state that is current at that moment. +Build-side updates continue to be applied to the state and become visible to subsequent probe-side rows. + +The load phase is what distinguishes a `LATERAL SNAPSHOT` join from the [processing-time temporal join](#processing-time-temporal-join), which has been disabled for Flink SQL. +The processing-time temporal join starts joining immediately at query start, so early probe-side rows are joined against whatever build-side data happens to have been loaded so far, producing missing or stale results that depend on the order in which the inputs are read. +By first loading the build side, a `LATERAL SNAPSHOT` join avoids this problem. + +**Syntax** + +The build side is wrapped in the `SNAPSHOT` table function inside a `LATERAL TABLE` clause. The outer (probe-side) table must be an append-only table. +Both `INNER JOIN` and `LEFT [OUTER] JOIN` are supported. The join requires at least one conjunctive equality predicate; additional non-equi predicates are allowed in the `ON` clause. + +```sql +SELECT [column_list] +FROM probe_table +[LEFT] JOIN LATERAL TABLE( + SNAPSHOT( + input => TABLE build_table, + [ load_completed_condition => <'compile_time' | 'user_time'>, ] + [ load_completed_time => , ] + [ load_completed_idle_timeout => , ] + [ state_ttl => ])) AS s +ON probe_table.col = s.col +``` + +The `SNAPSHOT` function accepts the following arguments: + +| Argument | Type | Required | Description | +| --- | --- | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `input` | TABLE | yes | The build-side table. It may use any [changelog mode]({{< ref "docs/sql/reference/queries/changelog" >}}) (inserts, updates, and deletes). In streaming mode it must declare a [watermark]({{< ref "docs/concepts/sql-table-concepts/time_attributes" >}}#event-time). | +| `load_completed_condition` | STRING | no | Determines when the initial load phase completes. One of `'compile_time'` (default) or `'user_time'`. With `'compile_time'`, the load phase completes once the build-side watermark reaches the wall-clock time at which the query was compiled. With `'user_time'`, it completes once the build-side watermark reaches the explicit `load_completed_time`. | +| `load_completed_time` | TIMESTAMP_LTZ(3) | no | The build-side event time that completes the load phase. Required when `load_completed_condition` is `'user_time'` and must not be set otherwise. | +| `load_completed_idle_timeout` | INTERVAL | no | A processing-time fallback to complete the load phase. The transition to the join phase happens when the build-side watermark does not advance for more than the configured interval. | +| `state_ttl` | INTERVAL | no | Retention time for build-side state. Join keys that are not accessed within this duration become eligible for eviction. Only applied during the join phase. Defaults to the pipeline's [state TTL]({{< ref "docs/dev/table/config" >}}#table-exec-state-ttl). | + +`load_completed_condition`, `load_completed_time`, `load_completed_idle_timeout`, and `state_ttl` only affect streaming execution and are ignored in batch mode (see **Batch mode** below). + +**Result and state characteristics** + +The result is append-only and preserves the probe-side time attributes. A build-side rowtime attribute that is projected into the output is materialized as a regular `TIMESTAMP` and is no longer a time attribute. Probe-side watermarks are forwarded downstream during the join phase; build-side watermarks are consumed internally and are not propagated. + +Because rows are joined against the build-side state that is current at processing time, the result is **not deterministic**. A given probe-side row may be joined with different build-side versions across different runs, depending on the relative timing of the two inputs. Probe and build-side inputs can be configured with watermark alignment to keep the two inputs roughly aligned on event time, so that a probe-side row tends to be joined with build-side changes of a similar event time. This is a best-effort alignment and does not make the result deterministic. + +The build-side state grows with the number of distinct build-side keys, and during the load phase the buffered probe-side rows add to the state footprint until the operator transitions to the join phase. Use `state_ttl` to bound the build-side state for keys that are no longer updated or joined. You can reduce the amount of data that is buffered and processed during the load phase by configuring scan start offsets on the build and probe-side inputs, for example with a `scan.startup.*` [dynamic table option hint]({{< ref "docs/sql/reference/queries/hints" >}}#dynamic-table-options). + +**Batch mode** + +In batch mode, a `LATERAL SNAPSHOT` join is executed as a regular (`INNER` or `LEFT`) join between the probe side and the complete build side. Batch execution reads the entire build side before joining, so there is no load phase and no incremental state build-up. The streaming-specific arguments (`load_completed_condition`, `load_completed_time`, `load_completed_idle_timeout`, and `state_ttl`) are accepted but have no effect, and the build side does not need to declare a watermark. + +Because every probe-side row is joined against the final, complete build side, the batch result is **deterministic**. + Lookup Join -------------- diff --git a/docs/content/release-notes/flink-2.3.md b/docs/content/release-notes/flink-2.3.md index 8ddc99dbd9245f..06635d5f6f9ef3 100644 --- a/docs/content/release-notes/flink-2.3.md +++ b/docs/content/release-notes/flink-2.3.md @@ -90,17 +90,11 @@ CREATE FUNCTION my_func AS 'com.example.MyUdf' Flink 2.3 reworks how `SinkUpsertMaterializer` handles the case where a query's upsert key differs from the sink's primary key. Previously this required maintaining the full history of -records and could blow up state. Two changes address this: +records and could blow up state. An enhancement allows you to manage this situation: - A new `ON CONFLICT` clause with `DO NOTHING`, `DO ERROR` and `DO DEDUPLICATE` strategies makes the behavior on key conflict explicit. By default, planning now fails when the upsert and primary keys differ, requiring the user to choose a conflict strategy. -- Watermark-based record compaction is introduced to fix internal changelog disorder. The - trigger and frequency of compaction are controlled by: - - `table.exec.sink.upserts.compaction-mode` (default: `WATERMARK`) — `WATERMARK` or - `CHECKPOINT`. - - `table.exec.sink.upserts.compaction-interval` — optional fallback interval for emitting - watermarks when none arrive naturally. #### Process Table Function enhancements diff --git a/docs/layouts/shortcodes/generated/checkpointing_configuration.html b/docs/layouts/shortcodes/generated/checkpointing_configuration.html index 704ac492aa8a2a..2fcbc5fc1edb54 100644 --- a/docs/layouts/shortcodes/generated/checkpointing_configuration.html +++ b/docs/layouts/shortcodes/generated/checkpointing_configuration.html @@ -176,6 +176,12 @@ Boolean Forces unaligned checkpoints, particularly allowing them for iterative jobs. + +
execution.checkpointing.unaligned.interruptible-timers.emit-intermediate-watermarks
+ true + Boolean + When unaligned checkpoints with interruptible timers are enabled (see 'execution.checkpointing.unaligned.interruptible-timers.enabled') and firing the timers due for a watermark advance is interrupted before completing, an intermediate watermark reflecting the progress made so far is emitted downstream, at most as often as configured by 'pipeline.auto-watermark-interval'. This keeps downstream operators from stalling on watermark progress during a long-running catch-up. Set to false to disable intermediate watermark emission. +
execution.checkpointing.unaligned.interruptible-timers.enabled
false diff --git a/docs/layouts/shortcodes/generated/history_server_configuration.html b/docs/layouts/shortcodes/generated/history_server_configuration.html index 896c68135636e7..ef521d5f999b58 100644 --- a/docs/layouts/shortcodes/generated/history_server_configuration.html +++ b/docs/layouts/shortcodes/generated/history_server_configuration.html @@ -32,6 +32,12 @@ Duration Interval for refreshing the archived job directories. + +
historyserver.archive.load.mode
+ EAGER +

Enum

+ The mode that HistoryServer loads archives.

Possible values:
  • "EAGER"
  • "LAZY"
+
historyserver.archive.retained-applications
-1 @@ -56,6 +62,18 @@

Enum

The type of archive storage.

Possible values:
  • "FILE"
  • "ROCKSDB"
+ +
historyserver.lazy.fetch.executor.common.pool-size
+ 4 + Integer + The size of the common pool for archive fetching. + + +
historyserver.lazy.fetch.executor.individual.pool-size
+ 4 + Integer + The size of the individual pool for archive fetching. +
historyserver.log.jobmanager.url-pattern
(none) diff --git a/docs/setup_docs.sh b/docs/setup_docs.sh index 3abe2c3d3365cc..5d463ceab0bb50 100755 --- a/docs/setup_docs.sh +++ b/docs/setup_docs.sh @@ -55,7 +55,7 @@ if [ "$SKIP_INTEGRATE_CONNECTOR_DOCS" = false ]; then integrate_connector_docs aws v6.0 integrate_connector_docs cassandra v3.2 integrate_connector_docs pulsar v4.1 - integrate_connector_docs jdbc v4.0 + integrate_connector_docs jdbc v4.1.0 integrate_connector_docs rabbitmq v3.0 integrate_connector_docs gcp-pubsub v3.1 integrate_connector_docs mongodb v2.0 diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/writeable-serializer-2.3/serializer-snapshot b/flink-connectors/flink-hadoop-compatibility/src/test/resources/writeable-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..0af5d65ae90123 Binary files /dev/null and b/flink-connectors/flink-hadoop-compatibility/src/test/resources/writeable-serializer-2.3/serializer-snapshot differ diff --git a/flink-connectors/flink-hadoop-compatibility/src/test/resources/writeable-serializer-2.3/test-data b/flink-connectors/flink-hadoop-compatibility/src/test/resources/writeable-serializer-2.3/test-data new file mode 100644 index 00000000000000..f43afaefb35204 Binary files /dev/null and b/flink-connectors/flink-hadoop-compatibility/src/test/resources/writeable-serializer-2.3/test-data differ diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java index fb653120944598..75abd4178a0ca6 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CheckpointingOptions.java @@ -611,6 +611,31 @@ public class CheckpointingOptions { + " For this feature to be enabled, it must be also supported by the operator." + " Currently this is supported by all TableStreamOperators and CepOperator."); + /** + * Controls whether an intermediate watermark is emitted while a watermark advance is + * interrupted before completing, for unaligned checkpoints with interruptible timers enabled + * (see {@link #ENABLE_UNALIGNED_INTERRUPTIBLE_TIMERS}). Has no effect unless interruptible + * timers are enabled. The emission interval is governed by {@link + * PipelineOptions#AUTO_WATERMARK_INTERVAL}, the same as regular periodic watermark generation, + * since both have comparable performance implications. + */ + @Experimental + public static final ConfigOption + UNALIGNED_INTERRUPTIBLE_TIMERS_EMIT_INTERMEDIATE_WATERMARKS = + ConfigOptions.key( + "execution.checkpointing.unaligned.interruptible-timers.emit-intermediate-watermarks") + .booleanType() + .defaultValue(true) + .withDescription( + "When unaligned checkpoints with interruptible timers are enabled (see" + + " 'execution.checkpointing.unaligned.interruptible-timers.enabled') and" + + " firing the timers due for a watermark advance is interrupted before" + + " completing, an intermediate watermark reflecting the progress made so" + + " far is emitted downstream, at most as often as configured by" + + " 'pipeline.auto-watermark-interval'. This keeps downstream operators" + + " from stalling on watermark progress during a long-running catch-up." + + " Set to false to disable intermediate watermark emission."); + public static final ConfigOption ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH = ConfigOptions.key("execution.checkpointing.checkpoints-after-tasks-finish") .booleanType() diff --git a/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java index c90f92ccc3d647..05de6dee6ba37a 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/HistoryServerOptions.java @@ -257,6 +257,44 @@ public class HistoryServerOptions { .text("The type of archive storage.") .build()); + /** + * The mode that HistoryServer loads archives. + * + *
    + *
  • EAGER: Loads all archives by scheduled executor. + *
  • LAZY: Loads archives asynchronously only when requested. + *
+ */ + public static final ConfigOption + HISTORY_SERVER_ARCHIVE_LOAD_MODE = + key("historyserver.archive.load.mode") + .enumType(HistoryServerArchiveLoadMode.class) + .defaultValue(HistoryServerArchiveLoadMode.EAGER) + .withDescription( + Description.builder() + .text("The mode that HistoryServer loads archives.") + .build()); + + public static final ConfigOption HISTORY_SERVER_LAZY_FETCH_EXECUTOR_COMMON_POOL_SIZE = + key("historyserver.lazy.fetch.executor.common.pool-size") + .intType() + .defaultValue(4) + .withDescription( + Description.builder() + .text("The size of the common pool for archive fetching.") + .build()); + + public static final ConfigOption + HISTORY_SERVER_LAZY_FETCH_EXECUTOR_INDIVIDUAL_POOL_SIZE = + key("historyserver.lazy.fetch.executor.individual.pool-size") + .intType() + .defaultValue(4) + .withDescription( + Description.builder() + .text( + "The size of the individual pool for archive fetching.") + .build()); + /** The type of archive storage. */ public enum HistoryServerArchiveStorageType { /** Local file system. */ @@ -266,5 +304,18 @@ public enum HistoryServerArchiveStorageType { ROCKSDB } + /** The mode that HistoryServer loads archives. */ + public enum HistoryServerArchiveLoadMode { + + /** + * Eager mode (default). Archive files will be downloaded and persisted with the default + * retention behavior. + */ + EAGER, + + /** Lazy mode. Archive files will be downloaded and persisted if necessary. */ + LAZY + } + private HistoryServerOptions() {} } diff --git a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariant.java b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariant.java index ce5664d34e3f29..c5dd0bf3efe683 100644 --- a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariant.java +++ b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariant.java @@ -356,8 +356,16 @@ private static void toJsonImpl( sb.append(escapeJson(BinaryVariantUtil.getString(value, pos))); break; case DOUBLE: - sb.append(BinaryVariantUtil.getDouble(value, pos)); - break; + { + final double d = BinaryVariantUtil.getDouble(value, pos); + if (Double.isInfinite(d) || Double.isNaN(d)) { + throw new VariantTypeException( + String.format( + "Non-finite value %s cannot be serialized to JSON.", d)); + } + sb.append(d); + break; + } case DECIMAL: sb.append(BinaryVariantUtil.getDecimal(value, pos).toPlainString()); break; @@ -382,8 +390,17 @@ private static void toJsonImpl( .atZone(ZoneOffset.UTC))); break; case FLOAT: - sb.append(BinaryVariantUtil.getFloat(value, pos)); - break; + { + final float f = BinaryVariantUtil.getFloat(value, pos); + if (Float.isInfinite(f) || Float.isNaN(f)) { + throw new VariantTypeException( + String.format( + "Non-finite value %s cannot be serialized to JSON.", + (double) f)); + } + sb.append(f); + break; + } case BYTES: appendQuoted( sb, diff --git a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java index 928c267f3658c9..1674d166266f78 100644 --- a/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java +++ b/flink-core/src/main/java/org/apache/flink/types/variant/BinaryVariantInternalBuilder.java @@ -91,6 +91,7 @@ public class BinaryVariantInternalBuilder { new VariantTypeException("VARIANT_SIZE_LIMIT"); public static final VariantTypeException VARIANT_DUPLICATE_KEY_EXCEPTION = new VariantTypeException("VARIANT_DUPLICATE_KEY"); + private static final JsonFactory JSON_FACTORY = new JsonFactory(); public BinaryVariantInternalBuilder(boolean allowDuplicateKeys) { this.allowDuplicateKeys = allowDuplicateKeys; @@ -103,7 +104,7 @@ public BinaryVariantInternalBuilder(boolean allowDuplicateKeys) { */ public static BinaryVariant parseJson(String json, boolean allowDuplicateKeys) throws IOException { - try (JsonParser parser = new JsonFactory().createParser(json)) { + try (JsonParser parser = JSON_FACTORY.createParser(json)) { parser.nextToken(); return parseJson(parser, allowDuplicateKeys); } @@ -622,7 +623,17 @@ private int getIntegerSize(int value) { private void parseFloatingPoint(JsonParser parser) throws IOException { if (!tryParseDecimal(parser.getText())) { - appendDouble(parser.getDoubleValue()); + final double d = parser.getDoubleValue(); + // Jackson coerces out-of-range numbers like 1e400 to +/-Infinity. Reject them instead + // of storing a non-finite double that toJson() could not render as valid JSON. + if (Double.isInfinite(d) || Double.isNaN(d)) { + throw new JsonParseException( + parser, + String.format( + "Numeric value '%s' is out of the range of double precision and cannot be stored as a Variant.", + parser.getText())); + } + appendDouble(d); } } diff --git a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java index 924e538f40ad60..cec12149ceb5f8 100644 --- a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java +++ b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantInternalBuilderTest.java @@ -19,6 +19,8 @@ package org.apache.flink.types.variant; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.math.BigDecimal; @@ -120,6 +122,15 @@ void testParseJsonObject() throws IOException { assertThat(variant.getField("k2").getDecimal()).isEqualTo(BigDecimal.valueOf(1.5)); } + @ParameterizedTest + @ValueSource(strings = {"NaN", "Infinity", "-Infinity", "1e400", "-1e400"}) + void testParseJsonRejectsNonFiniteNumbers(final String nonFiniteNumber) { + // NaN and the infinities are not valid JSON; 1e400 is valid JSON but overflows the double + // range. Both must be rejected so PARSE_JSON errors and TRY_PARSE_JSON returns NULL. + assertThatThrownBy(() -> BinaryVariantInternalBuilder.parseJson(nonFiniteNumber, false)) + .isInstanceOf(IOException.class); + } + @Test void testAppendFloat() { BinaryVariantInternalBuilder builder = new BinaryVariantInternalBuilder(false); diff --git a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java index 83896ec53e1419..77235e968eebce 100644 --- a/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java +++ b/flink-core/src/test/java/org/apache/flink/types/variant/BinaryVariantTest.java @@ -20,6 +20,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.math.BigDecimal; import java.time.Instant; @@ -236,6 +238,22 @@ void testToJsonNested() { .isEqualTo("{" + "\"list\":[\"hello\",1]," + "\"object\":{\"ff\":10.0,\"ss\":1}}"); } + @ParameterizedTest + @ValueSource(doubles = {Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, Double.NaN}) + void testToJsonRejectsNonFiniteDouble(final double nonFinite) { + assertThatThrownBy(() -> builder.of(nonFinite).toJson()) + .isInstanceOf(VariantTypeException.class) + .hasMessageContaining("cannot be serialized to JSON"); + } + + @ParameterizedTest + @ValueSource(floats = {Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, Float.NaN}) + void testToJsonRejectsNonFiniteFloat(final float nonFinite) { + assertThatThrownBy(() -> builder.of(nonFinite).toJson()) + .isInstanceOf(VariantTypeException.class) + .hasMessageContaining("cannot be serialized to JSON"); + } + @Test void testVariantException() { assertThatThrownBy(() -> new BinaryVariant(new byte[0], new byte[0])) diff --git a/flink-core/src/test/resources/big-dec-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/big-dec-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..ccd31f3e8f9f31 Binary files /dev/null and b/flink-core/src/test/resources/big-dec-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/big-dec-serializer-2.3/test-data b/flink-core/src/test/resources/big-dec-serializer-2.3/test-data new file mode 100644 index 00000000000000..c59e4326824021 Binary files /dev/null and b/flink-core/src/test/resources/big-dec-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/big-int-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/big-int-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..b525669a164b76 Binary files /dev/null and b/flink-core/src/test/resources/big-int-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/big-int-serializer-2.3/test-data b/flink-core/src/test/resources/big-int-serializer-2.3/test-data new file mode 100644 index 00000000000000..3d0079e672c0a5 Binary files /dev/null and b/flink-core/src/test/resources/big-int-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/bitmap-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/bitmap-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..d41e8e63d781c1 Binary files /dev/null and b/flink-core/src/test/resources/bitmap-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/bitmap-serializer-2.3/test-data b/flink-core/src/test/resources/bitmap-serializer-2.3/test-data new file mode 100644 index 00000000000000..64cbeedb9ec83f Binary files /dev/null and b/flink-core/src/test/resources/bitmap-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/boolean-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/boolean-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..dfa6707ca06537 Binary files /dev/null and b/flink-core/src/test/resources/boolean-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/boolean-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/boolean-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..0e3eeb595e9d47 Binary files /dev/null and b/flink-core/src/test/resources/boolean-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/boolean-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/boolean-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..b1e867b8bc5cbf Binary files /dev/null and b/flink-core/src/test/resources/boolean-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/boolean-serializer-2.3/test-data b/flink-core/src/test/resources/boolean-serializer-2.3/test-data new file mode 100644 index 00000000000000..6b2aaa76407265 --- /dev/null +++ b/flink-core/src/test/resources/boolean-serializer-2.3/test-data @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/flink-core/src/test/resources/boolean-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/boolean-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..8511a731d7b424 Binary files /dev/null and b/flink-core/src/test/resources/boolean-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/boolean-value-serializer-2.3/test-data b/flink-core/src/test/resources/boolean-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..6b2aaa76407265 --- /dev/null +++ b/flink-core/src/test/resources/boolean-value-serializer-2.3/test-data @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/flink-core/src/test/resources/byte-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/byte-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..071fdfdfff4f5f Binary files /dev/null and b/flink-core/src/test/resources/byte-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/byte-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/byte-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..1ea699b1ed218a Binary files /dev/null and b/flink-core/src/test/resources/byte-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/byte-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/byte-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..372634d912a25e Binary files /dev/null and b/flink-core/src/test/resources/byte-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/byte-serializer-2.3/test-data b/flink-core/src/test/resources/byte-serializer-2.3/test-data new file mode 100644 index 00000000000000..f59ec20aabf584 --- /dev/null +++ b/flink-core/src/test/resources/byte-serializer-2.3/test-data @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/flink-core/src/test/resources/byte-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/byte-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..af6c1f0e67af9d Binary files /dev/null and b/flink-core/src/test/resources/byte-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/byte-value-serializer-2.3/test-data b/flink-core/src/test/resources/byte-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..f59ec20aabf584 --- /dev/null +++ b/flink-core/src/test/resources/byte-value-serializer-2.3/test-data @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/flink-core/src/test/resources/char-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/char-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..31a3a3452447a3 Binary files /dev/null and b/flink-core/src/test/resources/char-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/char-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/char-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..cc4337be169c1c Binary files /dev/null and b/flink-core/src/test/resources/char-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/char-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/char-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..5f6308de382dc9 Binary files /dev/null and b/flink-core/src/test/resources/char-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/char-serializer-2.3/test-data b/flink-core/src/test/resources/char-serializer-2.3/test-data new file mode 100644 index 00000000000000..f96c401f328b28 --- /dev/null +++ b/flink-core/src/test/resources/char-serializer-2.3/test-data @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/flink-core/src/test/resources/char-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/char-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..67c2a46ee1ec28 Binary files /dev/null and b/flink-core/src/test/resources/char-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/char-value-serializer-2.3/test-data b/flink-core/src/test/resources/char-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..d399cc0405eb30 Binary files /dev/null and b/flink-core/src/test/resources/char-value-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/copyable-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/copyable-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..f1b8eb6f37004b Binary files /dev/null and b/flink-core/src/test/resources/copyable-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/copyable-value-serializer-2.3/test-data b/flink-core/src/test/resources/copyable-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..9ff9b97f863be2 Binary files /dev/null and b/flink-core/src/test/resources/copyable-value-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/date-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/date-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..4794ebd6815d62 Binary files /dev/null and b/flink-core/src/test/resources/date-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/date-serializer-2.3/test-data b/flink-core/src/test/resources/date-serializer-2.3/test-data new file mode 100644 index 00000000000000..1faf9a01d95ca7 Binary files /dev/null and b/flink-core/src/test/resources/date-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/double-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/double-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..98ae2a649925f7 Binary files /dev/null and b/flink-core/src/test/resources/double-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/double-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/double-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..349e318c93c3d1 Binary files /dev/null and b/flink-core/src/test/resources/double-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/double-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/double-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..1383b642e931ba Binary files /dev/null and b/flink-core/src/test/resources/double-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/double-serializer-2.3/test-data b/flink-core/src/test/resources/double-serializer-2.3/test-data new file mode 100644 index 00000000000000..5cfd3714ef6431 --- /dev/null +++ b/flink-core/src/test/resources/double-serializer-2.3/test-data @@ -0,0 +1 @@ +@1 \ No newline at end of file diff --git a/flink-core/src/test/resources/double-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/double-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..604cef8589fd62 Binary files /dev/null and b/flink-core/src/test/resources/double-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/double-value-serializer-2.3/test-data b/flink-core/src/test/resources/double-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..5cfd3714ef6431 --- /dev/null +++ b/flink-core/src/test/resources/double-value-serializer-2.3/test-data @@ -0,0 +1 @@ +@1 \ No newline at end of file diff --git a/flink-core/src/test/resources/either-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/either-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..9357df72245167 Binary files /dev/null and b/flink-core/src/test/resources/either-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/either-serializer-2.3/test-data b/flink-core/src/test/resources/either-serializer-2.3/test-data new file mode 100644 index 00000000000000..87ec4466b7685d --- /dev/null +++ b/flink-core/src/test/resources/either-serializer-2.3/test-data @@ -0,0 +1 @@ + ApacheFlink \ No newline at end of file diff --git a/flink-core/src/test/resources/enum-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/enum-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..27cc1daa6463a7 Binary files /dev/null and b/flink-core/src/test/resources/enum-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/enum-serializer-2.3/test-data b/flink-core/src/test/resources/enum-serializer-2.3/test-data new file mode 100644 index 00000000000000..3b6fc84f4eecaa Binary files /dev/null and b/flink-core/src/test/resources/enum-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/enum-serializerreconfig-2.3/serializer-snapshot b/flink-core/src/test/resources/enum-serializerreconfig-2.3/serializer-snapshot new file mode 100644 index 00000000000000..a43be8dc0fff2a Binary files /dev/null and b/flink-core/src/test/resources/enum-serializerreconfig-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/enum-serializerreconfig-2.3/test-data b/flink-core/src/test/resources/enum-serializerreconfig-2.3/test-data new file mode 100644 index 00000000000000..ded2df2a51c333 Binary files /dev/null and b/flink-core/src/test/resources/enum-serializerreconfig-2.3/test-data differ diff --git a/flink-core/src/test/resources/float-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/float-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7342807802231c Binary files /dev/null and b/flink-core/src/test/resources/float-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/float-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/float-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..cb6654fd12700e Binary files /dev/null and b/flink-core/src/test/resources/float-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/float-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/float-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..62370bdddf999d Binary files /dev/null and b/flink-core/src/test/resources/float-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/float-serializer-2.3/test-data b/flink-core/src/test/resources/float-serializer-2.3/test-data new file mode 100644 index 00000000000000..f9c0e21ae86d67 --- /dev/null +++ b/flink-core/src/test/resources/float-serializer-2.3/test-data @@ -0,0 +1 @@ +By \ No newline at end of file diff --git a/flink-core/src/test/resources/float-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/float-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..6100e5746355ee Binary files /dev/null and b/flink-core/src/test/resources/float-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/float-value-serializer-2.3/test-data b/flink-core/src/test/resources/float-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..f9c0e21ae86d67 --- /dev/null +++ b/flink-core/src/test/resources/float-value-serializer-2.3/test-data @@ -0,0 +1 @@ +By \ No newline at end of file diff --git a/flink-core/src/test/resources/generic-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/generic-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..2bb4df4efcb20c Binary files /dev/null and b/flink-core/src/test/resources/generic-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/generic-array-serializer-2.3/test-data b/flink-core/src/test/resources/generic-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..6d06a98cad99c8 Binary files /dev/null and b/flink-core/src/test/resources/generic-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/int-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/int-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..2087b50341fa70 Binary files /dev/null and b/flink-core/src/test/resources/int-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/int-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/int-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..dd766c0e40557f Binary files /dev/null and b/flink-core/src/test/resources/int-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/int-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/int-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..597fc6b78debf3 Binary files /dev/null and b/flink-core/src/test/resources/int-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/int-serializer-2.3/test-data b/flink-core/src/test/resources/int-serializer-2.3/test-data new file mode 100644 index 00000000000000..e9271b31bdb53b Binary files /dev/null and b/flink-core/src/test/resources/int-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/int-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/int-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..f6ae0720db0bcb Binary files /dev/null and b/flink-core/src/test/resources/int-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/int-value-serializer-2.3/test-data b/flink-core/src/test/resources/int-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..e9271b31bdb53b Binary files /dev/null and b/flink-core/src/test/resources/int-value-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/kryo-custom-type-serializer-changed-registration-order-2.3/serializer-snapshot b/flink-core/src/test/resources/kryo-custom-type-serializer-changed-registration-order-2.3/serializer-snapshot new file mode 100644 index 00000000000000..07b541e4a23c27 Binary files /dev/null and b/flink-core/src/test/resources/kryo-custom-type-serializer-changed-registration-order-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/kryo-custom-type-serializer-changed-registration-order-2.3/test-data b/flink-core/src/test/resources/kryo-custom-type-serializer-changed-registration-order-2.3/test-data new file mode 100644 index 00000000000000..7d1fe65f710175 --- /dev/null +++ b/flink-core/src/test/resources/kryo-custom-type-serializer-changed-registration-order-2.3/test-data @@ -0,0 +1 @@ + Hass \ No newline at end of file diff --git a/flink-core/src/test/resources/kryo-type-serializer-changed-registration-order-2.3/serializer-snapshot b/flink-core/src/test/resources/kryo-type-serializer-changed-registration-order-2.3/serializer-snapshot new file mode 100644 index 00000000000000..ec2538c6dd06ed Binary files /dev/null and b/flink-core/src/test/resources/kryo-type-serializer-changed-registration-order-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/kryo-type-serializer-changed-registration-order-2.3/test-data b/flink-core/src/test/resources/kryo-type-serializer-changed-registration-order-2.3/test-data new file mode 100644 index 00000000000000..3637a3f5f87089 --- /dev/null +++ b/flink-core/src/test/resources/kryo-type-serializer-changed-registration-order-2.3/test-data @@ -0,0 +1 @@ + Hass \ No newline at end of file diff --git a/flink-core/src/test/resources/kryo-type-serializer-empty-config-2.3/serializer-snapshot b/flink-core/src/test/resources/kryo-type-serializer-empty-config-2.3/serializer-snapshot new file mode 100644 index 00000000000000..b7555735b81638 Binary files /dev/null and b/flink-core/src/test/resources/kryo-type-serializer-empty-config-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/kryo-type-serializer-empty-config-2.3/test-data b/flink-core/src/test/resources/kryo-type-serializer-empty-config-2.3/test-data new file mode 100644 index 00000000000000..3bb1715c8c15ea Binary files /dev/null and b/flink-core/src/test/resources/kryo-type-serializer-empty-config-2.3/test-data differ diff --git a/flink-core/src/test/resources/kryo-type-serializer-unrelated-config-after-restore-2.3/serializer-snapshot b/flink-core/src/test/resources/kryo-type-serializer-unrelated-config-after-restore-2.3/serializer-snapshot new file mode 100644 index 00000000000000..b7555735b81638 Binary files /dev/null and b/flink-core/src/test/resources/kryo-type-serializer-unrelated-config-after-restore-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/kryo-type-serializer-unrelated-config-after-restore-2.3/test-data b/flink-core/src/test/resources/kryo-type-serializer-unrelated-config-after-restore-2.3/test-data new file mode 100644 index 00000000000000..3bb1715c8c15ea Binary files /dev/null and b/flink-core/src/test/resources/kryo-type-serializer-unrelated-config-after-restore-2.3/test-data differ diff --git a/flink-core/src/test/resources/list-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/list-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..bbd86a683d884a Binary files /dev/null and b/flink-core/src/test/resources/list-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/list-serializer-2.3/test-data b/flink-core/src/test/resources/list-serializer-2.3/test-data new file mode 100644 index 00000000000000..ac798598cd7d02 Binary files /dev/null and b/flink-core/src/test/resources/list-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/long-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/long-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..6bf0336119e506 Binary files /dev/null and b/flink-core/src/test/resources/long-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/long-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/long-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..873fe645d6ffda Binary files /dev/null and b/flink-core/src/test/resources/long-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/long-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/long-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..4f51b34cfc2553 Binary files /dev/null and b/flink-core/src/test/resources/long-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/long-serializer-2.3/test-data b/flink-core/src/test/resources/long-serializer-2.3/test-data new file mode 100644 index 00000000000000..2ffef6c2652f25 Binary files /dev/null and b/flink-core/src/test/resources/long-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/long-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/long-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..b9e8f6938a1270 Binary files /dev/null and b/flink-core/src/test/resources/long-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/long-value-serializer-2.3/test-data b/flink-core/src/test/resources/long-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..2ffef6c2652f25 Binary files /dev/null and b/flink-core/src/test/resources/long-value-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/map-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/map-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..6d47ba46bc50c1 Binary files /dev/null and b/flink-core/src/test/resources/map-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/map-serializer-2.3/test-data b/flink-core/src/test/resources/map-serializer-2.3/test-data new file mode 100644 index 00000000000000..8999b1fbbac085 Binary files /dev/null and b/flink-core/src/test/resources/map-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/null-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/null-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..fcecf15adb98cd Binary files /dev/null and b/flink-core/src/test/resources/null-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/null-value-serializer-2.3/test-data b/flink-core/src/test/resources/null-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/flink-core/src/test/resources/nullable-not-padded-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/nullable-not-padded-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..71c489a4c5a369 Binary files /dev/null and b/flink-core/src/test/resources/nullable-not-padded-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/nullable-not-padded-serializer-2.3/test-data b/flink-core/src/test/resources/nullable-not-padded-serializer-2.3/test-data new file mode 100644 index 00000000000000..6b2aaa76407265 --- /dev/null +++ b/flink-core/src/test/resources/nullable-not-padded-serializer-2.3/test-data @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/flink-core/src/test/resources/nullable-padded-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/nullable-padded-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7831be8bcdbd98 Binary files /dev/null and b/flink-core/src/test/resources/nullable-padded-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/nullable-padded-serializer-2.3/test-data b/flink-core/src/test/resources/nullable-padded-serializer-2.3/test-data new file mode 100644 index 00000000000000..22fd28d4ec331a Binary files /dev/null and b/flink-core/src/test/resources/nullable-padded-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-identical-schema-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-identical-schema-2.3/serializer-snapshot new file mode 100644 index 00000000000000..5bded0878c8485 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-identical-schema-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-identical-schema-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-identical-schema-2.3/test-data new file mode 100644 index 00000000000000..670022368fc849 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-identical-schema-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-different-field-types-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-2.3/serializer-snapshot new file mode 100644 index 00000000000000..a1b44c0d196b94 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-different-field-types-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-2.3/test-data new file mode 100644 index 00000000000000..9cf45f4c08d73d Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-different-field-types-in-registered-subclass-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-in-registered-subclass-2.3/serializer-snapshot new file mode 100644 index 00000000000000..9f5609d12cb447 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-in-registered-subclass-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-different-field-types-in-registered-subclass-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-in-registered-subclass-2.3/test-data new file mode 100644 index 00000000000000..195b040a97afec Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-different-field-types-in-registered-subclass-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-different-subclass-registration-order-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-different-subclass-registration-order-2.3/serializer-snapshot new file mode 100644 index 00000000000000..e771338b74531f Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-different-subclass-registration-order-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-different-subclass-registration-order-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-different-subclass-registration-order-2.3/test-data new file mode 100644 index 00000000000000..d1569d857ae295 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-different-subclass-registration-order-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-missing-registered-subclass-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-missing-registered-subclass-2.3/serializer-snapshot new file mode 100644 index 00000000000000..e771338b74531f Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-missing-registered-subclass-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-missing-registered-subclass-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-missing-registered-subclass-2.3/test-data new file mode 100644 index 00000000000000..d1569d857ae295 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-missing-registered-subclass-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-modified-schema-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-2.3/serializer-snapshot new file mode 100644 index 00000000000000..63bf2dcbebe8f8 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-modified-schema-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-2.3/test-data new file mode 100644 index 00000000000000..0ce06771ec4194 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-modified-schema-in-registered-subclass-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-in-registered-subclass-2.3/serializer-snapshot new file mode 100644 index 00000000000000..5868d773eafb10 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-in-registered-subclass-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-modified-schema-in-registered-subclass-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-in-registered-subclass-2.3/test-data new file mode 100644 index 00000000000000..9d83d0b40feacb Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-modified-schema-in-registered-subclass-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-new-and-missing-registered-subclasses-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-new-and-missing-registered-subclasses-2.3/serializer-snapshot new file mode 100644 index 00000000000000..e771338b74531f Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-new-and-missing-registered-subclasses-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-new-and-missing-registered-subclasses-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-new-and-missing-registered-subclasses-2.3/test-data new file mode 100644 index 00000000000000..d1569d857ae295 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-new-and-missing-registered-subclasses-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-new-registered-subclass-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-new-registered-subclass-2.3/serializer-snapshot new file mode 100644 index 00000000000000..5bf16ffc5ca83a Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-new-registered-subclass-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-new-registered-subclass-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-new-registered-subclass-2.3/test-data new file mode 100644 index 00000000000000..10ca450e9c9de7 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-new-registered-subclass-2.3/test-data differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-non-registered-subclass-2.3/serializer-snapshot b/flink-core/src/test/resources/pojo-serializer-with-non-registered-subclass-2.3/serializer-snapshot new file mode 100644 index 00000000000000..5bf16ffc5ca83a Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-non-registered-subclass-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/pojo-serializer-with-non-registered-subclass-2.3/test-data b/flink-core/src/test/resources/pojo-serializer-with-non-registered-subclass-2.3/test-data new file mode 100644 index 00000000000000..10ca450e9c9de7 Binary files /dev/null and b/flink-core/src/test/resources/pojo-serializer-with-non-registered-subclass-2.3/test-data differ diff --git a/flink-core/src/test/resources/row-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/row-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..8590479338ad87 Binary files /dev/null and b/flink-core/src/test/resources/row-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/row-serializer-2.3/test-data b/flink-core/src/test/resources/row-serializer-2.3/test-data new file mode 100644 index 00000000000000..a734bd41431c92 Binary files /dev/null and b/flink-core/src/test/resources/row-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/set-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/set-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7506c7227f0cf4 Binary files /dev/null and b/flink-core/src/test/resources/set-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/set-serializer-2.3/test-data b/flink-core/src/test/resources/set-serializer-2.3/test-data new file mode 100644 index 00000000000000..c61ce3c9e82d19 Binary files /dev/null and b/flink-core/src/test/resources/set-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/short-primitive-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/short-primitive-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..5d2d0fc4028859 Binary files /dev/null and b/flink-core/src/test/resources/short-primitive-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/short-primitive-array-serializer-2.3/test-data b/flink-core/src/test/resources/short-primitive-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..cc4337be169c1c Binary files /dev/null and b/flink-core/src/test/resources/short-primitive-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/short-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/short-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..0960c9b6c66f35 Binary files /dev/null and b/flink-core/src/test/resources/short-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/short-serializer-2.3/test-data b/flink-core/src/test/resources/short-serializer-2.3/test-data new file mode 100644 index 00000000000000..aa9c34108ba5e9 Binary files /dev/null and b/flink-core/src/test/resources/short-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/short-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/short-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..c7808f5390fb7a Binary files /dev/null and b/flink-core/src/test/resources/short-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/short-value-serializer-2.3/test-data b/flink-core/src/test/resources/short-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..aa9c34108ba5e9 Binary files /dev/null and b/flink-core/src/test/resources/short-value-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/sql-date-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/sql-date-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..1042bba8cbe317 Binary files /dev/null and b/flink-core/src/test/resources/sql-date-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/sql-date-serializer-2.3/test-data b/flink-core/src/test/resources/sql-date-serializer-2.3/test-data new file mode 100644 index 00000000000000..1faf9a01d95ca7 Binary files /dev/null and b/flink-core/src/test/resources/sql-date-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/sql-time-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/sql-time-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..039309bb32c116 Binary files /dev/null and b/flink-core/src/test/resources/sql-time-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/sql-time-serializer-2.3/test-data b/flink-core/src/test/resources/sql-time-serializer-2.3/test-data new file mode 100644 index 00000000000000..1faf9a01d95ca7 Binary files /dev/null and b/flink-core/src/test/resources/sql-time-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/sql-timestamp-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/sql-timestamp-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..295fa6ea211fbc Binary files /dev/null and b/flink-core/src/test/resources/sql-timestamp-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/sql-timestamp-serializer-2.3/test-data b/flink-core/src/test/resources/sql-timestamp-serializer-2.3/test-data new file mode 100644 index 00000000000000..929596bbc66dac Binary files /dev/null and b/flink-core/src/test/resources/sql-timestamp-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/string-array-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/string-array-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..acc4e9db44b5b3 Binary files /dev/null and b/flink-core/src/test/resources/string-array-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/string-array-serializer-2.3/test-data b/flink-core/src/test/resources/string-array-serializer-2.3/test-data new file mode 100644 index 00000000000000..f1a5645d9ce2ae Binary files /dev/null and b/flink-core/src/test/resources/string-array-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/string-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/string-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..f796fddc59320c Binary files /dev/null and b/flink-core/src/test/resources/string-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/string-serializer-2.3/test-data b/flink-core/src/test/resources/string-serializer-2.3/test-data new file mode 100644 index 00000000000000..67ad74d387f530 --- /dev/null +++ b/flink-core/src/test/resources/string-serializer-2.3/test-data @@ -0,0 +1 @@ +%123456789012345678901234567890123456 \ No newline at end of file diff --git a/flink-core/src/test/resources/string-value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/string-value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..c7055ea38038f2 Binary files /dev/null and b/flink-core/src/test/resources/string-value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/string-value-serializer-2.3/test-data b/flink-core/src/test/resources/string-value-serializer-2.3/test-data new file mode 100644 index 00000000000000..9233b5ab0555a3 --- /dev/null +++ b/flink-core/src/test/resources/string-value-serializer-2.3/test-data @@ -0,0 +1 @@ +$123456789012345678901234567890123456 \ No newline at end of file diff --git a/flink-core/src/test/resources/tuple-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/tuple-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..ac31cfe15693a7 Binary files /dev/null and b/flink-core/src/test/resources/tuple-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/tuple-serializer-2.3/test-data b/flink-core/src/test/resources/tuple-serializer-2.3/test-data new file mode 100644 index 00000000000000..808ccbcfa07332 Binary files /dev/null and b/flink-core/src/test/resources/tuple-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/value-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/value-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..26ee2df37b2269 Binary files /dev/null and b/flink-core/src/test/resources/value-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/value-serializer-2.3/test-data b/flink-core/src/test/resources/value-serializer-2.3/test-data new file mode 100644 index 00000000000000..c1e49370d62e7e Binary files /dev/null and b/flink-core/src/test/resources/value-serializer-2.3/test-data differ diff --git a/flink-core/src/test/resources/variant-serializer-2.3/serializer-snapshot b/flink-core/src/test/resources/variant-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..cbefe6bbe9186a Binary files /dev/null and b/flink-core/src/test/resources/variant-serializer-2.3/serializer-snapshot differ diff --git a/flink-core/src/test/resources/variant-serializer-2.3/test-data b/flink-core/src/test/resources/variant-serializer-2.3/test-data new file mode 100644 index 00000000000000..114e6120265cdc Binary files /dev/null and b/flink-core/src/test/resources/variant-serializer-2.3/test-data differ diff --git a/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java b/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java index 95dae776d7d837..ed2d987be504a9 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java @@ -105,7 +105,8 @@ public class ConfigurationOptionLocator { "org.apache.flink.state.rocksdb.PredefinedOptions", "org.apache.flink.python.PythonConfig", "org.apache.flink.cep.configuration.SharedBufferCacheConfig", - "org.apache.flink.table.api.config.LookupJoinHintOptions")); + "org.apache.flink.table.api.config.LookupJoinHintOptions", + "org.apache.flink.table.api.config.EarlyFireJoinHintOptions")); private static final String DEFAULT_PATH_PREFIX = "src/main/java"; diff --git a/flink-formats/flink-avro/src/test/resources/generic-avro-serializer-2.3/serializer-snapshot b/flink-formats/flink-avro/src/test/resources/generic-avro-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7bd4d7210b8931 Binary files /dev/null and b/flink-formats/flink-avro/src/test/resources/generic-avro-serializer-2.3/serializer-snapshot differ diff --git a/flink-formats/flink-avro/src/test/resources/generic-avro-serializer-2.3/test-data b/flink-formats/flink-avro/src/test/resources/generic-avro-serializer-2.3/test-data new file mode 100644 index 00000000000000..5a7679a9ad2df0 Binary files /dev/null and b/flink-formats/flink-avro/src/test/resources/generic-avro-serializer-2.3/test-data differ diff --git a/flink-formats/flink-avro/src/test/resources/specific-avro-serializer-2.3/serializer-snapshot b/flink-formats/flink-avro/src/test/resources/specific-avro-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..d67dffc69ad012 Binary files /dev/null and b/flink-formats/flink-avro/src/test/resources/specific-avro-serializer-2.3/serializer-snapshot differ diff --git a/flink-formats/flink-avro/src/test/resources/specific-avro-serializer-2.3/test-data b/flink-formats/flink-avro/src/test/resources/specific-avro-serializer-2.3/test-data new file mode 100644 index 00000000000000..5a7679a9ad2df0 Binary files /dev/null and b/flink-formats/flink-avro/src/test/resources/specific-avro-serializer-2.3/test-data differ diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1784576564638-flink2.3-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1784576564638-flink2.3-snapshot new file mode 100644 index 00000000000000..c8206636edef30 Binary files /dev/null and b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1784576564638-flink2.3-snapshot differ diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink2.3-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink2.3-snapshot new file mode 100644 index 00000000000000..f0be13a09a621a Binary files /dev/null and b/flink-fs-tests/src/test/resources/reader-migration-test-flink2.3-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink2.3-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink2.3-snapshot new file mode 100644 index 00000000000000..ce6aebd132cde0 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-after-branching-flink2.3-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-conditions-flink2.3-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-conditions-flink2.3-snapshot new file mode 100644 index 00000000000000..935aca0744f5b9 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-conditions-flink2.3-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink2.3-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink2.3-snapshot new file mode 100644 index 00000000000000..31d8dbeeafb9a1 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-single-pattern-afterwards-flink2.3-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink2.3-snapshot b/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink2.3-snapshot new file mode 100644 index 00000000000000..b6d1ce511797cf Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/cep-migration-starting-new-pattern-flink2.3-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/dewey-number-serializer-2.3/serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/dewey-number-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..338c62886d758f Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/dewey-number-serializer-2.3/serializer-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/dewey-number-serializer-2.3/test-data b/flink-libraries/flink-cep/src/test/resources/dewey-number-serializer-2.3/test-data new file mode 100644 index 00000000000000..f8705181edc585 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/dewey-number-serializer-2.3/test-data differ diff --git a/flink-libraries/flink-cep/src/test/resources/event-id-serializer-2.3/serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/event-id-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..40dcf61713dbd4 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/event-id-serializer-2.3/serializer-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/event-id-serializer-2.3/test-data b/flink-libraries/flink-cep/src/test/resources/event-id-serializer-2.3/test-data new file mode 100644 index 00000000000000..146a1fc0a8be7d Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/event-id-serializer-2.3/test-data differ diff --git a/flink-libraries/flink-cep/src/test/resources/lockable-type-serializer-2.3/serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/lockable-type-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..1a984b61ebfc82 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/lockable-type-serializer-2.3/serializer-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/lockable-type-serializer-2.3/test-data b/flink-libraries/flink-cep/src/test/resources/lockable-type-serializer-2.3/test-data new file mode 100644 index 00000000000000..893957a6c1f98c Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/lockable-type-serializer-2.3/test-data differ diff --git a/flink-libraries/flink-cep/src/test/resources/nfa-state-serializer-2.3/serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/nfa-state-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7f01027f443214 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/nfa-state-serializer-2.3/serializer-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/nfa-state-serializer-2.3/test-data b/flink-libraries/flink-cep/src/test/resources/nfa-state-serializer-2.3/test-data new file mode 100644 index 00000000000000..1b1cb4d44c57c2 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/nfa-state-serializer-2.3/test-data differ diff --git a/flink-libraries/flink-cep/src/test/resources/node-id-serializer-2.3/serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/node-id-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..a63858d70052ed Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/node-id-serializer-2.3/serializer-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/node-id-serializer-2.3/test-data b/flink-libraries/flink-cep/src/test/resources/node-id-serializer-2.3/test-data new file mode 100644 index 00000000000000..0d333922c90bb2 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/node-id-serializer-2.3/test-data differ diff --git a/flink-libraries/flink-cep/src/test/resources/shared-buffer-edge-serializer-2.3/serializer-snapshot b/flink-libraries/flink-cep/src/test/resources/shared-buffer-edge-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..bd1cd93061613e Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/shared-buffer-edge-serializer-2.3/serializer-snapshot differ diff --git a/flink-libraries/flink-cep/src/test/resources/shared-buffer-edge-serializer-2.3/test-data b/flink-libraries/flink-cep/src/test/resources/shared-buffer-edge-serializer-2.3/test-data new file mode 100644 index 00000000000000..8746c6f117c272 Binary files /dev/null and b/flink-libraries/flink-cep/src/test/resources/shared-buffer-edge-serializer-2.3/test-data differ diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointDeepCopyTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointDeepCopyTest.java index 144aeb9d1c6dab..ab9119ad56a4a0 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointDeepCopyTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointDeepCopyTest.java @@ -32,15 +32,16 @@ import org.apache.flink.state.rocksdb.EmbeddedRocksDBStateBackend; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.test.util.AbstractTestBaseJUnit4; +import org.apache.flink.test.util.AbstractTestBase; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameter; +import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameters; import org.apache.flink.util.AbstractID; import org.apache.flink.util.Collector; import org.apache.commons.lang3.RandomStringUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; import java.io.File; import java.io.IOException; @@ -54,26 +55,20 @@ import java.util.stream.Stream; import static org.apache.flink.configuration.CheckpointingOptions.FS_SMALL_FILE_THRESHOLD; -import static org.hamcrest.Matchers.everyItem; -import static org.hamcrest.Matchers.isIn; -import static org.junit.Assert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; /** Test the savepoint deep copy. */ -@RunWith(value = Parameterized.class) -public class SavepointDeepCopyTest extends AbstractTestBaseJUnit4 { +@ExtendWith(ParameterizedTestExtension.class) +class SavepointDeepCopyTest extends AbstractTestBase { private static final MemorySize FILE_STATE_SIZE_THRESHOLD = new MemorySize(1); private static final String TEXT = "The quick brown fox jumps over the lazy dog"; private static final String RANDOM_VALUE = RandomStringUtils.randomAlphanumeric(120); - private final StateBackend backend; + @Parameter public StateBackend backend; - public SavepointDeepCopyTest(StateBackend backend) throws Exception { - this.backend = backend; - } - - @Parameterized.Parameters(name = "State Backend: {0}") + @Parameters(name = "State Backend: {0}") public static Collection data() { return Arrays.asList(new HashMapStateBackend(), new EmbeddedRocksDBStateBackend()); } @@ -132,8 +127,8 @@ public void readKey(String key, Context ctx, Collector> o * * @throws Exception throw exceptions when anything goes wrong */ - @Test - public void testSavepointDeepCopy() throws Exception { + @TestTemplate + void testSavepointDeepCopy() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); @@ -156,9 +151,9 @@ public void testSavepointDeepCopy() throws Exception { Set stateFiles1 = getFileNamesInDirectory(Paths.get(savepointPath1)); - Assert.assertTrue( - "Failed to bootstrap savepoint1 with additional state files", - stateFiles1.size() > 1); + assertThat(stateFiles1) + .as("Failed to bootstrap savepoint1 with additional state files") + .hasSizeGreaterThan(1); // create savepoint2 from savepoint1 created above File savepointUrl2 = createAndRegisterTempFile(new AbstractID().toHexString()); @@ -175,14 +170,13 @@ public void testSavepointDeepCopy() throws Exception { Set stateFiles2 = getFileNamesInDirectory(Paths.get(savepointPath1)); - Assert.assertTrue( - "Failed to create savepoint2 from savepoint1 with additional state files", - stateFiles2.size() > 1); + assertThat(stateFiles2) + .as("Failed to create savepoint2 from savepoint1 with additional state files") + .hasSizeGreaterThan(1); - assertThat( - "At least one state file in savepoint1 are not in savepoint2", - stateFiles1, - everyItem(isIn(stateFiles2))); + assertThat(stateFiles1) + .as("At least one state file in savepoint1 are not in savepoint2") + .isSubsetOf(stateFiles2); // Try to fromExistingSavepoint savepoint2 and read the state of "Operator1" (which has not // been @@ -197,10 +191,9 @@ public void testSavepointDeepCopy() throws Exception { .size(); long expectedKeyNum = Arrays.stream(TEXT.split(" ")).distinct().count(); - Assert.assertEquals( - "Unexpected number of keys in the state of Operator1", - expectedKeyNum, - actuallyKeyNum); + assertThat(actuallyKeyNum) + .as("Unexpected number of keys in the state of Operator1") + .isEqualTo(expectedKeyNum); } private static Set getFileNamesInDirectory(Path path) throws IOException { diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderITTestBase.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderITTestBase.java index 809dc201be0815..7e694a864356d4 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderITTestBase.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderITTestBase.java @@ -30,7 +30,7 @@ import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.core.execution.SavepointFormatType; import org.apache.flink.core.io.InputStatus; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -43,7 +43,8 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction; import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; -import org.apache.flink.test.util.AbstractTestBaseJUnit4; +import org.apache.flink.test.junit5.InjectClusterClient; +import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.test.util.source.AbstractTestSource; import org.apache.flink.test.util.source.SingleSplitEnumerator; import org.apache.flink.test.util.source.TestSourceReader; @@ -51,23 +52,24 @@ import org.apache.flink.util.AbstractID; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.flink.state.api.utils.SavepointTestBase.waitForAllRunningOrSomeTerminal; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; /** IT case for reading state. */ -public abstract class SavepointReaderITTestBase extends AbstractTestBaseJUnit4 { +abstract class SavepointReaderITTestBase extends AbstractTestBase { static final String UID = "stateful-operator"; static final String LIST_NAME = "list"; @@ -82,6 +84,13 @@ public abstract class SavepointReaderITTestBase extends AbstractTestBaseJUnit4 { private final MapStateDescriptor broadcast; + private RestClusterClient clusterClient; + + @BeforeEach + void setClusterClient(@InjectClusterClient RestClusterClient clusterClient) { + this.clusterClient = clusterClient; + } + SavepointReaderITTestBase( ListStateDescriptor list, ListStateDescriptor union, @@ -93,7 +102,7 @@ public abstract class SavepointReaderITTestBase extends AbstractTestBaseJUnit4 { } @Test - public void testOperatorStateInputFormat() throws Exception { + void testOperatorStateInputFormat() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); @@ -131,23 +140,19 @@ abstract DataStream> readBroadcastState(SavepointReader private void verifyListState(String path, StreamExecutionEnvironment env) throws Exception { SavepointReader savepoint = SavepointReader.read(env, path, new HashMapStateBackend()); List listResult = JobResultRetriever.collect(readListState(savepoint)); - listResult.sort(Comparator.naturalOrder()); - Assert.assertEquals( - "Unexpected elements read from list state", - SavepointSource.getElements(), - listResult); + assertThat(listResult) + .as("Unexpected elements read from list state") + .containsExactlyInAnyOrderElementsOf(SavepointSource.getElements()); } private void verifyUnionState(String path, StreamExecutionEnvironment env) throws Exception { SavepointReader savepoint = SavepointReader.read(env, path, new HashMapStateBackend()); List unionResult = JobResultRetriever.collect(readUnionState(savepoint)); - unionResult.sort(Comparator.naturalOrder()); - Assert.assertEquals( - "Unexpected elements read from union state", - SavepointSource.getElements(), - unionResult); + assertThat(unionResult) + .as("Unexpected elements read from union state") + .containsExactlyInAnyOrderElementsOf(SavepointSource.getElements()); } private void verifyBroadcastState(String path, StreamExecutionEnvironment env) @@ -156,36 +161,25 @@ private void verifyBroadcastState(String path, StreamExecutionEnvironment env) List> broadcastResult = JobResultRetriever.collect(readBroadcastState(savepoint)); - List broadcastStateKeys = - broadcastResult.stream() - .map(entry -> entry.f0) - .sorted(Comparator.naturalOrder()) - .collect(Collectors.toList()); - - List broadcastStateValues = - broadcastResult.stream() - .map(entry -> entry.f1) - .sorted(Comparator.naturalOrder()) + List expectedValues = + SavepointSource.getElements().stream() + .map(Object::toString) .collect(Collectors.toList()); - Assert.assertEquals( - "Unexpected element in broadcast state keys", - SavepointSource.getElements(), - broadcastStateKeys); + assertThat(broadcastResult) + .extracting(entry -> entry.f0) + .as("Unexpected element in broadcast state keys") + .containsExactlyInAnyOrderElementsOf(SavepointSource.getElements()); - Assert.assertEquals( - "Unexpected element in broadcast state values", - SavepointSource.getElements().stream() - .map(Object::toString) - .sorted() - .collect(Collectors.toList()), - broadcastStateValues); + assertThat(broadcastResult) + .extracting(entry -> entry.f1) + .as("Unexpected element in broadcast state values") + .containsExactlyInAnyOrderElementsOf(expectedValues); } private String takeSavepoint(JobGraph jobGraph) throws Exception { SavepointSource.initializeForTest(); - ClusterClient client = MINI_CLUSTER_RESOURCE.getClusterClient(); JobID jobId = jobGraph.getJobID(); Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5)); @@ -193,9 +187,9 @@ private String takeSavepoint(JobGraph jobGraph) throws Exception { String dirPath = getTempDirPath(new AbstractID().toHexString()); try { - JobID jobID = client.submitJob(jobGraph).get(); + JobID jobID = clusterClient.submitJob(jobGraph).get(); - waitForAllRunningOrSomeTerminal(jobID, MINI_CLUSTER_RESOURCE); + waitForAllRunningOrSomeTerminal(jobID, clusterClient); boolean finished = false; while (deadline.hasTimeLeft()) { if (SavepointSource.isFinished()) { @@ -212,14 +206,14 @@ private String takeSavepoint(JobGraph jobGraph) throws Exception { } if (!finished) { - Assert.fail("Failed to initialize state within deadline"); + fail("Failed to initialize state within deadline"); } CompletableFuture path = - client.triggerSavepoint(jobID, dirPath, SavepointFormatType.CANONICAL); + clusterClient.triggerSavepoint(jobID, dirPath, SavepointFormatType.CANONICAL); return path.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); } finally { - client.cancel(jobId).get(); + clusterClient.cancel(jobId).get(); } } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderKeyedStateITCase.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderKeyedStateITCase.java index 41dbc4e4b1148c..00746a9285a94f 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderKeyedStateITCase.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointReaderKeyedStateITCase.java @@ -38,8 +38,7 @@ import org.apache.flink.streaming.util.testing.CollectingSink; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.HashSet; @@ -52,8 +51,7 @@ import static org.assertj.core.api.Assertions.assertThat; /** IT case for reading state. */ -public abstract class SavepointReaderKeyedStateITCase - extends SavepointTestBase { +abstract class SavepointReaderKeyedStateITCase extends SavepointTestBase { private static final String uid = "stateful-operator"; private static ValueStateDescriptor valueState = @@ -67,7 +65,7 @@ public abstract class SavepointReaderKeyedStateITCase protected abstract Tuple2 getStateBackendTuple(); @Test - public void testUserKeyedStateReader() throws Exception { + void testUserKeyedStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -85,12 +83,13 @@ public void testUserKeyedStateReader() throws Exception { Set expected = new HashSet<>(elements); - Assert.assertEquals( - "Unexpected results from keyed state", expected, new HashSet<>(results)); + assertThat(new HashSet<>(results)) + .as("Unexpected results from keyed state") + .isEqualTo(expected); } @Test - public void testReadKeyedStateWithExactFilter() throws Exception { + void testReadKeyedStateWithExactFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -109,7 +108,7 @@ public void testReadKeyedStateWithExactFilter() throws Exception { } @Test - public void testReadKeyedStateWithMultiKeyExactFilter() throws Exception { + void testReadKeyedStateWithMultiKeyExactFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -128,7 +127,7 @@ public void testReadKeyedStateWithMultiKeyExactFilter() throws Exception { } @Test - public void testReadKeyedStateWithInclusiveRangeFilter() throws Exception { + void testReadKeyedStateWithInclusiveRangeFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -148,7 +147,7 @@ public void testReadKeyedStateWithInclusiveRangeFilter() throws Exception { } @Test - public void testReadKeyedStateWithInclusiveLowerExclusiveUpperRangeFilter() throws Exception { + void testReadKeyedStateWithInclusiveLowerExclusiveUpperRangeFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -168,7 +167,7 @@ public void testReadKeyedStateWithInclusiveLowerExclusiveUpperRangeFilter() thro } @Test - public void testReadKeyedStateWithExclusiveLowerInclusiveUpperRangeFilter() throws Exception { + void testReadKeyedStateWithExclusiveLowerInclusiveUpperRangeFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -188,7 +187,7 @@ public void testReadKeyedStateWithExclusiveLowerInclusiveUpperRangeFilter() thro } @Test - public void testReadKeyedStateWithExclusiveRangeFilter() throws Exception { + void testReadKeyedStateWithExclusiveRangeFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -208,7 +207,7 @@ public void testReadKeyedStateWithExclusiveRangeFilter() throws Exception { } @Test - public void testReadKeyedStateWithEmptyFilter() throws Exception { + void testReadKeyedStateWithEmptyFilter() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWindowReaderITCase.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWindowReaderITCase.java index 91834123ac6d40..3f535d820a90a6 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWindowReaderITCase.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWindowReaderITCase.java @@ -47,16 +47,15 @@ import org.apache.flink.streaming.runtime.operators.windowing.TimestampedValue; import org.apache.flink.util.Collector; -import org.hamcrest.Matchers; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.time.Duration; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; + /** IT Case for reading window operator state. */ -public abstract class SavepointWindowReaderITCase - extends SavepointTestBase { +abstract class SavepointWindowReaderITCase extends SavepointTestBase { private static final String uid = "stateful-operator"; private static final Integer[] numbers = {1, 2, 3}; @@ -64,7 +63,7 @@ public abstract class SavepointWindowReaderITCase protected abstract Tuple2 getStateBackendTuple(); @Test - public void testReduceWindowStateReader() throws Exception { + void testReduceWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -91,14 +90,13 @@ public void testReduceWindowStateReader() throws Exception { .window(TumblingEventTimeWindows.of(Duration.ofMillis(10))) .reduce(uid, new ReduceSum(), Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testReduceEvictorWindowStateReader() throws Exception { + void testReduceEvictorWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -127,14 +125,13 @@ public void testReduceEvictorWindowStateReader() throws Exception { .evictor() .reduce(uid, new ReduceSum(), Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testAggregateWindowStateReader() throws Exception { + void testAggregateWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -162,14 +159,13 @@ public void testAggregateWindowStateReader() throws Exception { .aggregate( uid, new AggregateSum(), Types.INT, Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testAggregateEvictorWindowStateReader() throws Exception { + void testAggregateEvictorWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -199,14 +195,13 @@ public void testAggregateEvictorWindowStateReader() throws Exception { .aggregate( uid, new AggregateSum(), Types.INT, Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testProcessWindowStateReader() throws Exception { + void testProcessWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -238,14 +233,13 @@ public void testProcessWindowStateReader() throws Exception { Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testProcessEvictorWindowStateReader() throws Exception { + void testProcessEvictorWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -279,14 +273,13 @@ public void testProcessEvictorWindowStateReader() throws Exception { Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testApplyWindowStateReader() throws Exception { + void testApplyWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -318,14 +311,13 @@ public void testApplyWindowStateReader() throws Exception { Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testApplyEvictorWindowStateReader() throws Exception { + void testApplyEvictorWindowStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -359,14 +351,13 @@ public void testApplyEvictorWindowStateReader() throws Exception { Types.INT, Types.INT)); - Assert.assertThat( - "Unexpected results from keyed state", - results, - Matchers.containsInAnyOrder(numbers)); + assertThat(results) + .as("Unexpected results from keyed state") + .containsExactlyInAnyOrder(numbers); } @Test - public void testWindowTriggerStateReader() throws Exception { + void testWindowTriggerStateReader() throws Exception { Tuple2 backendTuple = getStateBackendTuple(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(backendTuple.f0); @@ -397,8 +388,7 @@ public void testWindowTriggerStateReader() throws Exception { Types.INT, Types.LONG)); - Assert.assertThat( - "Unexpected results from trigger state", results, Matchers.contains(1L, 1L, 1L)); + assertThat(results).as("Unexpected results from trigger state").containsExactly(1L, 1L, 1L); } private static class NoOpProcessWindowFunction @@ -433,11 +423,10 @@ public void readWindow( Iterable elements, Collector out) throws Exception { - Assert.assertEquals("Unexpected window", new TimeWindow(0, 10), context.window()); - Assert.assertThat( - "Unexpected registered timers", - context.registeredEventTimeTimers(), - Matchers.contains(9L)); + assertThat(context.window()).as("Unexpected window").isEqualTo(new TimeWindow(0, 10)); + assertThat(context.registeredEventTimeTimers()) + .as("Unexpected registered timers") + .containsExactly(9L); out.collect(elements.iterator().next()); } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterITCase.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterITCase.java index d7a5586b6024f8..6d9a9052ac2bec 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterITCase.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterITCase.java @@ -47,13 +47,12 @@ import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction; import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; import org.apache.flink.streaming.api.graph.StreamGraph; -import org.apache.flink.test.util.AbstractTestBaseJUnit4; +import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.util.AbstractID; import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; @@ -64,9 +63,10 @@ import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; /** IT test for writing savepoints. */ -public class SavepointWriterITCase extends AbstractTestBaseJUnit4 { +class SavepointWriterITCase extends AbstractTestBase { private static final long CHECKPOINT_ID = 42; @@ -86,25 +86,25 @@ public class SavepointWriterITCase extends AbstractTestBaseJUnit4 { Arrays.asList(new CurrencyRate("USD", 1.0), new CurrencyRate("EUR", 1.3)); @Test - public void testDefaultStateBackend() throws Exception { + void testDefaultStateBackend() throws Exception { testStateBootstrapAndModification(new Configuration(), null); } @Test - public void testHashMapStateBackend() throws Exception { + void testHashMapStateBackend() throws Exception { testStateBootstrapAndModification( new Configuration().set(StateBackendOptions.STATE_BACKEND, "hashmap"), new HashMapStateBackend()); } @Test - public void testEmbeddedRocksDBStateBackend() throws Exception { + void testEmbeddedRocksDBStateBackend() throws Exception { testStateBootstrapAndModification( new Configuration().set(StateBackendOptions.STATE_BACKEND, "rocksdb"), new EmbeddedRocksDBStateBackend()); } - public void testStateBootstrapAndModification(Configuration config, StateBackend backend) + void testStateBootstrapAndModification(Configuration config, StateBackend backend) throws Exception { final String savepointPath = getTempDirPath(new AbstractID().toHexString()); @@ -387,14 +387,11 @@ public void initializeState(FunctionInitializationContext context) throws Except expected.add(3); for (Integer number : state.get()) { - Assert.assertTrue("Duplicate state", expected.contains(number)); + assertThat(expected).as("Duplicate state").contains(number); expected.remove(number); } - Assert.assertTrue( - "Failed to bootstrap all state elements: " - + Arrays.toString(expected.toArray()), - expected.isEmpty()); + assertThat(expected).as("Failed to bootstrap all state elements").isEmpty(); } } @@ -421,11 +418,9 @@ public static class CurrencyValidationFunction @Override public void processElement(CurrencyRate value, ReadOnlyContext ctx, Collector out) throws Exception { - Assert.assertEquals( - "Incorrect currency rate", - value.rate, - ctx.getBroadcastState(descriptor).get(value.currency), - 0.0001); + assertThat(ctx.getBroadcastState(descriptor).get(value.currency)) + .as("Incorrect currency rate") + .isCloseTo(value.rate, within(0.0001)); } @Override diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterWindowITCase.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterWindowITCase.java index 514f9ac7d3e3d6..8fb998680747bf 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterWindowITCase.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/SavepointWriterWindowITCase.java @@ -45,14 +45,16 @@ import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.evictors.CountEvictor; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; -import org.apache.flink.test.util.AbstractTestBaseJUnit4; +import org.apache.flink.test.util.AbstractTestBase; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameter; +import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameters; import org.apache.flink.util.AbstractID; import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.Collector; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; import java.time.Duration; import java.util.ArrayList; @@ -65,8 +67,8 @@ /** IT Test for writing savepoints to the {@code WindowOperator}. */ @SuppressWarnings("unchecked") -@RunWith(Parameterized.class) -public class SavepointWriterWindowITCase extends AbstractTestBaseJUnit4 { +@ExtendWith(ParameterizedTestExtension.class) +class SavepointWriterWindowITCase extends AbstractTestBase { private static final String UID = "uid"; @@ -113,7 +115,7 @@ public class SavepointWriterWindowITCase extends AbstractTestBaseJUnit4 { new EmbeddedRocksDBStateBackend(), new Configuration().set(StateBackendOptions.STATE_BACKEND, "rocksdb"))); - @Parameterized.Parameters(name = "{0}") + @Parameters(name = "{0}") public static Collection data() { List parameterList = new ArrayList<>(); for (Tuple3 stateBackend : STATE_BACKENDS) { @@ -133,29 +135,24 @@ public static Collection data() { return parameterList; } - private final WindowBootstrap windowBootstrap; + @SuppressWarnings("unused") + @Parameter + public String ignore; - private final WindowStream windowStream; + @Parameter(1) + public WindowBootstrap windowBootstrap; - private final StateBackend stateBackend; + @Parameter(2) + public WindowStream windowStream; - private final Configuration configuration; + @Parameter(3) + public StateBackend stateBackend; - @SuppressWarnings("unused") - public SavepointWriterWindowITCase( - String ignore, - WindowBootstrap windowBootstrap, - WindowStream windowStream, - StateBackend stateBackend, - Configuration configuration) { - this.windowBootstrap = windowBootstrap; - this.windowStream = windowStream; - this.stateBackend = stateBackend; - this.configuration = configuration; - } + @Parameter(4) + public Configuration configuration; - @Test - public void testTumbleWindow() throws Exception { + @TestTemplate + void testTumbleWindow() throws Exception { final String savepointPath = getTempDirPath(new AbstractID().toHexString()); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration); @@ -198,8 +195,8 @@ public void testTumbleWindow() throws Exception { .containsAll(STANDARD_MATCHER); } - @Test - public void testTumbleWindowWithEvictor() throws Exception { + @TestTemplate + void testTumbleWindowWithEvictor() throws Exception { final String savepointPath = getTempDirPath(new AbstractID().toHexString()); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration); @@ -244,8 +241,8 @@ public void testTumbleWindowWithEvictor() throws Exception { .containsAll(EVICTOR_MATCHER); } - @Test - public void testSlideWindow() throws Exception { + @TestTemplate + void testSlideWindow() throws Exception { final String savepointPath = getTempDirPath(new AbstractID().toHexString()); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration); @@ -291,8 +288,8 @@ public void testSlideWindow() throws Exception { .containsAll(STANDARD_MATCHER); } - @Test - public void testSlideWindowWithEvictor() throws Exception { + @TestTemplate + void testSlideWindowWithEvictor() throws Exception { final String savepointPath = getTempDirPath(new AbstractID().toHexString()); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration); diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/StateBootstrapTransformationTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/StateBootstrapTransformationTest.java index 5928c809c6dbf2..3935a493b85ca0 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/StateBootstrapTransformationTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/StateBootstrapTransformationTest.java @@ -31,16 +31,17 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.graph.StreamConfig; -import org.apache.flink.test.util.AbstractTestBaseJUnit4; +import org.apache.flink.test.util.AbstractTestBase; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; /** Tests for bootstrap transformations. */ -public class StateBootstrapTransformationTest extends AbstractTestBaseJUnit4 { +class StateBootstrapTransformationTest extends AbstractTestBase { @Test - public void testBroadcastStateTransformationParallelism() { + void testBroadcastStateTransformationParallelism() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(10); @@ -58,14 +59,13 @@ public void testBroadcastStateTransformationParallelism() { new Path(), maxParallelism); - Assert.assertEquals( - "Broadcast transformations should always be run at parallelism 1", - 1, - result.getParallelism()); + assertThat(result.getParallelism()) + .as("Broadcast transformations should always be run at parallelism 1") + .isOne(); } @Test - public void testDefaultParallelismRespectedWhenLessThanMaxParallelism() { + void testDefaultParallelismRespectedWhenLessThanMaxParallelism() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); @@ -83,14 +83,14 @@ public void testDefaultParallelismRespectedWhenLessThanMaxParallelism() { new Path(), maxParallelism); - Assert.assertEquals( - "The parallelism of a data set should not change when less than the max parallelism of the savepoint", - env.getParallelism(), - result.getParallelism()); + assertThat(result.getParallelism()) + .as( + "The parallelism of a data set should not change when less than the max parallelism of the savepoint") + .isEqualTo(env.getParallelism()); } @Test - public void testMaxParallelismRespected() { + void testMaxParallelismRespected() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(10); @@ -108,14 +108,14 @@ public void testMaxParallelismRespected() { new Path(), maxParallelism); - Assert.assertEquals( - "The parallelism of a data set should be constrained my the savepoint max parallelism", - 4, - result.getParallelism()); + assertThat(result.getParallelism()) + .as( + "The parallelism of a data set should be constrained my the savepoint max parallelism") + .isEqualTo(4); } @Test - public void testOperatorSpecificMaxParallelismRespected() { + void testOperatorSpecificMaxParallelismRespected() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); @@ -134,14 +134,14 @@ public void testOperatorSpecificMaxParallelismRespected() { new Path(), maxParallelism); - Assert.assertEquals( - "The parallelism of a data set should be constrained my the savepoint max parallelism", - 1, - result.getParallelism()); + assertThat(result.getParallelism()) + .as( + "The parallelism of a data set should be constrained my the savepoint max parallelism") + .isOne(); } @Test - public void testStreamConfig() { + void testStreamConfig() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream input = env.fromData(""); @@ -159,10 +159,9 @@ public void testStreamConfig() { KeySelector selector = config.getStatePartitioner(0, Thread.currentThread().getContextClassLoader()); - Assert.assertEquals( - "Incorrect key selector forwarded to stream operator", - CustomKeySelector.class, - selector.getClass()); + assertThat(selector.getClass()) + .as("Incorrect key selector forwarded to stream operator") + .isEqualTo(CustomKeySelector.class); } private static class CustomKeySelector implements KeySelector { diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BroadcastStateInputFormatTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BroadcastStateInputFormatTest.java index 518fca97b2ba52..0d1b2e65dbec3e 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BroadcastStateInputFormatTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BroadcastStateInputFormatTest.java @@ -34,20 +34,21 @@ import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.assertj.core.api.Assertions.assertThat; + /** Test for operator broadcast state input format. */ -public class BroadcastStateInputFormatTest { +class BroadcastStateInputFormatTest { private static MapStateDescriptor descriptor = new MapStateDescriptor<>("state", Types.INT, Types.INT); @Test - public void testReadBroadcastState() throws Exception { + void testReadBroadcastState() throws Exception { try (TwoInputStreamOperatorTestHarness testHarness = getTestHarness()) { testHarness.open(); @@ -83,8 +84,9 @@ public void testReadBroadcastState() throws Exception { expected.put(2, 2); expected.put(3, 3); - Assert.assertEquals( - "Failed to read correct list state from state backend", expected, results); + assertThat(results) + .as("Failed to read correct list state from state backend") + .isEqualTo(expected); } } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BufferingCollectorTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BufferingCollectorTest.java index 933f4caed536b2..9ddb838dae5e07 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BufferingCollectorTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/BufferingCollectorTest.java @@ -18,27 +18,27 @@ package org.apache.flink.state.api.input; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; /** Test of the buffering collector. */ -public class BufferingCollectorTest { +class BufferingCollectorTest { @Test - public void testNestRemovesElement() { + void testNestRemovesElement() { BufferingCollector collector = new BufferingCollector<>(); collector.collect(1); - Assert.assertTrue("Failed to add element to collector", collector.hasNext()); - Assert.assertEquals( - "Incorrect element removed from collector", Integer.valueOf(1), collector.next()); - Assert.assertFalse("Failed to drop element from collector", collector.hasNext()); + assertThat(collector).as("Failed to add element to collector").hasNext(); + assertThat(collector.next()).as("Incorrect element removed from collector").isOne(); + assertThat(collector).as("Failed to drop element from collector").isExhausted(); } @Test - public void testEmptyCollectorReturnsNull() { + void testEmptyCollectorReturnsNull() { BufferingCollector collector = new BufferingCollector<>(); - Assert.assertNull("Empty collector did not return null", collector.next()); + assertThat(collector.next()).isNull(); } } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java index 292eb61d2c44af..bde20fd0f579cc 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/KeyedStateInputFormatTest.java @@ -47,18 +47,14 @@ import org.apache.flink.streaming.util.asyncprocessing.AsyncKeyedOneInputStreamOperatorTestHarness; import org.apache.flink.util.Collector; -import org.junit.Assert; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Set; @@ -67,7 +63,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for keyed state input format. */ -@RunWith(Parameterized.class) class KeyedStateInputFormatTest { private static ValueStateDescriptor stateDescriptor = new ValueStateDescriptor<>("state", Types.INT); @@ -90,8 +85,7 @@ void testCreatePartitionedInputSplits(boolean asyncState) throws Exception { new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT), new ExecutionConfig()); KeyGroupRangeInputSplit[] splits = format.createInputSplits(4); - Assert.assertEquals( - "Failed to properly partition operator state into input splits", 4, splits.length); + assertThat(splits).hasSize(4); } @ParameterizedTest(name = "Enable async state = {0}") @@ -112,10 +106,9 @@ void testMaxParallelismRespected(boolean asyncState) throws Exception { new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT), new ExecutionConfig()); KeyGroupRangeInputSplit[] splits = format.createInputSplits(129); - Assert.assertEquals( - "Failed to properly partition operator state into input splits", - 128, - splits.length); + assertThat(splits) + .as("Failed to properly partition operator state into input splits") + .hasSize(128); } @ParameterizedTest(name = "Enable async state = {0}") @@ -220,7 +213,7 @@ void testReadState(boolean asyncState) throws Exception { List data = readInputSplit(split, userFunction); - Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 2, 3), data); + assertThat(data).as("Incorrect data read from input split").containsExactly(1, 2, 3); } @ParameterizedTest(name = "Enable async state = {0}") @@ -246,8 +239,9 @@ void testReadMultipleOutputPerKey(boolean asyncState) throws Exception { List data = readInputSplit(split, userFunction); - Assert.assertEquals( - "Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data); + assertThat(data) + .as("Incorrect data read from input split") + .containsExactly(1, 1, 2, 2, 3, 3); } @ParameterizedTest(name = "Enable async state = {0}") @@ -298,8 +292,9 @@ void testReadTime() throws Exception { List data = readInputSplit(split, userFunction); - Assert.assertEquals( - "Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data); + assertThat(data) + .as("Incorrect data read from input split") + .containsExactly(1, 1, 2, 2, 3, 3); } @Nonnull @@ -471,18 +466,16 @@ public void readKey( Integer key, KeyedStateReaderFunction.Context ctx, Collector out) throws Exception { Set eventTimers = ctx.registeredEventTimeTimers(); - Assert.assertEquals( - "Each key should have exactly one event timer for key " + key, - 1, - eventTimers.size()); + assertThat(eventTimers) + .as("Each key should have exactly one event timer for key %s", key) + .hasSize(1); out.collect(eventTimers.iterator().next().intValue()); Set procTimers = ctx.registeredProcessingTimeTimers(); - Assert.assertEquals( - "Each key should have exactly one processing timer for key " + key, - 1, - procTimers.size()); + assertThat(procTimers) + .as("Each key should have exactly one processing timer for key %s", key) + .hasSize(1); out.collect(procTimers.iterator().next().intValue()); } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/ListStateInputFormatTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/ListStateInputFormatTest.java index ff9252ad5944d4..53e1ef939e3485 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/ListStateInputFormatTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/ListStateInputFormatTest.java @@ -36,21 +36,20 @@ import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; + /** Test for operator list state input format. */ -public class ListStateInputFormatTest { +class ListStateInputFormatTest { private static ListStateDescriptor descriptor = new ListStateDescriptor<>("state", Types.INT); @Test - public void testReadListOperatorState() throws Exception { + void testReadListOperatorState() throws Exception { try (OneInputStreamOperatorTestHarness testHarness = getTestHarness()) { testHarness.open(); @@ -84,12 +83,9 @@ public void testReadListOperatorState() throws Exception { results.add(format.nextRecord(0)); } - results.sort(Comparator.naturalOrder()); - - Assert.assertEquals( - "Failed to read correct list state from state backend", - Arrays.asList(1, 2, 3), - results); + assertThat(results) + .as("Failed to read correct list state from state backend") + .containsExactlyInAnyOrder(1, 2, 3); } } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/MultiStateKeyIteratorTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/MultiStateKeyIteratorTest.java index 260df82dfa4fc4..c6f011721d2386 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/MultiStateKeyIteratorTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/MultiStateKeyIteratorTest.java @@ -57,13 +57,11 @@ import org.apache.flink.runtime.state.ttl.mock.MockRestoreOperation; import org.apache.flink.runtime.state.ttl.mock.MockStateBackend; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import javax.annotation.Nonnull; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -73,8 +71,10 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; + /** Test for the multi-state key iterator. */ -public class MultiStateKeyIteratorTest { +class MultiStateKeyIteratorTest { private static final List> descriptors; static { @@ -161,7 +161,7 @@ private static void clearKey( } @Test - public void testIteratorPullsKeyFromAllDescriptors() throws Exception { + void testIteratorPullsKeyFromAllDescriptors() throws Exception { AbstractKeyedStateBackend keyedStateBackend = createKeyedStateBackend(); setKey(keyedStateBackend, descriptors.get(0), 1); @@ -176,12 +176,11 @@ public void testIteratorPullsKeyFromAllDescriptors() throws Exception { keys.add(iterator.next()); } - Assert.assertEquals("Unexpected number of keys", 2, keys.size()); - Assert.assertEquals("Unexpected keys found", Arrays.asList(1, 2), keys); + assertThat(keys).containsExactly(1, 2); } @Test - public void testIteratorSkipsEmptyDescriptors() throws Exception { + void testIteratorSkipsEmptyDescriptors() throws Exception { AbstractKeyedStateBackend keyedStateBackend = createKeyedStateBackend(); List> threeDescriptors = new ArrayList<>(3); @@ -206,13 +205,12 @@ public void testIteratorSkipsEmptyDescriptors() throws Exception { keys.add(iterator.next()); } - Assert.assertEquals("Unexpected number of keys", 2, keys.size()); - Assert.assertEquals("Unexpected keys found", Arrays.asList(1, 2), keys); + assertThat(keys).containsExactly(1, 2); } /** Test for lazy enumeration of inner iterators. */ @Test - public void testIteratorPullsSingleKeyFromAllDescriptors() throws AssertionError { + void testIteratorPullsSingleKeyFromAllDescriptors() throws AssertionError { CountingKeysKeyedStateBackend keyedStateBackend = createCountingKeysKeyedStateBackend(100_000_000); MultiStateKeyIterator testedIterator = @@ -220,10 +218,9 @@ public void testIteratorPullsSingleKeyFromAllDescriptors() throws AssertionError testedIterator.hasNext(); - Assert.assertEquals( - "Unexpected number of keys enumerated", - 1, - keyedStateBackend.numberOfKeysEnumerated); + assertThat(keyedStateBackend.numberOfKeysEnumerated) + .as("Unexpected number of keys enumerated") + .isOne(); } /** diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/StreamOperatorContextBuilderTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/StreamOperatorContextBuilderTest.java index 91fd3781797744..3de0ffec0b5fca 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/StreamOperatorContextBuilderTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/StreamOperatorContextBuilderTest.java @@ -28,18 +28,20 @@ import org.apache.flink.state.api.utils.CustomStateBackendFactory; import org.apache.flink.streaming.util.MockStreamingRuntimeContext; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + /** Tests for the stream operator context builder. */ -public class StreamOperatorContextBuilderTest { +class StreamOperatorContextBuilderTest { private static final Logger LOG = LoggerFactory.getLogger(StreamOperatorContextBuilderTest.class); - @Test(expected = CustomStateBackendFactory.ExpectedException.class) - public void testStateBackendLoading() throws Exception { + @Test + void testStateBackendLoading() throws Exception { Configuration configuration = new Configuration(); configuration.set( StateBackendOptions.STATE_BACKEND, @@ -67,6 +69,7 @@ public int getSplitNumber() { null, context.getExecutionConfig()); - builder.build(LOG); + assertThatThrownBy(() -> builder.build(LOG)) + .isInstanceOf(CustomStateBackendFactory.ExpectedException.class); } } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/UnionStateInputFormatTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/UnionStateInputFormatTest.java index 248a0ea5a6eeb6..e2f67782ad37cd 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/UnionStateInputFormatTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/UnionStateInputFormatTest.java @@ -37,21 +37,20 @@ import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; + /** Test for operator union state input format. */ -public class UnionStateInputFormatTest { +class UnionStateInputFormatTest { private static ListStateDescriptor descriptor = new ListStateDescriptor<>("state", Types.INT); @Test - public void testReadUnionOperatorState() throws Exception { + void testReadUnionOperatorState() throws Exception { try (OneInputStreamOperatorTestHarness testHarness = getTestHarness()) { testHarness.open(); @@ -80,12 +79,9 @@ public void testReadUnionOperatorState() throws Exception { results.add(format.nextRecord(0)); } - results.sort(Comparator.naturalOrder()); - - Assert.assertEquals( - "Failed to read correct list state from state backend", - Arrays.asList(1, 2, 3), - results); + assertThat(results) + .as("Failed to read correct list state from state backend") + .containsExactlyInAnyOrder(1, 2, 3); } } diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/WindowReaderTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/WindowReaderTest.java index 96313f8c40d67f..1ee290943146b9 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/WindowReaderTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/input/WindowReaderTest.java @@ -55,31 +55,30 @@ import org.apache.flink.streaming.util.MockStreamingRuntimeContext; import org.apache.flink.util.Collector; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import javax.annotation.Nonnull; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.function.Function; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.mockito.Mockito.mock; /** Tests reading window state. */ @SuppressWarnings("unchecked") -public class WindowReaderTest { +class WindowReaderTest { private static final int MAX_PARALLELISM = 128; private static final String UID = "uid"; @Test - public void testReducingWindow() throws Exception { + void testReducingWindow() throws Exception { WindowOperator operator = getWindowOperator( stream -> @@ -102,11 +101,11 @@ public void testReducingWindow() throws Exception { new ExecutionConfig()); List list = readState(format); - Assert.assertEquals(Arrays.asList(1, 1), list); + assertThat(list).containsExactly(1, 1); } @Test - public void testSessionWindow() throws Exception { + void testSessionWindow() throws Exception { WindowOperator operator = getWindowOperator( stream -> @@ -129,11 +128,11 @@ public void testSessionWindow() throws Exception { new ExecutionConfig()); List list = readState(format); - Assert.assertEquals(Collections.singletonList(2), list); + assertThat(list).containsExactly(2); } @Test - public void testAggregateWindow() throws Exception { + void testAggregateWindow() throws Exception { WindowOperator operator = getWindowOperator( stream -> @@ -156,11 +155,11 @@ public void testAggregateWindow() throws Exception { new ExecutionConfig()); List list = readState(format); - Assert.assertEquals(Arrays.asList(1, 1), list); + assertThat(list).containsExactly(1, 1); } @Test - public void testProcessReader() throws Exception { + void testProcessReader() throws Exception { WindowOperator operator = getWindowOperator( stream -> @@ -182,11 +181,11 @@ public void testProcessReader() throws Exception { new ExecutionConfig()); List list = readState(format); - Assert.assertEquals(Arrays.asList(1, 1), list); + assertThat(list).containsExactly(1, 1); } @Test - public void testPerPaneAndPerKeyState() throws Exception { + void testPerPaneAndPerKeyState() throws Exception { WindowOperator operator = getWindowOperator( stream -> @@ -209,7 +208,7 @@ public void testPerPaneAndPerKeyState() throws Exception { new ExecutionConfig()); List> list = readState(format); - Assert.assertEquals(Arrays.asList(Tuple2.of(2, 1), Tuple2.of(2, 1)), list); + assertThat(list).containsExactly(Tuple2.of(2, 1), Tuple2.of(2, 1)); } private static WindowOperator getWindowOperator( @@ -257,14 +256,14 @@ private static OperatorState getOperatorState( DataStream dataStream) { Transformation transformation = dataStream.getTransformation(); if (!(transformation instanceof OneInputTransformation)) { - Assert.fail("This test only supports window operators"); + fail("This test only supports window operators"); } OneInputTransformation oneInput = (OneInputTransformation) transformation; StreamOperator operator = oneInput.getOperator(); if (!(operator instanceof WindowOperator)) { - Assert.fail("This test only supports window operators"); + fail("This test only supports window operators"); } return (WindowOperator) operator; diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java index b5b034a0762578..242381c44ed644 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java @@ -36,16 +36,16 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.StreamMap; import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; +import org.apache.flink.testutils.junit.utils.TempDirUtils; import org.apache.flink.util.Collector; -import org.hamcrest.Matchers; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; /** Test writing keyed bootstrap state. */ -public class KeyedStateBootstrapOperatorTest { +class KeyedStateBootstrapOperatorTest { private static final ValueStateDescriptor descriptor = new ValueStateDescriptor<>("state", Types.LONG); @@ -54,11 +54,11 @@ public class KeyedStateBootstrapOperatorTest { private static final Long PROC_TIMER = Long.MAX_VALUE - 2; - @Rule public TemporaryFolder folder = new TemporaryFolder(); + @TempDir private java.nio.file.Path folder; @Test - public void testTimerStateRestorable() throws Exception { - Path path = new Path(folder.newFolder().toURI()); + void testTimerStateRestorable() throws Exception { + Path path = new Path(TempDirUtils.newFolder(folder).toURI()); OperatorSubtaskState state; KeyedStateBootstrapOperator bootstrapOperator = @@ -88,8 +88,8 @@ public void testTimerStateRestorable() throws Exception { } @Test - public void testNonTimerStatesRestorableByNonProcessesOperator() throws Exception { - Path path = new Path(folder.newFolder().toURI()); + void testNonTimerStatesRestorableByNonProcessesOperator() throws Exception { + Path path = new Path(TempDirUtils.newFolder(folder).toURI()); OperatorSubtaskState state; KeyedStateBootstrapOperator bootstrapOperator = @@ -123,7 +123,8 @@ private KeyedOneInputStreamOperatorTestHarness getHarness( bootstrapOperator, id -> id, Types.LONG, 128, 1, 0); harness.setStateBackend(new EmbeddedRocksDBStateBackend()); - harness.setCheckpointStorage(new FileSystemCheckpointStorage(folder.newFolder().toURI())); + harness.setCheckpointStorage( + new FileSystemCheckpointStorage(TempDirUtils.newFolder(folder).toURI())); if (state != null) { harness.initializeState(state); } @@ -151,10 +152,9 @@ private OperatorSubtaskState getState( private void assertHarnessOutput( KeyedOneInputStreamOperatorTestHarness harness, T... output) { - Assert.assertThat( - "The output from the operator does not match the expected values", - harness.extractOutputValues(), - Matchers.containsInAnyOrder(output)); + assertThat(harness.extractOutputValues()) + .as("The output from the operator does not match the expected values") + .containsExactlyInAnyOrder(output); } private static class TimerBootstrapFunction extends KeyedStateBootstrapFunction { diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SavepointOutputFormatTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SavepointOutputFormatTest.java index 6a425bea0dc042..8716cefff74256 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SavepointOutputFormatTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SavepointOutputFormatTest.java @@ -28,29 +28,32 @@ import org.apache.flink.state.api.runtime.SavepointLoader; import org.apache.flink.streaming.util.MockStreamingRuntimeContext; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.util.Collections; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + /** Test for writing output savepoint metadata. */ -public class SavepointOutputFormatTest { +class SavepointOutputFormatTest { - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @TempDir private File temporaryFolder; - @Test(expected = IllegalStateException.class) - public void testSavepointOutputFormatOnlyWorksWithParallelismOne() throws Exception { - Path path = new Path(temporaryFolder.newFolder().getAbsolutePath()); + @Test + void testSavepointOutputFormatOnlyWorksWithParallelismOne() throws Exception { + Path path = new Path(temporaryFolder.getAbsolutePath()); SavepointOutputFormat format = createSavepointOutputFormat(path); - format.open(FirstAttemptInitializationContext.of(0, 2)); + assertThatThrownBy(() -> format.open(FirstAttemptInitializationContext.of(0, 2))) + .isInstanceOf(IllegalStateException.class); } @Test - public void testSavepointOutputFormat() throws Exception { - Path path = new Path(temporaryFolder.newFolder().getAbsolutePath()); + void testSavepointOutputFormat() throws Exception { + Path path = new Path(temporaryFolder.getAbsolutePath()); SavepointOutputFormat format = createSavepointOutputFormat(path); CheckpointMetadata metadata = createSavepoint(); @@ -61,20 +64,17 @@ public void testSavepointOutputFormat() throws Exception { CheckpointMetadata metadataOnDisk = SavepointLoader.loadSavepointMetadata(path.getPath()); - Assert.assertEquals( - "Incorrect checkpoint id", - metadata.getCheckpointId(), - metadataOnDisk.getCheckpointId()); + assertThat(metadataOnDisk.getCheckpointId()) + .as("Incorrect checkpoint id") + .isEqualTo(metadata.getCheckpointId()); - Assert.assertEquals( - "Incorrect number of operator states in savepoint", - metadata.getOperatorStates().size(), - metadataOnDisk.getOperatorStates().size()); + assertThat(metadataOnDisk.getOperatorStates()) + .as("Incorrect number of operator states in savepoint") + .hasSameSizeAs(metadata.getOperatorStates()); - Assert.assertEquals( - "Incorrect operator state in savepoint", - metadata.getOperatorStates().iterator().next(), - metadataOnDisk.getOperatorStates().iterator().next()); + assertThat(metadataOnDisk.getOperatorStates().iterator().next()) + .as("Incorrect operator state in savepoint") + .isEqualTo(metadata.getOperatorStates().iterator().next()); } private CheckpointMetadata createSavepoint() { diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java index 00e37e63d4456a..302a9e65ac2d2c 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/SnapshotUtilsTest.java @@ -33,23 +33,24 @@ import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; + /** Tests that snapshot utils can properly snapshot an operator. */ -public class SnapshotUtilsTest { +class SnapshotUtilsTest { private static final List EXPECTED_CALL_OPERATOR_SNAPSHOT = Arrays.asList("prepareSnapshotPreBarrier", "snapshotState", "notifyCheckpointComplete"); - @Rule public TemporaryFolder folder = new TemporaryFolder(); + @TempDir private File folder; private static final List ACTUAL_ORDER_TRACKING = Collections.synchronizedList(new ArrayList<>(EXPECTED_CALL_OPERATOR_SNAPSHOT.size())); @@ -57,17 +58,17 @@ public class SnapshotUtilsTest { private static SnapshotType actualSnapshotType; @Test - public void testSnapshotUtilsLifecycleWithDefaultSavepointFormatType() throws Exception { + void testSnapshotUtilsLifecycleWithDefaultSavepointFormatType() throws Exception { testSnapshotUtilsLifecycleWithSavepointFormatType(SavepointFormatType.DEFAULT); } @Test - public void testSnapshotUtilsLifecycleWithCanonicalSavepointFormatType() throws Exception { + void testSnapshotUtilsLifecycleWithCanonicalSavepointFormatType() throws Exception { testSnapshotUtilsLifecycleWithSavepointFormatType(SavepointFormatType.CANONICAL); } @Test - public void testSnapshotUtilsLifecycleWithNativeSavepointFormatType() throws Exception { + void testSnapshotUtilsLifecycleWithNativeSavepointFormatType() throws Exception { testSnapshotUtilsLifecycleWithSavepointFormatType(SavepointFormatType.NATIVE); } @@ -75,7 +76,7 @@ private void testSnapshotUtilsLifecycleWithSavepointFormatType( SavepointFormatType savepointFormatType) throws Exception { ACTUAL_ORDER_TRACKING.clear(); StreamOperator operator = new LifecycleOperator(); - Path path = new Path(folder.newFolder().getAbsolutePath()); + Path path = new Path(folder.getAbsolutePath()); SnapshotUtils.snapshot( 0L, @@ -88,8 +89,8 @@ private void testSnapshotUtilsLifecycleWithSavepointFormatType( path, savepointFormatType); - Assert.assertEquals(SavepointType.savepoint(savepointFormatType), actualSnapshotType); - Assert.assertEquals(EXPECTED_CALL_OPERATOR_SNAPSHOT, ACTUAL_ORDER_TRACKING); + assertThat(actualSnapshotType).isEqualTo(SavepointType.savepoint(savepointFormatType)); + assertThat(ACTUAL_ORDER_TRACKING).isEqualTo(EXPECTED_CALL_OPERATOR_SNAPSHOT); } private static class LifecycleOperator implements StreamOperator { diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/runtime/OperatorIDGeneratorTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/runtime/OperatorIDGeneratorTest.java index 9e471297b9f5f1..f6d29a49201d6f 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/runtime/OperatorIDGeneratorTest.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/runtime/OperatorIDGeneratorTest.java @@ -25,27 +25,28 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.stream.StreamSupport; +import static org.assertj.core.api.Assertions.assertThat; + /** * Test that {@code OperatorIDGenerator} creates ids from uids exactly the same as the job graph * generator. */ -public class OperatorIDGeneratorTest { +class OperatorIDGeneratorTest { private static final String UID = "uid"; private static final String OPERATOR_NAME = "operator"; @Test - public void testOperatorIdMatchesUid() { + void testOperatorIdMatchesUid() { OperatorID expectedId = getOperatorID(); OperatorID generatedId = OperatorIDGenerator.fromUid(UID); - Assert.assertEquals(expectedId, generatedId); + assertThat(generatedId).isEqualTo(expectedId); } private static OperatorID getOperatorID() { diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/utils/SavepointTestBase.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/utils/SavepointTestBase.java index 97d14baa121134..0da89badf9adef 100644 --- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/utils/SavepointTestBase.java +++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/utils/SavepointTestBase.java @@ -23,6 +23,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.core.execution.SavepointFormatType; import org.apache.flink.runtime.execution.ExecutionState; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -30,10 +31,12 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.source.legacy.FromElementsFunction; import org.apache.flink.streaming.api.functions.source.legacy.SourceFunction; -import org.apache.flink.test.util.AbstractTestBaseJUnit4; -import org.apache.flink.test.util.MiniClusterWithClientResource; +import org.apache.flink.test.junit5.InjectClusterClient; +import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.util.AbstractID; +import org.junit.jupiter.api.BeforeEach; + import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -46,7 +49,14 @@ import static org.apache.flink.runtime.execution.ExecutionState.RUNNING; /** A test base that includes utilities for taking a savepoint. */ -public abstract class SavepointTestBase extends AbstractTestBaseJUnit4 { +public abstract class SavepointTestBase extends AbstractTestBase { + + private RestClusterClient clusterClient; + + @BeforeEach + void setClusterClient(@InjectClusterClient RestClusterClient clusterClient) { + this.clusterClient = clusterClient; + } public String takeSavepoint(StreamExecutionEnvironment executionEnvironment) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -56,26 +66,23 @@ public String takeSavepoint(StreamExecutionEnvironment executionEnvironment) { JobID jobId = jobGraph.getJobID(); - ClusterClient client = MINI_CLUSTER_RESOURCE.getClusterClient(); - try { - JobID jobID = client.submitJob(jobGraph).get(); + JobID jobID = clusterClient.submitJob(jobGraph).get(); - waitForAllRunningOrSomeTerminal(jobID, MINI_CLUSTER_RESOURCE); + waitForAllRunningOrSomeTerminal(jobID, clusterClient); - return triggerSavepoint(client, jobID).get(5, TimeUnit.MINUTES); + return triggerSavepoint(clusterClient, jobID).get(5, TimeUnit.MINUTES); } catch (Exception e) { throw new RuntimeException("Failed to take savepoint", e); } finally { - client.cancel(jobId); + clusterClient.cancel(jobId); } } public static void waitForAllRunningOrSomeTerminal( - JobID jobID, MiniClusterWithClientResource miniClusterResource) throws Exception { + JobID jobID, RestClusterClient clusterClient) throws Exception { while (true) { - JobDetailsInfo jobInfo = - miniClusterResource.getRestClusterClient().getJobDetails(jobID).get(); + JobDetailsInfo jobInfo = clusterClient.getJobDetails(jobID).get(); Set vertexStates = jobInfo.getJobVertexInfos().stream() .map(JobDetailsInfo.JobVertexDetailsInfo::getExecutionState) diff --git a/flink-python/pyflink/fn_execution/table/async_function/operations.py b/flink-python/pyflink/fn_execution/table/async_function/operations.py index 5d536c4749663b..da325ab9aa3bd2 100644 --- a/flink-python/pyflink/fn_execution/table/async_function/operations.py +++ b/flink-python/pyflink/fn_execution/table/async_function/operations.py @@ -60,6 +60,11 @@ def __init__(self, serialized_fn): operation_utils.extract_user_defined_function( serialized_fn.udfs[0], one_arg_optimization=False) + # Mirror PythonScalarFunctionOperator.createInputCoderInfoDescriptor: + # Java picks FlattenRowCoder unless some UDF takes a row as input. + self._input_is_flatten_row = not any( + udf.takes_row_as_input for udf in serialized_fn.udfs) + # Create the eval function self._eval_func = eval('lambda value: %s' % scalar_function, variable_dict) @@ -140,6 +145,12 @@ def process_element(self, value): """ self._raise_exception_if_exists() + # The Cython FlattenRowCoderImpl returns a reused list whose slots are + # overwritten by the next decode, so we must snapshot the row before + # the async closure can capture it. + if self._input_is_flatten_row: + value = list(value) + entry = self._queue.put(None, 0, 0, value) async def execute_async(rh): diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml index 94709b108797d5..966a7462e2e100 100644 --- a/flink-runtime-web/pom.xml +++ b/flink-runtime-web/pom.xml @@ -265,7 +265,7 @@ under the License. com.github.eirslett frontend-maven-plugin - 1.15.1 + 2.0.1 install node and npm diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandler.java index 8b93ef01349ff5..d86191d512fd3f 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandler.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandler.java @@ -19,12 +19,17 @@ package org.apache.flink.runtime.webmonitor.history; +import org.apache.flink.configuration.HistoryServerOptions; +import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.rest.NotFoundException; import org.apache.flink.runtime.rest.handler.RestHandlerException; import org.apache.flink.runtime.rest.handler.legacy.files.StaticFileServerHandler; import org.apache.flink.runtime.rest.handler.router.RoutedRequest; import org.apache.flink.runtime.rest.handler.util.HandlerUtils; +import org.apache.flink.runtime.rest.messages.ApplicationsOverviewHeaders; import org.apache.flink.runtime.rest.messages.ErrorResponseBody; +import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders; +import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelFuture; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelFutureListener; @@ -45,6 +50,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -57,7 +64,11 @@ import java.util.Collections; import java.util.Date; import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.JSON_FILE_ENDING; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaderNames.IF_MODIFIED_SINCE; import static org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; @@ -73,14 +84,34 @@ public abstract class AbstractHistoryServerHandler private static final Logger LOG = LoggerFactory.getLogger(AbstractHistoryServerHandler.class); + /** Matches jobs/{jobId} or jobs/{jobId}/... paths. */ + private static final Pattern JOB_ID_PATTERN = Pattern.compile("^jobs/([^/]+)(?:/.*)?$"); + + /** Matches applications/{appId}. */ + private static final Pattern APPLICATION_ID_PATTERN = Pattern.compile("^applications/([^/]+)$"); + /** The path in which the static documents are. */ protected final File rootPath; protected final ArchiveStorage archiveStorage; - protected AbstractHistoryServerHandler(ArchiveStorage archiveStorage, File rootPath) + private final HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode; + + // for lazy load + @Nullable protected HistoryServerArchiveFetcher archiveFetcher; + @Nullable protected HistoryServerApplicationArchiveFetcher applicationArchiveFetcher; + + protected AbstractHistoryServerHandler( + ArchiveStorage archiveStorage, + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode, + @Nullable HistoryServerArchiveFetcher archiveFetcher, + @Nullable HistoryServerApplicationArchiveFetcher applicationArchiveFetcher, + File rootPath) throws IOException { this.archiveStorage = archiveStorage; + this.archiveLoadMode = archiveLoadMode; + this.archiveFetcher = archiveFetcher; + this.applicationArchiveFetcher = applicationArchiveFetcher; this.rootPath = checkNotNull(rootPath).getCanonicalFile(); } @@ -136,11 +167,12 @@ protected void respondToRequest(ChannelHandlerContext ctx, RoutedRequest routedR requestPath = requestPath + "index.html"; } - if (!requestPath.contains(".")) { // we assume that the path ends in either .html or .js + // we assume that the path ends in either .html or .js + if (!requestPath.contains(".")) { requestPath = requestPath + ".json"; LOG.debug("Responding to request for path {}", requestPath); - Entry resource = loadResource(requestPath); + Entry resource = loadResource(requestPath, archiveLoadMode); if (resource == null) { LOG.debug("Unable to load requested resource {}", requestPath); @@ -266,14 +298,67 @@ protected void responseWithFile( /** * Loads the resource for the given request path from the archive storage. * + *

The resource has four cases: + * + *

1. /index.html or other web resource files, should be loaded from classloader {@link + * #tryLoadFromClassloader} + * + *

2. /config.json, will be created when HistoryServer started + * + *

3. /jobs/overview.json (and /jobs/jobid.json) or /applications/overview.json (and + * /applications/applicationid.json), will be loaded synchronously + * + *

4. /jobs/<jobid>/.. will be loaded asynchronously + * * @param requestPath The request path + * @param archiveLoadMode The archive load mode * @return The resource for the given request path, or null if not found */ - private Entry loadResource(String requestPath) throws Exception { + private Entry loadResource( + String requestPath, HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode) + throws Exception { String requestKey = requestPath.startsWith("/") ? requestPath.substring(1) : requestPath; + + if (LAZY.equals(archiveLoadMode)) { + Preconditions.checkNotNull(archiveFetcher); + Preconditions.checkNotNull(applicationArchiveFetcher); + // need to update for overview + if (requestKey.equals(JobsOverviewHeaders.URL.substring(1) + JSON_FILE_ENDING) + || requestKey.equals( + ApplicationsOverviewHeaders.URL.substring(1) + JSON_FILE_ENDING)) { + archiveFetcher.fetchArchives(archiveLoadMode); + applicationArchiveFetcher.fetchArchives(archiveLoadMode); + return archiveStorage.getEntry(requestKey); + } + // for application/${applicationId}, return directly + String applicationId = extractApplicationId(requestKey); + if (applicationId != null) { + return archiveStorage.getEntry(requestKey); + } + // for job/${jobId}... + String jobId = extractJobId(requestKey); + if (jobId != null) { + if (archiveStorage.exists(requestKey)) { + return archiveStorage.getEntry(requestKey); + } + ArchiveMetaInfo jobArchiveMetaInfo = archiveFetcher.getArchiveMetaInfo(jobId); + if (archiveFetcher.needLazyLoadIndividually(jobId)) { + Path archivePath = + jobArchiveMetaInfo == null ? null : jobArchiveMetaInfo.getArchivePath(); + archiveFetcher.lazyFetchArchiveProactively(jobId, archivePath); + } + // wait for the job archive to be loaded + if (!requestKey.endsWith(jobId + JSON_FILE_ENDING)) { + archiveFetcher.waitLazyFetchArchiveFinished(jobId); + } + return archiveStorage.getEntry(requestKey); + } + } + if (archiveStorage.exists(requestKey)) { return archiveStorage.getEntry(requestKey); } + return null; } @@ -320,4 +405,42 @@ protected void tryLoadFromClassloader(File destFile, String requestPath) throws } } } + + /** + * Extracts the job ID from the request path. + * + * @param requestPath Request path, e.g., {@code /jobs/abc123.../vertices} + * @return jobId string; returns {@code null} if path does not match + */ + @Nullable + protected static String extractJobId(String requestPath) { + Matcher matcher = JOB_ID_PATTERN.matcher(requestPath); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } + + /** + * Extracts the application ID from the request path. + * + * @param requestPath Request path, e.g., {@code /applications/abc123...} + * @return applicationId string; returns {@code null} if path does not match + */ + @Nullable + protected static String extractApplicationId(String requestPath) { + Matcher matcher = APPLICATION_ID_PATTERN.matcher(requestPath); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } + + /** Factory for creating instances of {@link AbstractHistoryServerHandler}. */ + public interface HistoryServerHandlerFactory { + AbstractHistoryServerHandler createHistoryServerHandler( + HistoryServerArchiveFetcher archiveFetcher, + HistoryServerApplicationArchiveFetcher applicationArchiveFetcher) + throws IOException; + } } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/ArchiveMetaInfo.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/ArchiveMetaInfo.java new file mode 100644 index 00000000000000..7a520506034e9e --- /dev/null +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/ArchiveMetaInfo.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.webmonitor.history; + +import org.apache.flink.core.fs.Path; + +/** Meta info for archived job. */ +public class ArchiveMetaInfo { + + private final String archiveId; + private volatile HistoryServerArchiveFetcher.ArchiveEventType eventType; + private final Path archivePath; + + public ArchiveMetaInfo( + String archiveId, + HistoryServerArchiveFetcher.ArchiveEventType eventType, + Path archivePath) { + this.archiveId = archiveId; + this.eventType = eventType; + this.archivePath = archivePath; + } + + public String getArchiveId() { + return archiveId; + } + + public HistoryServerArchiveFetcher.ArchiveEventType getEventType() { + return eventType; + } + + public void setEventType(HistoryServerArchiveFetcher.ArchiveEventType eventType) { + this.eventType = eventType; + } + + public Path getArchivePath() { + return archivePath; + } +} diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServer.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServer.java index ed59bf2db4c3fc..151fd43b8233f3 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServer.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServer.java @@ -68,6 +68,7 @@ import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -75,6 +76,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static org.apache.flink.configuration.HistoryServerOptions.HISTORY_SERVER_LAZY_FETCH_EXECUTOR_COMMON_POOL_SIZE; +import static org.apache.flink.configuration.HistoryServerOptions.HISTORY_SERVER_LAZY_FETCH_EXECUTOR_INDIVIDUAL_POOL_SIZE; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; import static org.apache.flink.runtime.webmonitor.history.HistoryServerApplicationArchiveFetcher.APPLICATIONS_SUBDIR; import static org.apache.flink.runtime.webmonitor.history.HistoryServerApplicationArchiveFetcher.APPLICATION_OVERVIEWS_SUBDIR; import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.JOBS_SUBDIR; @@ -141,6 +145,7 @@ public class HistoryServer { private final Thread shutdownHook; private final ArchiveStorage archiveStorage; + private final HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode; private final AbstractHistoryServerHandler historyServerHandler; public static void main(String[] args) throws Exception { @@ -250,12 +255,13 @@ public HistoryServer( throw new FlinkException( "Failed to validate any of the configured directories to monitor."); } - refreshIntervalMillis = config.get(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_REFRESH_INTERVAL).toMillis(); + archiveLoadMode = config.get(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_LOAD_MODE); HistoryServerOptions.HistoryServerArchiveStorageType archiveStorageType = config.get(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_STORAGE_TYPE); + AbstractHistoryServerHandler.HistoryServerHandlerFactory historyServerHandlerFactory; switch (archiveStorageType) { case FILE: // create directories for job and application overview updates @@ -264,22 +270,27 @@ public HistoryServer( Files.createDirectories(webDir.toPath().resolve(APPLICATIONS_SUBDIR)); Files.createDirectories(webDir.toPath().resolve(APPLICATION_OVERVIEWS_SUBDIR)); archiveStorage = new FileArchiveStorage(webDir); - historyServerHandler = - new HistoryServerStaticFileServerHandler( - (FileArchiveStorage) archiveStorage, webDir); + historyServerHandlerFactory = + createFileHandlerFactory((FileArchiveStorage) archiveStorage, webDir); break; case ROCKSDB: File dbPath = new File(webDir, "rocksdb-" + UUID.randomUUID()); Files.createDirectories(dbPath.toPath()); archiveStorage = new RocksDBArchiveStorage(dbPath, config); - historyServerHandler = - new HistoryServerRocksDBHandler( - (RocksDBArchiveStorage) archiveStorage, webDir); + historyServerHandlerFactory = + createRocksDBHandlerFactory((RocksDBArchiveStorage) archiveStorage, webDir); break; default: throw new FlinkException("Unsupported archive storage type: " + archiveStorageType); } + ConcurrentHashMap archiveMetaInfoCache = new ConcurrentHashMap<>(); + ConcurrentHashMap applicationArchiveMetaInfoCache = + new ConcurrentHashMap<>(); + int lazyFetchExecutorCommonPoolSize = + config.get(HISTORY_SERVER_LAZY_FETCH_EXECUTOR_COMMON_POOL_SIZE); + int lazyFetchExecutorIndividualPoolSize = + config.get(HISTORY_SERVER_LAZY_FETCH_EXECUTOR_INDIVIDUAL_POOL_SIZE); archiveFetcher = new HistoryServerArchiveFetcher<>( refreshDirs, @@ -287,7 +298,10 @@ public HistoryServer( jobArchiveEventListener, cleanupExpiredJobs, CompositeArchiveRetainedStrategy.createForJobFromConfig(config), - archiveStorage); + archiveStorage, + archiveMetaInfoCache, + lazyFetchExecutorCommonPoolSize, + lazyFetchExecutorIndividualPoolSize); applicationArchiveFetcher = new HistoryServerApplicationArchiveFetcher<>( refreshDirs, @@ -295,7 +309,15 @@ public HistoryServer( applicationArchiveEventListener, cleanupExpiredApplications, CompositeArchiveRetainedStrategy.createForApplicationFromConfig(config), - archiveStorage); + archiveStorage, + archiveMetaInfoCache, + applicationArchiveMetaInfoCache, + lazyFetchExecutorCommonPoolSize, + lazyFetchExecutorIndividualPoolSize); + + historyServerHandler = + historyServerHandlerFactory.createHistoryServerHandler( + archiveFetcher, applicationArchiveFetcher); this.shutdownHook = ShutdownHookUtil.addShutdownHook( @@ -337,7 +359,7 @@ int getWebPort() { @VisibleForTesting void fetchArchives() { - executor.execute(getArchiveFetchingRunnable()); + executor.execute(getArchiveFetchingRunnable(archiveLoadMode)); } public void run() { @@ -351,6 +373,30 @@ public void run() { } } + @SuppressWarnings("unchecked") + private AbstractHistoryServerHandler.HistoryServerHandlerFactory createFileHandlerFactory( + FileArchiveStorage fileArchiveStorage, File webDir) { + return (archiveFetcher, applicationArchiveFetcher) -> + new HistoryServerStaticFileServerHandler( + fileArchiveStorage, + archiveLoadMode, + (HistoryServerArchiveFetcher) archiveFetcher, + (HistoryServerApplicationArchiveFetcher) applicationArchiveFetcher, + webDir); + } + + @SuppressWarnings("unchecked") + private AbstractHistoryServerHandler.HistoryServerHandlerFactory createRocksDBHandlerFactory( + RocksDBArchiveStorage rocksDBArchiveStorage, File webDir) { + return (archiveFetcher, applicationArchiveFetcher) -> + new HistoryServerRocksDBHandler( + rocksDBArchiveStorage, + archiveLoadMode, + (HistoryServerArchiveFetcher) archiveFetcher, + (HistoryServerApplicationArchiveFetcher) applicationArchiveFetcher, + webDir); + } + // ------------------------------------------------------------------------ // Life-cycle // ------------------------------------------------------------------------ @@ -384,11 +430,22 @@ void start() throws IOException, InterruptedException { CompletableFuture.completedFuture(pattern)))); createDashboardConfigFile(); - router.addGet("/:*", historyServerHandler); - executor.scheduleWithFixedDelay( - getArchiveFetchingRunnable(), 0, refreshIntervalMillis, TimeUnit.MILLISECONDS); + if (LAZY.equals(archiveLoadMode)) { + executor.submit(getArchiveFetchingRunnable(archiveLoadMode)); + executor.scheduleWithFixedDelay( + getArchiveCleaningRunnable(), + refreshIntervalMillis, + refreshIntervalMillis, + TimeUnit.MILLISECONDS); + } else { + executor.scheduleWithFixedDelay( + getArchiveFetchingRunnable(archiveLoadMode), + 0, + refreshIntervalMillis, + TimeUnit.MILLISECONDS); + } netty = new WebFrontendBootstrap( @@ -396,11 +453,21 @@ void start() throws IOException, InterruptedException { } } - private Runnable getArchiveFetchingRunnable() { + private Runnable getArchiveFetchingRunnable( + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode) { + return Runnables.withUncaughtExceptionHandler( + () -> { + archiveFetcher.fetchArchives(archiveLoadMode); + applicationArchiveFetcher.fetchArchives(archiveLoadMode); + }, + FatalExitExceptionHandler.INSTANCE); + } + + private Runnable getArchiveCleaningRunnable() { return Runnables.withUncaughtExceptionHandler( () -> { - archiveFetcher.fetchArchives(); - applicationArchiveFetcher.fetchArchives(); + archiveFetcher.cleanUpArchives(archiveLoadMode); + applicationArchiveFetcher.cleanUpArchives(archiveLoadMode); }, FatalExitExceptionHandler.INSTANCE); } @@ -424,6 +491,18 @@ void stop() { LOG.warn("Error while closing archive storage.", t); } + try { + archiveFetcher.close(); + } catch (Throwable t) { + LOG.warn("Error while closing archive fetcher.", t); + } + + try { + applicationArchiveFetcher.close(); + } catch (Throwable t) { + LOG.warn("Error while closing application archive fetcher.", t); + } + try { LOG.info("Removing web dashboard root cache directory {}", webDir); FileUtils.deleteDirectory(webDir); @@ -466,7 +545,7 @@ static class RefreshLocation { private final Path path; private final FileSystem fs; - private RefreshLocation(Path path, FileSystem fs) { + RefreshLocation(Path path, FileSystem fs) { this.path = path; this.fs = fs; } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcher.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcher.java index 10b8161b95bd80..038abc856323cc 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcher.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcher.java @@ -42,8 +42,13 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.EAGER; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.OVERVIEW_PARSING; + /** * This class is used by the {@link HistoryServer} to fetch the application and job archives that * are located at {@link HistoryServerOptions#HISTORY_SERVER_ARCHIVE_DIRS}. The directories are @@ -73,21 +78,32 @@ public class HistoryServerApplicationArchiveFetcher private final Map>> cachedApplicationIdsToJobIds = new HashMap<>(); + private final ConcurrentHashMap applicationArchiveMetaInfoCache; + HistoryServerApplicationArchiveFetcher( List refreshDirs, File webDir, - Consumer archiveEventListener, + Consumer archiveEventListener, boolean cleanupExpiredArchives, ArchiveRetainedStrategy retainedStrategy, - ArchiveStorage archiveStorage) { + ArchiveStorage archiveStorage, + ConcurrentHashMap archiveMetaInfoCache, + ConcurrentHashMap applicationArchiveMetaInfoCache, + int lazyFetchExecutorCommonPoolSize, + int lazyFetchExecutorIndividualPoolSize) + throws IOException { super( refreshDirs, webDir, archiveEventListener, cleanupExpiredArchives, retainedStrategy, - archiveStorage); + archiveStorage, + archiveMetaInfoCache, + lazyFetchExecutorCommonPoolSize, + lazyFetchExecutorIndividualPoolSize); + this.applicationArchiveMetaInfoCache = applicationArchiveMetaInfoCache; for (HistoryServer.RefreshLocation refreshDir : refreshDirs) { cachedApplicationIdsToJobIds.put(refreshDir.getPath(), new HashMap<>()); } @@ -143,7 +159,16 @@ private boolean isValidId(String id, Path refreshDir) { @Override List processArchive(String archiveId, Path archivePath, Path refreshDir) - throws IOException { + throws Exception { + return processArchive(archiveId, archivePath, refreshDir, EAGER); + } + + List processArchive( + String archiveId, + Path archivePath, + Path refreshDir, + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode) + throws Exception { FileSystem fs = archivePath.getFileSystem(); Path applicationArchive = new Path(archivePath, ArchivePathUtils.APPLICATION_ARCHIVE_NAME); if (!fs.exists(applicationArchive)) { @@ -162,7 +187,11 @@ List processArchive(String archiveId, Path archivePath, Path refre .get(refreshDir) .computeIfAbsent(archiveId, k -> new HashSet<>()) .add(jobId); - events.add(processJobArchive(jobId, jobArchive.getPath())); + ArchiveEvent processArchiveEvents = + LAZY.equals(archiveLoadMode) + ? lazyProcessJobArchive(jobId, jobArchive.getPath(), false) + : processJobArchive(jobId, jobArchive.getPath()); + events.add(processArchiveEvents); } return events; @@ -234,6 +263,7 @@ private ArchiveEvent deleteApplicationFiles(String applicationId) { LOG.warn("Could not delete file from application directory.", ioe); } + applicationArchiveMetaInfoCache.remove(applicationId); return new ArchiveEvent(applicationId, ArchiveEventType.DELETED); } @@ -282,4 +312,35 @@ private void updateApplicationOverview() { LOG.error("Failed to update application overview.", e); } } + + @Override + List lazyProcessArchive(String archiveId, Path archivePath, Path refreshDir) + throws Exception { + List events = new ArrayList<>(); + ArchiveMetaInfo archiveMetaInfo = + new ArchiveMetaInfo(archiveId, OVERVIEW_PARSING, archivePath); + ArchiveMetaInfo existing = + applicationArchiveMetaInfoCache.putIfAbsent(archiveId, archiveMetaInfo); + if (existing != null) { + events.add(new ArchiveEvent(archiveId, existing.getEventType())); + return events; + } + + events.addAll(processArchive(archiveId, archivePath, refreshDir, LAZY)); + + archiveMetaInfo.setEventType(ArchiveEventType.CREATED); + return events; + } + + @Override + void cleanUpLazyFetchTask(String archiveId) { + for (HistoryServer.RefreshLocation refreshDir : refreshDirs) { + Path refreshDirPath = refreshDir.getPath(); + if (cachedApplicationIdsToJobIds.get(refreshDirPath).containsKey(archiveId)) { + Set jobIds = + cachedApplicationIdsToJobIds.get(refreshDirPath).get(archiveId); + jobIds.forEach(super::cleanUpLazyFetchTask); + } + } + } } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java index e64114ef3da1b7..e667d9cd365214 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.webmonitor.history; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobStatus; import org.apache.flink.configuration.HistoryServerOptions; @@ -30,6 +31,8 @@ import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails; import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders; import org.apache.flink.runtime.webmonitor.history.retaining.ArchiveRetainedStrategy; +import org.apache.flink.util.ExecutorUtils; +import org.apache.flink.util.concurrent.ExecutorThreadFactory; import org.apache.flink.util.jackson.JacksonMapperFactory; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; @@ -38,6 +41,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.File; import java.io.IOException; import java.io.StringWriter; @@ -50,8 +55,15 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.PENDING; import static org.apache.flink.util.Preconditions.checkNotNull; /** @@ -66,10 +78,18 @@ * * @param the type of entries returned by the underlying {@link ArchiveStorage}. */ -public class HistoryServerArchiveFetcher { +public class HistoryServerArchiveFetcher implements AutoCloseable { /** Possible archive operations in history-server. */ public enum ArchiveEventType { + /** Archive is pending to be processed. */ + PENDING, + /** Overview content is currently parsing. */ + OVERVIEW_PARSING, + /** Overview content of archive was parsed and created in history server successfully. */ + OVERVIEW_CREATED, + /** Detail content of archive is currently parsing. */ + DETAIL_PARSING, /** Archive was found in one refresh location and created in history server. */ CREATED, /** Archive was deleted from one of refresh locations and deleted from history server. */ @@ -115,13 +135,25 @@ public ArchiveEventType getType() { protected final ArchiveStorage archiveStorage; + /** Executor for loading archives. */ + private final ExecutorService commonFetchExecutor; + + private final ExecutorService individualFetchExecutor; + private final Map> commonFetchTasks; + private final Map> individualFetchTasks; + private final ConcurrentHashMap archiveMetaInfoCache; + HistoryServerArchiveFetcher( List refreshDirs, File webDir, Consumer archiveEventListener, boolean cleanupExpiredArchives, ArchiveRetainedStrategy retainedStrategy, - ArchiveStorage archiveStorage) { + ArchiveStorage archiveStorage, + ConcurrentHashMap archiveMetaInfoCache, + int lazyFetchExecutorCommonPoolSize, + int lazyFetchExecutorIndividualPoolSize) + throws IOException { this.refreshDirs = checkNotNull(refreshDirs); this.archiveEventListener = archiveEventListener; this.processExpiredArchiveDeletion = cleanupExpiredArchives; @@ -132,6 +164,17 @@ public ArchiveEventType getType() { } checkNotNull(webDir); this.archiveStorage = archiveStorage; + this.archiveMetaInfoCache = archiveMetaInfoCache; + this.commonFetchExecutor = + Executors.newFixedThreadPool( + lazyFetchExecutorCommonPoolSize, + new ExecutorThreadFactory("HistoryServer-commonFetchExecutor")); + this.individualFetchExecutor = + Executors.newFixedThreadPool( + lazyFetchExecutorIndividualPoolSize, + new ExecutorThreadFactory("HistoryServer-individualFetchExecutor")); + this.commonFetchTasks = new ConcurrentHashMap<>(); + this.individualFetchTasks = new ConcurrentHashMap<>(); updateJobOverview(); if (LOG.isInfoEnabled()) { @@ -141,9 +184,15 @@ public ArchiveEventType getType() { } } - void fetchArchives() { + void fetchArchives(HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode) { + LOG.debug("Starting archive fetching."); + scanArchives(archiveLoadMode, true); + } + + void scanArchives( + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode, boolean fetch) { + LOG.debug("Starting archive fetching."); try { - LOG.debug("Starting archive fetching."); List events = new ArrayList<>(); Map> archivesToRemove = new HashMap<>(); cachedArchivesPerRefreshDirectory.forEach( @@ -180,28 +229,18 @@ void fetchArchives() { continue; } - if (cachedArchivesPerRefreshDirectory.get(refreshDir).contains(archiveId)) { - LOG.trace( - "Ignoring archive {} because it was already fetched.", archivePath); - } else { - LOG.info("Processing archive {}.", archivePath); - try { - events.addAll(processArchive(archiveId, archivePath, refreshDir)); - cachedArchivesPerRefreshDirectory.get(refreshDir).add(archiveId); - LOG.info("Processing archive {} finished.", archivePath); - } catch (IOException e) { - LOG.error( - "Failure while fetching/processing archive {}.", archiveId, e); - deleteCachedArchives(archiveId, refreshDir); - } + if (fetch) { + fetchArchive(refreshDir, archiveId, archivePath, archiveLoadMode, events); } } } + // clean local if (archivesToRemove.values().stream().flatMap(Set::stream).findAny().isPresent() && processExpiredArchiveDeletion) { events.addAll(cleanupExpiredArchives(archivesToRemove)); } + // clean remote and local if (!archivesBeyondRetainedLimit.isEmpty()) { events.addAll(cleanupArchivesBeyondRetainedLimit(archivesBeyondRetainedLimit)); } @@ -209,12 +248,38 @@ void fetchArchives() { updateOverview(); } events.forEach(archiveEventListener); - LOG.debug("Finished archive fetching."); + LOG.debug("Finished archive scan."); } catch (Exception e) { LOG.error("Critical failure while fetching/processing archives.", e); } } + private void fetchArchive( + Path refreshDir, + String archiveId, + Path archivePath, + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode, + List events) + throws Exception { + if (cachedArchivesPerRefreshDirectory.get(refreshDir).contains(archiveId)) { + LOG.trace("Ignoring archive {} because it was already fetched.", archivePath); + } else { + LOG.info("Processing archive {}.", archivePath); + try { + List processArchiveEvents = + LAZY.equals(archiveLoadMode) + ? lazyProcessArchive(archiveId, archivePath, refreshDir) + : processArchive(archiveId, archivePath, refreshDir); + events.addAll(processArchiveEvents); + cachedArchivesPerRefreshDirectory.get(refreshDir).add(archiveId); + LOG.info("Processing archive {} finished.", archivePath); + } catch (Exception e) { + LOG.error("Failure while fetching/processing archive {}.", archiveId, e); + deleteCachedArchives(archiveId, refreshDir); + } + } + } + List listValidArchives(FileSystem refreshFS, Path refreshDir) throws IOException { return listValidJobArchives(refreshFS, refreshDir); } @@ -255,7 +320,7 @@ boolean isValidJobId(String jobId, Path refreshDir) { } List processArchive(String archiveId, Path archivePath, Path refreshDir) - throws IOException { + throws Exception { return Collections.singletonList(processJobArchive(archiveId, archivePath)); } @@ -314,8 +379,10 @@ List cleanupExpiredArchives(Map> archivesToRemov (refreshDir, archives) -> { cachedArchivesPerRefreshDirectory.get(refreshDir).removeAll(archives); archives.forEach( - archiveId -> - deleteLog.addAll(deleteCachedArchives(archiveId, refreshDir))); + archiveId -> { + cleanUpLazyFetchTask(archiveId); + deleteLog.addAll(deleteCachedArchives(archiveId, refreshDir)); + }); }); return deleteLog; @@ -348,6 +415,7 @@ ArchiveEvent deleteJobFiles(String jobId) { LOG.warn("Could not delete file from job directory.", ioe); } + archiveMetaInfoCache.remove(jobId); return new ArchiveEvent(jobId, ArchiveEventType.DELETED); } @@ -458,4 +526,186 @@ void updateJobOverview() { LOG.error("Failed to update job overview.", e); } } + + // -------------------------------- Lazy Load ---------------------------------------- + List lazyProcessArchive(String archiveId, Path archivePath, Path refreshDir) + throws Exception { + return Collections.singletonList(lazyProcessJobArchive(archiveId, archivePath, false)); + } + + ArchiveEvent lazyProcessJobArchive(String jobId, Path jobArchive, boolean individual) + throws Exception { + final ArchiveMetaInfo archiveMetaInfo = new ArchiveMetaInfo(jobId, PENDING, jobArchive); + ArchiveMetaInfo existing = archiveMetaInfoCache.putIfAbsent(jobId, archiveMetaInfo); + if (existing != null) { + return new ArchiveEvent(jobId, existing.getEventType()); + } + archiveMetaInfo.setEventType(ArchiveEventType.OVERVIEW_PARSING); + + ExecutorService fetchExecutor; + Map> fetchTasks; + if (individual) { + fetchExecutor = individualFetchExecutor; + fetchTasks = individualFetchTasks; + } else { + fetchExecutor = commonFetchExecutor; + fetchTasks = commonFetchTasks; + } + + archiveMetaInfo.setEventType(ArchiveEventType.OVERVIEW_PARSING); + + Collection archivedJsons = FsJsonArchivist.readArchivedJsons(jobArchive); + List detailArchives = new ArrayList<>(); + boolean overviewCreated = false; + + for (ArchivedJson archive : archivedJsons) { + String path = archive.getPath(); + String json = archive.getJson(); + + if (path.equals(JobsOverviewHeaders.URL)) { + String key = JOB_OVERVIEWS_KEY_PREFIX + jobId + JSON_FILE_ENDING; + archiveStorage.putArchiveContent(key, json); + overviewCreated = true; + } else if (path.equals("/joboverview")) { // legacy path + LOG.debug("Migrating legacy archive {}", jobArchive); + json = convertLegacyJobOverview(json); + String key = JOB_OVERVIEWS_KEY_PREFIX + jobId + JSON_FILE_ENDING; + archiveStorage.putArchiveContent(key, json); + overviewCreated = true; + } else if (path.equals("/jobs/" + jobId)) { + String key = JOBS_KEY_PREFIX + jobId + JSON_FILE_ENDING; + archiveStorage.putArchiveContent(key, json); + } else { + detailArchives.add(archive); + } + } + + if (!overviewCreated && detailArchives.isEmpty()) { + archiveMetaInfoCache.remove(jobId); + throw new RuntimeException("Archive of job " + jobId + " is empty"); + } + + if (!detailArchives.isEmpty()) { + Future future = + fetchExecutor.submit( + () -> { + try { + archiveMetaInfo.setEventType(ArchiveEventType.DETAIL_PARSING); + for (ArchivedJson archive : detailArchives) { + String path = archive.getPath(); + String json = archive.getJson(); + // this implicitly writes into webJobDir; strip the leading + // '/' from the + // REST path so that the key is a relative sub-path under + // the storage root + String key = path.substring(1) + JSON_FILE_ENDING; + try { + archiveStorage.putArchiveContent(key, json); + } catch (IOException e) { + LOG.error( + "Failed to write detail archive file for job {}, path {}.", + jobId, + path, + e); + } + } + archiveMetaInfo.setEventType(ArchiveEventType.CREATED); + if (individual) { + updateOverview(); + } + LOG.debug("Async detail parsing for job {} finished.", jobId); + } finally { + fetchTasks.remove(jobId); + } + }); + fetchTasks.put(jobId, future); + } + + ArchiveEventType archiveEventType = + overviewCreated ? ArchiveEventType.OVERVIEW_CREATED : ArchiveEventType.CREATED; + archiveMetaInfo.setEventType(archiveEventType); + return new ArchiveEvent(jobId, archiveEventType); + } + + void lazyFetchArchiveProactively(String jobId, @Nullable Path archivePath) throws Exception { + resetWhenTriggerLazyFetch(jobId); + + if (archivePath != null) { + lazyProcessJobArchive(jobId, archivePath, true); + return; + } + + for (HistoryServer.RefreshLocation refreshDir : refreshDirs) { + archivePath = new Path(refreshDir.getPath(), jobId); + if (refreshDir.getFs().exists(archivePath)) { + lazyProcessJobArchive(jobId, archivePath, true); + } + } + } + + void cleanUpArchives(HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode) { + LOG.debug("Starting archive cleanup."); + scanArchives(archiveLoadMode, false); + } + + boolean needLazyLoadIndividually(String jobId) { + ArchiveMetaInfo archiveMetaInfo = archiveMetaInfoCache.get(jobId); + if (archiveMetaInfo == null) { + return true; + } + + switch (archiveMetaInfo.getEventType()) { + case PENDING: + case OVERVIEW_PARSING: + case OVERVIEW_CREATED: + return commonFetchTasks.containsKey(jobId) + && !individualFetchTasks.containsKey(jobId); + default: + return false; + } + } + + void cleanUpLazyFetchTask(String jobId) { + Future commonFetchTask = commonFetchTasks.get(jobId); + if (commonFetchTask != null) { + commonFetchTask.cancel(true); + commonFetchTasks.remove(jobId); + } + Future individualFetchTask = individualFetchTasks.get(jobId); + if (individualFetchTask != null) { + individualFetchTask.cancel(true); + individualFetchTasks.remove(jobId); + } + } + + @Override + public void close() { + ExecutorUtils.gracefulShutdown(1L, TimeUnit.SECONDS, commonFetchExecutor); + ExecutorUtils.gracefulShutdown(1L, TimeUnit.SECONDS, individualFetchExecutor); + } + + @VisibleForTesting + Future getCommonFetchTask(String jobId) { + return commonFetchTasks.get(jobId); + } + + void resetWhenTriggerLazyFetch(String jobId) { + archiveMetaInfoCache.remove(jobId); + cleanUpLazyFetchTask(jobId); + } + + void waitLazyFetchArchiveFinished(String jobId) throws Exception { + Future commonFetchTask = commonFetchTasks.get(jobId); + if (commonFetchTask != null) { + commonFetchTask.get(); + } + Future individualFetchTask = individualFetchTasks.get(jobId); + if (individualFetchTask != null) { + individualFetchTask.get(); + } + } + + ArchiveMetaInfo getArchiveMetaInfo(String jobId) { + return archiveMetaInfoCache.get(jobId); + } } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerRocksDBHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerRocksDBHandler.java index 147bc93eaf9408..f6b5a67a0ce48d 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerRocksDBHandler.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerRocksDBHandler.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.webmonitor.history; +import org.apache.flink.configuration.HistoryServerOptions; import org.apache.flink.runtime.rest.handler.util.HandlerUtils; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandler; @@ -40,9 +41,14 @@ @ChannelHandler.Sharable public class HistoryServerRocksDBHandler extends AbstractHistoryServerHandler { - public HistoryServerRocksDBHandler(RocksDBArchiveStorage rocksDBArchiveStorage, File rootPath) + public HistoryServerRocksDBHandler( + ArchiveStorage archiveStorage, + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode, + HistoryServerArchiveFetcher archiveFetcher, + HistoryServerApplicationArchiveFetcher applicationArchiveFetcher, + File rootPath) throws IOException { - super(rocksDBArchiveStorage, rootPath); + super(archiveStorage, archiveLoadMode, archiveFetcher, applicationArchiveFetcher, rootPath); } @Override diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerStaticFileServerHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerStaticFileServerHandler.java index 695f8edfffeb3b..28e0e0b7a7cbd5 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerStaticFileServerHandler.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerStaticFileServerHandler.java @@ -26,6 +26,8 @@ * https://github.com/netty/netty/blob/4.0/example/src/main/java/io/netty/example/http/file/HttpStaticFileServerHandler.java * *************************************************************************** */ +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.configuration.HistoryServerOptions; import org.apache.flink.runtime.rest.handler.legacy.files.StaticFileServerHandler; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandler; @@ -35,6 +37,8 @@ import java.io.File; import java.io.IOException; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.EAGER; + /** * Simple file server handler used by the {@link HistoryServer} that serves requests to web * frontend's static files, such as HTML, CSS, JS or JSON files. @@ -52,13 +56,20 @@ public class HistoryServerStaticFileServerHandler extends AbstractHistoryServerH // ------------------------------------------------------------------------ - public HistoryServerStaticFileServerHandler(File rootPath) throws IOException { - this(new FileArchiveStorage(rootPath), rootPath); + @VisibleForTesting + public HistoryServerStaticFileServerHandler(ArchiveStorage archiveStorage, File rootPath) + throws IOException { + this(archiveStorage, EAGER, null, null, rootPath); } public HistoryServerStaticFileServerHandler( - FileArchiveStorage fileArchiveStorage, File rootPath) throws IOException { - super(fileArchiveStorage, rootPath); + ArchiveStorage archiveStorage, + HistoryServerOptions.HistoryServerArchiveLoadMode archiveLoadMode, + HistoryServerArchiveFetcher archiveFetcher, + HistoryServerApplicationArchiveFetcher applicationArchiveFetcher, + File rootPath) + throws IOException { + super(archiveStorage, archiveLoadMode, archiveFetcher, applicationArchiveFetcher, rootPath); } // ------------------------------------------------------------------------ diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandlerTest.java index 479f3dcb07b539..cbff6ef4496980 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandlerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/AbstractHistoryServerHandlerTest.java @@ -18,8 +18,10 @@ package org.apache.flink.runtime.webmonitor.history; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode; import org.apache.flink.runtime.rest.handler.router.Router; import org.apache.flink.runtime.webmonitor.testutils.HttpUtils; import org.apache.flink.runtime.webmonitor.utils.WebFrontendBootstrap; @@ -35,12 +37,23 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.EAGER; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createJobArchive; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createRefreshLocation; import static org.assertj.core.api.Assertions.assertThat; /** @@ -50,10 +63,37 @@ @ExtendWith(ParameterizedTestExtension.class) public class AbstractHistoryServerHandlerTest { + @Parameter public HandlerFactory handlerFactory; + + @TempDir private Path tmpDir; + private Path uploadDir; + private Path remoteArchiveRootPath; + + private Path webDir; + private AbstractHistoryServerHandler handler; + private WebFrontendBootstrap webUI; + private String baseUrl; + /** Factory that creates a concrete handler bound to the given web directory. */ @FunctionalInterface public interface HandlerFactory { - AbstractHistoryServerHandler create(File webDir) throws Exception; + AbstractHistoryServerHandler create( + File webDir, + HistoryServerArchiveLoadMode archiveLoadMode, + List refreshDirs) + throws Exception; + } + + /** Constructor reference for a concrete {@link AbstractHistoryServerHandler} subclass. */ + @FunctionalInterface + private interface HandlerConstructor { + AbstractHistoryServerHandler create( + ArchiveStorage archiveStorage, + HistoryServerArchiveLoadMode archiveLoadMode, + HistoryServerArchiveFetcher archiveFetcher, + HistoryServerApplicationArchiveFetcher applicationArchiveFetcher, + File webDir) + throws IOException; } /** @@ -62,32 +102,98 @@ public interface HandlerFactory { */ @Parameters(name = "handlerFactory={0}") private static Collection handlerFactories() { - HandlerFactory staticFileServerHandlerFactory = HistoryServerStaticFileServerHandler::new; + HandlerFactory staticFileServerHandlerFactory = + (webDir, mode, refreshDirs) -> + buildHandler( + webDir, + mode, + refreshDirs, + new FileArchiveStorage(webDir), + HistoryServerStaticFileServerHandler::new); + HandlerFactory rocksDBHandlerFactory = - webDir -> - new HistoryServerRocksDBHandler( - new RocksDBArchiveStorage( - new File(webDir, "rocksdb-" + UUID.randomUUID()), - new Configuration()), - webDir); + (webDir, mode, refreshDirs) -> { + File dbPath = new File(webDir, "rocksdb-" + UUID.randomUUID()); + Files.createDirectories(dbPath.toPath()); + return buildHandler( + webDir, + mode, + refreshDirs, + new RocksDBArchiveStorage(dbPath, new Configuration()), + HistoryServerRocksDBHandler::new); + }; + return Arrays.asList(staticFileServerHandlerFactory, rocksDBHandlerFactory); } - @Parameter public HandlerFactory handlerFactory; + private static AbstractHistoryServerHandler buildHandler( + File webDir, + HistoryServerArchiveLoadMode mode, + List refreshDirs, + ArchiveStorage baseStorage, + HandlerConstructor handlerCtor) + throws Exception { + ArchiveStorage storage = + LAZY == mode + ? new HistoryServerTestUtils.BlockingArchiveStorage<>( + baseStorage, "/config") + : baseStorage; - @TempDir private Path tmpDir; + ConcurrentHashMap jobMetaInfoCache = new ConcurrentHashMap<>(); + ConcurrentHashMap applicationMetaInfoCache = + new ConcurrentHashMap<>(); - private Path webDir; - private AbstractHistoryServerHandler handler; - private WebFrontendBootstrap webUI; - private String baseUrl; + HistoryServerArchiveFetcher archiveFetcher = + new HistoryServerArchiveFetcher<>( + refreshDirs, + webDir, + ignored -> {}, + false, + HistoryServerTestUtils.RETAIN_ALL, + storage, + jobMetaInfoCache, + 4, + 4); + HistoryServerApplicationArchiveFetcher applicationArchiveFetcher = + new HistoryServerApplicationArchiveFetcher<>( + refreshDirs, + webDir, + ignored -> {}, + false, + HistoryServerTestUtils.RETAIN_ALL, + storage, + jobMetaInfoCache, + applicationMetaInfoCache, + 4, + 4); + + return handlerCtor.create(storage, mode, archiveFetcher, applicationArchiveFetcher, webDir); + } @BeforeEach void setUp() throws Exception { webDir = Files.createDirectory(tmpDir.resolve("webDir")); - final Path uploadDir = Files.createDirectory(tmpDir.resolve("uploadDir")); + uploadDir = Files.createDirectory(tmpDir.resolve("uploadDir")); + remoteArchiveRootPath = Files.createDirectories(tmpDir.resolve("remote")); + + // Default: eager mode + startServer(EAGER); + } + + @AfterEach + void tearDown() { + stopServer(); + } + + /** Support recreates the handler and web frontend for the given load mode. */ + private void startServer(HistoryServerArchiveLoadMode archiveLoadMode) throws Exception { + stopServer(); + + List refreshDirs = + Collections.singletonList(createRefreshLocation(remoteArchiveRootPath.toFile())); + + this.handler = handlerFactory.create(webDir.toFile(), archiveLoadMode, refreshDirs); - handler = handlerFactory.create(webDir.toFile()); Router router = new Router().addGet("/:*", handler); webUI = new WebFrontendBootstrap( @@ -101,10 +207,10 @@ void setUp() throws Exception { baseUrl = "http://localhost:" + webUI.getServerPort(); } - @AfterEach - void tearDown() { + private void stopServer() { if (webUI != null) { webUI.shutdown(); + webUI = null; } } @@ -184,4 +290,88 @@ void testRespondWithResource() throws Exception { assertThat(missing.f0).isEqualTo(404); assertThat(missing.f1).contains("not found"); } + + /** + * Tests {@code AbstractHistoryServerHandler#loadResource} in {@code LAZY} mode using a {@link + * HistoryServerTestUtils.BlockingArchiveStorage} to suspend writes for the detail key {@code + * jobs//config}. + * + *

Verifies the three core lazy-load behaviours: + * + *

    + *
  • requesting {@code /jobs/overview} triggers a synchronous phase-1 fetch that writes the + * overview keys but leaves the detail key blocked in an asynchronous task; + *
  • once phase-1 has completed, requesting {@code /jobs/} is served immediately from + * the archive storage without waiting for the asynchronous detail task; + *
  • requesting {@code /jobs//config} blocks until the asynchronous detail task is + * allowed to finish, after which the response is served successfully. + *
+ */ + @TestTemplate + void testLazyModeLoadResource() throws Exception { + // Default mode is EAGER, we need to recreate the handler for LAZY mode with + // BlockingArchiveStorage. + startServer(LAZY); + + JobID jobId = JobID.generate(); + createJobArchive(remoteArchiveRootPath.toFile(), jobId, true); + + // Phase 1: requesting /jobs/overview triggers a synchronous fetch + // that writes the overview keys; the detail key is queued for an + // asynchronous write that is now blocked on releaseLatch. + Tuple2 overviewResponse = + HttpUtils.getFromHTTP(baseUrl + "/jobs/overview"); + assertThat(overviewResponse.f0).isEqualTo(200); + assertThat(overviewResponse.f1).contains(jobId.toString()); + + // The asynchronous detail write must have been reached by now. + assertThat(handler.archiveStorage) + .isInstanceOf(HistoryServerTestUtils.BlockingArchiveStorage.class); + HistoryServerTestUtils.BlockingArchiveStorage blockingArchiveStorage = + (HistoryServerTestUtils.BlockingArchiveStorage) handler.archiveStorage; + boolean asyncReached = blockingArchiveStorage.asyncStartLatch.await(10, TimeUnit.SECONDS); + assertThat(asyncReached).isTrue(); + + // Phase 1 keys are present, but the detail key is still blocked. + assertThat(blockingArchiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(blockingArchiveStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(blockingArchiveStorage.exists("jobs/overview.json")).isTrue(); + assertThat(blockingArchiveStorage.exists("jobs/" + jobId + "/config.json")).isFalse(); + + // Phase 2: a request for /jobs/ can be served immediately + // from the storage even though the detail task is still blocked. + Tuple2 jobResponse = HttpUtils.getFromHTTP(baseUrl + "/jobs/" + jobId); + assertThat(jobResponse.f0).isEqualTo(200); + assertThat(jobResponse.f1).contains(jobId.toString()); + assertThat(blockingArchiveStorage.exists("jobs/" + jobId + "/config.json")).isFalse(); + + // Phase 3: a request for /jobs//config will wait for the + // asynchronous detail task to finish. Issue it on a separate + // thread, verify it is still in flight, then release the latch. + AtomicReference> detailResponse = new AtomicReference<>(); + CompletableFuture detailFuture = + CompletableFuture.runAsync( + () -> { + try { + detailResponse.set( + HttpUtils.getFromHTTP( + baseUrl + "/jobs/" + jobId + "/config")); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + // Give the detail request enough time to reach + // waitLazyFetchArchiveFinished and block on the still-suspended async + // detail task. It must not have completed yet. + Thread.sleep(500); + assertThat(detailFuture.isDone()).isFalse(); + + blockingArchiveStorage.releaseLatch.countDown(); + + detailFuture.get(10, TimeUnit.SECONDS); + assertThat(detailResponse.get().f0).isEqualTo(200); + assertThat(detailResponse.get().f1).contains(jobId.toString()); + assertThat(blockingArchiveStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + } } diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcherTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcherTest.java new file mode 100644 index 00000000000000..05f5e168389d9d --- /dev/null +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerApplicationArchiveFetcherTest.java @@ -0,0 +1,407 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.webmonitor.history; + +import org.apache.flink.api.common.ApplicationID; +import org.apache.flink.api.common.JobID; +import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.history.ArchivePathUtils; +import org.apache.flink.runtime.history.FsJsonArchivist; +import org.apache.flink.runtime.messages.webmonitor.ApplicationDetails; +import org.apache.flink.runtime.messages.webmonitor.MultipleApplicationsDetails; +import org.apache.flink.runtime.rest.messages.ApplicationsOverviewHeaders; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameter; +import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameters; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.apache.flink.configuration.ClusterOptions.CLUSTER_ID; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.EAGER; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.CREATED; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.DETAIL_PARSING; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.OVERVIEW_CREATED; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.OBJECT_MAPPER; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.RETAIN_ALL; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createJobArchive; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createRefreshLocation; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.waitForArchiveLoaded; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Tests for {@link HistoryServerApplicationArchiveFetcher}. Only application-specific behaviours + * that are NOT covered by {@link HistoryServerArchiveFetcherTest} are tested here. + */ +@ExtendWith(ParameterizedTestExtension.class) +class HistoryServerApplicationArchiveFetcherTest { + + private static final String DEFAULT_CLUSTER_ID = CLUSTER_ID.defaultValue(); + + @TempDir File remoteArchiveRootPath; + @TempDir File localArchiveRootPath; + + @Parameter public ArchiveStorageFactory storageFactory; + + private ArchiveStorage archiveStorage; + + private ConcurrentHashMap jobMetaInfoCache; + private ConcurrentHashMap applicationMetaInfoCache; + private List archiveEvents; + + @Parameters(name = "storageFactory={0}") + private static Collection> storageFactories() { + ArchiveStorageFactory fileArchiveStorageFactory = FileArchiveStorage::new; + ArchiveStorageFactory rocksDBStorageFactory = RocksDBArchiveStorage::new; + return List.of(fileArchiveStorageFactory, rocksDBStorageFactory); + } + + /** Creates an {@link ArchiveStorage} instance under the given temporary directory. */ + @FunctionalInterface + interface ArchiveStorageFactory { + ArchiveStorage create(File tempDir) throws Exception; + } + + @BeforeEach + void setUp() throws Exception { + jobMetaInfoCache = new ConcurrentHashMap<>(); + applicationMetaInfoCache = new ConcurrentHashMap<>(); + archiveEvents = new ArrayList<>(); + archiveStorage = storageFactory.create(localArchiveRootPath); + } + + @AfterEach + void tearDown() throws Exception { + archiveEvents.clear(); + if (archiveStorage != null) { + archiveStorage.close(); + archiveStorage = null; + } + } + + /** + * Create an application archive at {@code remoteArchiveRootPath//applications/ + * /} which contains an {@code application-summary} archive (representing the + * application overview) and one job archive per job in {@code jobs/}. + * + * 
{@code
+     * remoteArchiveRootPath/
+     * ├── /
+     * │   └── applications/
+     * │       ├── /
+     * │       │   ├── application-summary
+     * │       │   └── jobs/
+     * │       │       ├── 
+     * │       │       └── ...
+     * │       └── ...
+     * }
+ * + * @param applicationId application id (must be a hex string parseable by {@link + * ApplicationID#fromHexString}) + * @param jobIds job ids whose archives are to be placed under the {@code jobs/} subdir + * @return the application archive directory path + */ + private Path createApplicationArchive(String applicationId, List jobIds) + throws Exception { + File applicationDir = + new File( + remoteArchiveRootPath, + DEFAULT_CLUSTER_ID + + "/" + + ArchivePathUtils.APPLICATIONS_DIR + + "/" + + applicationId); + Files.createDirectories(applicationDir.toPath()); + + // Write application-summary archive (the application overview entry). + ApplicationID appId = ApplicationID.fromHexString(applicationId); + Map jobInfo = new HashMap<>(); + jobInfo.put("FINISHED", jobIds.size()); + ApplicationDetails applicationDetails = + new ApplicationDetails(appId, "test-app", 0L, 1L, 1L, "FINISHED", jobInfo); + String applicationOverviewJson = + OBJECT_MAPPER.writeValueAsString( + new MultipleApplicationsDetails(Collections.singleton(applicationDetails))); + ArchivedJson applicationOverviewArchive = + new ArchivedJson(ApplicationsOverviewHeaders.URL, applicationOverviewJson); + + // mock a simple /applications/.json + String applicationJson = "{\"id\":\"" + applicationId + "\"}"; + List archives = new ArrayList<>(); + archives.add(applicationOverviewArchive); + archives.add(new ArchivedJson("/applications/" + applicationId, applicationJson)); + + Path applicationSummaryPath = + new Path( + applicationDir.toURI().toString(), + ArchivePathUtils.APPLICATION_ARCHIVE_NAME); + FsJsonArchivist.writeArchivedJsons(applicationSummaryPath, archives); + + // Write job archives under jobs/ subdir. + File jobsDir = new File(applicationDir, ArchivePathUtils.JOBS_DIR); + Files.createDirectories(jobsDir.toPath()); + for (JobID jobId : jobIds) { + createJobArchive(jobsDir, jobId, true); + } + + return new Path(applicationDir.toURI().toString()); + } + + private HistoryServerApplicationArchiveFetcher createApplicationArchiveFetcher( + boolean cleanupExpired, ArchiveStorage storage) throws Exception { + List refreshDirs = + Collections.singletonList(createRefreshLocation(remoteArchiveRootPath)); + return new HistoryServerApplicationArchiveFetcher<>( + refreshDirs, + localArchiveRootPath, + event -> archiveEvents.add(event), + cleanupExpired, + RETAIN_ALL, + storage, + jobMetaInfoCache, + applicationMetaInfoCache, + 4, + 4); + } + + private static String newApplicationId() { + return new ApplicationID().toHexString(); + } + + // ========================================================================= + // EAGER MODE TESTS + // + // localArchiveRootPath/ + // ├── application-overviews/ + // │ └── application-id-1.json + // │ └── ... + // ├── applications/ + // │ └── overview.json + // │ └── application-id-1.json + // │ └── ... + // ├── overviews/ + // │ └── job-id-1.json + // │ └── ... + // ├── jobs/ + // │ └── overview.json + // │ └── job-id-1.json + // │ └── job-id-1/ + // │ ├── detail.json + // │ └── ... + // ========================================================================= + + @TestTemplate + void testEagerModeLoadsAllAppsAndJobsSync() throws Exception { + int numApps = 2; + int numJobsPerApp = 2; + List appIds = new ArrayList<>(); + Map> appToJobs = new HashMap<>(); + for (int i = 0; i < numApps; i++) { + String appId = newApplicationId(); + appIds.add(appId); + List jobIds = new ArrayList<>(); + for (int j = 0; j < numJobsPerApp; j++) { + jobIds.add(JobID.generate()); + } + appToJobs.put(appId, jobIds); + createApplicationArchive(appId, jobIds); + } + + HistoryServerApplicationArchiveFetcher fetcher = + createApplicationArchiveFetcher(false, archiveStorage); + fetcher.fetchArchives(EAGER); + + // N (apps) + N*M (jobs) events, all CREATED + assertThat(archiveEvents).hasSize(numApps + numApps * numJobsPerApp); + for (HistoryServerArchiveFetcher.ArchiveEvent event : archiveEvents) { + assertThat(event.getType()).isEqualTo(CREATED); + } + + for (String appId : appIds) { + assertThat(archiveStorage.exists("application-overviews/" + appId + ".json")).isTrue(); + assertThat(archiveStorage.exists("applications/" + appId + ".json")).isTrue(); + for (JobID jobId : appToJobs.get(appId)) { + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + } + } + assertThat(archiveStorage.exists("jobs/overview.json")).isTrue(); + assertThat(archiveStorage.exists("applications/overview.json")).isTrue(); + } + + // ========================================================================= + // LAZY MODE TESTS + // ========================================================================= + + @TestTemplate + void testLazyModeApplicationSyncJobDetailAsync() throws Exception { + String appId = newApplicationId(); + JobID jobId = JobID.generate(); + createApplicationArchive(appId, Collections.singletonList(jobId)); + + HistoryServerTestUtils.BlockingArchiveStorage blockingStorage = + new HistoryServerTestUtils.BlockingArchiveStorage<>( + archiveStorage, "jobs/" + jobId + "/config"); + // Replace the field so that close() in tearDown() releases the underlying storage too. + archiveStorage = blockingStorage; + + HistoryServerApplicationArchiveFetcher fetcher = + createApplicationArchiveFetcher(false, blockingStorage); + fetcher.fetchArchives(LAZY); + + // The application-level event is CREATED (application is loaded synchronously); + // the embedded job-level event is OVERVIEW_CREATED (detail is loaded asynchronously). + List eventTypes = + archiveEvents.stream() + .map(HistoryServerArchiveFetcher.ArchiveEvent::getType) + .collect(Collectors.toList()); + assertThat(eventTypes).containsExactlyInAnyOrder(CREATED, OVERVIEW_CREATED); + + // localArchiveRootPath/ + // ├── application-overviews/ + // │ └── application-id-1.json + // │ └── ... + // ├── applications/ + // │ └── overview.json + // │ └── application-id-1.json + // │ └── ... + // ├── overviews/ + // │ └── job-id-1.json + // │ └── ... + // ├── jobs/ + // │ └── overview.json + // │ └── job-id-1.json + // │ └── job-id-1/ + // │ ├── detail.json + // │ └── ... + + // Phase 1: detail not yet written + boolean asyncReached = blockingStorage.asyncStartLatch.await(10, TimeUnit.SECONDS); + assertThat(asyncReached).isTrue(); + assertThat(blockingStorage.exists("jobs/" + jobId + "/config.json")).isFalse(); + // application-summary content has been written synchronously + assertThat(blockingStorage.exists("application-overviews/" + appId + ".json")).isTrue(); + assertThat(blockingStorage.exists("applications/" + appId + ".json")).isTrue(); + assertThat(blockingStorage.exists("applications/overview.json")).isTrue(); + assertThat(blockingStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(blockingStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(blockingStorage.exists("jobs/overview.json")).isTrue(); + + // Phase 2: release async task and wait for completion + blockingStorage.releaseLatch.countDown(); + waitForArchiveLoaded(jobMetaInfoCache, jobId.toString()); + assertThat(blockingStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + assertThat(jobMetaInfoCache.get(jobId.toString()).getEventType()).isEqualTo(CREATED); + } + + @TestTemplate + void testMissingApplicationSummaryFileThrows() throws Exception { + String appId = newApplicationId(); + // Create the directory layout WITHOUT writing the application-summary file. + File applicationDir = + new File( + remoteArchiveRootPath, + DEFAULT_CLUSTER_ID + "/" + ArchivePathUtils.APPLICATIONS_DIR + "/" + appId); + Files.createDirectories(new File(applicationDir, ArchivePathUtils.JOBS_DIR).toPath()); + + HistoryServerApplicationArchiveFetcher fetcher = + createApplicationArchiveFetcher(false, archiveStorage); + Path refreshPath = new Path(remoteArchiveRootPath.toURI().toString()); + Path applicationPath = new Path(applicationDir.toURI().toString()); + + assertThatThrownBy(() -> fetcher.processArchive(appId, applicationPath, refreshPath)) + .hasMessageContaining( + "Application archive " + + new Path( + applicationPath, ArchivePathUtils.APPLICATION_ARCHIVE_NAME) + + " does not exist."); + + // Cache should NOT contain a successful entry for this application. + assertThat(applicationMetaInfoCache.containsKey(appId)).isFalse(); + } + + // ========================================================================= + // ApplicationArchiveMetaInfoCache STATE TRANSITION TESTS + // ========================================================================= + + @TestTemplate + void testApplicationArchiveStateTransition() throws Exception { + String appId = newApplicationId(); + JobID jobId = JobID.generate(); + Path applicationArchivePath = + createApplicationArchive(appId, Collections.singletonList(jobId)); + + HistoryServerTestUtils.BlockingArchiveStorage blockingStorage = + new HistoryServerTestUtils.BlockingArchiveStorage<>( + archiveStorage, "jobs/" + jobId + "/config"); + archiveStorage = blockingStorage; + + HistoryServerApplicationArchiveFetcher fetcher = + createApplicationArchiveFetcher(false, blockingStorage); + Path refreshPath = new Path(remoteArchiveRootPath.toURI().toString()); + + fetcher.fetchArchives(LAZY); + + // Phase 1: application is loaded synchronously -> CREATED; + assertThat(applicationMetaInfoCache.get(appId).getEventType()).isEqualTo(CREATED); + + // the embedded job's detail loading is blocked at the storage -> DETAIL_PARSING. + boolean asyncReached = blockingStorage.asyncStartLatch.await(10, TimeUnit.SECONDS); + assertThat(asyncReached).isTrue(); + assertThat(jobMetaInfoCache.get(jobId.toString()).getEventType()).isEqualTo(DETAIL_PARSING); + + // try to call lazyProcessJobArchive again, should return CREATED. + List callAgainEvent = + fetcher.lazyProcessArchive(appId, applicationArchivePath, refreshPath); + assertThat(callAgainEvent.get(0).getType()).isEqualTo(CREATED); + assertThat(applicationMetaInfoCache.get(appId).getEventType()).isEqualTo(CREATED); + + // Phase 2: execute the asynchronous task + blockingStorage.releaseLatch.countDown(); + waitForArchiveLoaded(jobMetaInfoCache, jobId.toString()); + assertThat(jobMetaInfoCache.get(jobId.toString()).getEventType()).isEqualTo(CREATED); + + // Phase 3: cleanup all archives + Map> archivesToRemove = new HashMap<>(); + archivesToRemove.put(refreshPath, Collections.singleton(appId)); + fetcher.cleanupExpiredArchives(archivesToRemove); + + assertThat(applicationMetaInfoCache.containsKey(appId)).isFalse(); + assertThat(jobMetaInfoCache.containsKey(jobId.toString())).isFalse(); + } +} diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcherTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcherTest.java new file mode 100644 index 00000000000000..612d156f244206 --- /dev/null +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcherTest.java @@ -0,0 +1,429 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.webmonitor.history; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.history.FsJsonArchivist; +import org.apache.flink.runtime.messages.webmonitor.JobDetails; +import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameter; +import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameters; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.EAGER; +import static org.apache.flink.configuration.HistoryServerOptions.HistoryServerArchiveLoadMode.LAZY; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.CREATED; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.DETAIL_PARSING; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.OVERVIEW_CREATED; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.OBJECT_MAPPER; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.RETAIN_ALL; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createJobArchive; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createLegacyArchive; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createRefreshLocation; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.waitForArchiveLoaded; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link HistoryServerArchiveFetcher}. */ +@ExtendWith(ParameterizedTestExtension.class) +class HistoryServerArchiveFetcherTest { + + @TempDir File remoteArchiveRootPath; + @TempDir File localArchiveRootPath; + + @Parameter public ArchiveStorageFactory storageFactory; + + private ArchiveStorage archiveStorage; + private ConcurrentHashMap archiveMetaInfoCache; + private List archiveEvents; + + @Parameters(name = "storageFactory={0}") + private static Collection> storageFactories() { + ArchiveStorageFactory fileArchiveStorageFactory = FileArchiveStorage::new; + ArchiveStorageFactory rocksDBStorageFactory = RocksDBArchiveStorage::new; + return List.of(fileArchiveStorageFactory, rocksDBStorageFactory); + } + + /** Creates an {@link ArchiveStorage} instance under the given temporary directory. */ + @FunctionalInterface + interface ArchiveStorageFactory { + ArchiveStorage create(File tempDir) throws Exception; + } + + @BeforeEach + void setUp() throws Exception { + archiveMetaInfoCache = new ConcurrentHashMap<>(); + archiveEvents = new ArrayList<>(); + archiveStorage = storageFactory.create(localArchiveRootPath); + } + + @AfterEach + void tearDown() throws Exception { + archiveEvents.clear(); + if (archiveStorage != null) { + archiveStorage.close(); + archiveStorage = null; + } + } + + /** + * Create {@link HistoryServerArchiveFetcher} instance with a custom {@link ArchiveStorage}. + * + * @param refreshDir archive scan directory + * @param cleanupExpiredJobs whether to enable expired archive cleanup + * @param storage the archive storage to use + */ + private HistoryServerArchiveFetcher createArchiveFetcher( + File refreshDir, boolean cleanupExpiredJobs, ArchiveStorage storage) + throws Exception { + List refreshDirs = + Collections.singletonList(createRefreshLocation(refreshDir)); + return new HistoryServerArchiveFetcher<>( + refreshDirs, + localArchiveRootPath, + event -> archiveEvents.add(event), + cleanupExpiredJobs, + RETAIN_ALL, + storage, + archiveMetaInfoCache, + 4, + 4); + } + + // ========================================================================= + // EAGER MODE TESTS + // ========================================================================= + + @TestTemplate + void testEagerModeLoadsAllFilesSync() throws Exception { + int numJobs = 3; + List jobIds = new ArrayList<>(); + for (int i = 0; i < numJobs; i++) { + JobID jobId = JobID.generate(); + jobIds.add(jobId); + createJobArchive(remoteArchiveRootPath, jobId, true); + } + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, archiveStorage); + fetcher.fetchArchives(EAGER); + + assertThat(archiveEvents).hasSize(numJobs); + for (HistoryServerArchiveFetcher.ArchiveEvent event : archiveEvents) { + assertThat(event.getType()).isEqualTo(CREATED); + } + for (JobID jobId : jobIds) { + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + } + assertThat(archiveStorage.exists("jobs/overview.json")).isTrue(); + } + + // ========================================================================= + // LAZY MODE TESTS + // ========================================================================= + + @TestTemplate + void testLazyModeDetailNotWrittenBeforeAsyncTaskCompleted() throws Exception { + JobID jobId = JobID.generate(); + createJobArchive(remoteArchiveRootPath, jobId, true); + + HistoryServerTestUtils.BlockingArchiveStorage blockingStorage = + new HistoryServerTestUtils.BlockingArchiveStorage<>( + archiveStorage, "jobs/" + jobId + "/config"); + // Replace the field so that close() in tearDown() releases the underlying storage too. + archiveStorage = blockingStorage; + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, blockingStorage); + + fetcher.fetchArchives(LAZY); + + // Phase 1: overview archive is loaded synchronously, but detail archive haven't loaded yet. + boolean asyncReached = blockingStorage.asyncStartLatch.await(10, TimeUnit.SECONDS); + assertThat(asyncReached).isTrue(); + assertThat(archiveEvents.get(0).getType()).isEqualTo(OVERVIEW_CREATED); + assertThat(blockingStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(blockingStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(blockingStorage.exists("jobs/overview.json")).isTrue(); + assertThat(blockingStorage.exists("jobs/" + jobId + "/config.json")).isFalse(); + + // Phase 2: execute the asynchronous task + blockingStorage.releaseLatch.countDown(); + waitForArchiveLoaded(archiveMetaInfoCache, jobId.toString()); + + assertThat(archiveMetaInfoCache.get(jobId.toString()).getEventType()).isEqualTo(CREATED); + assertThat(blockingStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + } + + @TestTemplate + void testLazyModeLoadsAllFilesCompleted() throws Exception { + int numJobs = 5; + List jobIds = new ArrayList<>(); + for (int i = 0; i < numJobs; i++) { + JobID jobId = JobID.generate(); + jobIds.add(jobId); + createJobArchive(remoteArchiveRootPath, jobId, true); + } + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, archiveStorage); + fetcher.fetchArchives(LAZY); + + assertThat(archiveEvents).hasSize(numJobs); + for (HistoryServerArchiveFetcher.ArchiveEvent event : archiveEvents) { + assertThat(event.getType()).isEqualTo(OVERVIEW_CREATED); + } + + for (JobID jobId : jobIds) { + waitForArchiveLoaded(archiveMetaInfoCache, jobId.toString()); + } + + // when async task completed, all detail archives should be loaded. + for (JobID jobId : jobIds) { + assertThat(archiveMetaInfoCache.get(jobId.toString()).getEventType()) + .isEqualTo(CREATED); + assertThat(archiveStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + } + } + + @TestTemplate + void testLazyProcessJobArchiveThrowsForEmptyArchive() throws Exception { + JobID jobId = JobID.generate(); + Path archivePath = new Path(remoteArchiveRootPath.toURI().toString(), jobId.toString()); + FsJsonArchivist.writeArchivedJsons(archivePath, Collections.emptyList()); + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, archiveStorage); + + assertThatThrownBy( + () -> fetcher.lazyProcessJobArchive(jobId.toString(), archivePath, false)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Archive of job " + jobId + " is empty"); + + assertThat(archiveMetaInfoCache.containsKey(jobId.toString())).isFalse(); + } + + // ========================================================================= + // ArchiveMetaInfoCache STATE TRANSITION TESTS + // ========================================================================= + + /** STATE: PENDING -> OVERVIEW_PARSING -> OVERVIEW_CREATED -> DETAIL_PARSING -> CREATED. */ + @TestTemplate + void testArchiveStateTransition() throws Exception { + JobID jobIdWithDetail = JobID.generate(); + Path jobWithDetailArchivePath = + createJobArchive(remoteArchiveRootPath, jobIdWithDetail, true); + JobID jobIdWithoutDetail = JobID.generate(); + createJobArchive(remoteArchiveRootPath, jobIdWithoutDetail, false); + + HistoryServerTestUtils.BlockingArchiveStorage blockingStorage = + new HistoryServerTestUtils.BlockingArchiveStorage<>( + archiveStorage, "jobs/" + jobIdWithDetail + "/config"); + archiveStorage = blockingStorage; + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, blockingStorage); + + fetcher.fetchArchives(LAZY); + + assertThat(archiveMetaInfoCache.size()).isEqualTo(2); + assertThat(archiveMetaInfoCache.containsKey(jobIdWithoutDetail.toString())).isTrue(); + assertThat(archiveMetaInfoCache.get(jobIdWithoutDetail.toString()).getEventType()) + .isEqualTo(OVERVIEW_CREATED); + + assertThat(archiveMetaInfoCache.containsKey(jobIdWithDetail.toString())).isTrue(); + // Phase 1: overview archive is loaded synchronously, but detail archive haven't loaded yet. + boolean asyncReached = blockingStorage.asyncStartLatch.await(10, TimeUnit.SECONDS); + assertThat(asyncReached).isTrue(); + assertThat(archiveMetaInfoCache.get(jobIdWithDetail.toString()).getEventType()) + .isEqualTo(DETAIL_PARSING); + + // try to call lazyProcessJobArchive again, should return DETAIL_PARSING. + HistoryServerArchiveFetcher.ArchiveEvent callAgainEvent = + fetcher.lazyProcessJobArchive( + jobIdWithDetail.toString(), jobWithDetailArchivePath, false); + assertThat(callAgainEvent.getType()).isEqualTo(DETAIL_PARSING); + + // Phase 2: execute the asynchronous task + blockingStorage.releaseLatch.countDown(); + waitForArchiveLoaded(archiveMetaInfoCache, jobIdWithDetail.toString()); + assertThat(archiveMetaInfoCache.get(jobIdWithDetail.toString()).getEventType()) + .isEqualTo(CREATED); + + // Phase 3: remove all archives + Map> archivesToRemove = new HashMap<>(); + Set archiveIdsToRemove = new HashSet<>(); + archiveIdsToRemove.add(jobIdWithDetail.toString()); + archiveIdsToRemove.add(jobIdWithoutDetail.toString()); + archivesToRemove.put( + new Path(remoteArchiveRootPath.toURI().toString()), archiveIdsToRemove); + fetcher.cleanupExpiredArchives(archivesToRemove); + assertThat(archiveMetaInfoCache.size()).isEqualTo(0); + } + + // ========================================================================= + // Other Tests + // ========================================================================= + + /** + * This test reads {@code jobs/overview.json} directly from the local file system, so it only + * applies to the {@link FileArchiveStorage} backend. + */ + @TestTemplate + void testUpdateJobOverview() throws Exception { + JobID job1 = JobID.generate(); + JobID job2 = JobID.generate(); + createJobArchive(remoteArchiveRootPath, job1, true); + createJobArchive(remoteArchiveRootPath, job2, true); + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, true, archiveStorage); + fetcher.fetchArchives(EAGER); + + Object overviewObject = archiveStorage.getEntry("jobs/overview.json"); + String overviewContent = archiveStorage.readArchiveContent(overviewObject); + MultipleJobsDetails overview = + OBJECT_MAPPER.readValue(overviewContent, MultipleJobsDetails.class); + + assertThat(overview.getJobs()).hasSize(2); + Set jobIds = new HashSet<>(); + for (JobDetails jobDetails : overview.getJobs()) { + jobIds.add(jobDetails.getJobId().toString()); + } + assertThat(jobIds).containsExactlyInAnyOrder(job1.toString(), job2.toString()); + + // remove job1 + new File(remoteArchiveRootPath, job1.toString()).delete(); + fetcher.fetchArchives(EAGER); + + // verify overview now only contains job2 + overviewObject = archiveStorage.getEntry("jobs/overview.json"); + overviewContent = archiveStorage.readArchiveContent(overviewObject); + overview = OBJECT_MAPPER.readValue(overviewContent, MultipleJobsDetails.class); + assertThat(overview.getJobs()).hasSize(1); + assertThat(overview.getJobs().iterator().next().getJobId()).isEqualTo(job2); + } + + @TestTemplate + void testLegacyJobOverviewMigration() throws Exception { + JobID jobId = createLegacyArchive(remoteArchiveRootPath.toPath(), false); + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, archiveStorage); + fetcher.fetchArchives(LAZY); + + assertThat(archiveEvents).hasSize(1); + assertThat(archiveEvents.get(0).getType()).isEqualTo(OVERVIEW_CREATED); + + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/overview.json")).isTrue(); + } + + @TestTemplate + void testScanArchivesWithoutFetch() throws Exception { + JobID jobId = JobID.generate(); + createJobArchive(remoteArchiveRootPath, jobId, true); + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, true, archiveStorage); + + fetcher.scanArchives(EAGER, false); + assertThat(archiveEvents).isEmpty(); + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isFalse(); + } + + @TestTemplate + void testLazyFetchArchiveProactively() throws Exception { + // with explicit path + JobID jobId = JobID.generate(); + Path archivePath = createJobArchive(remoteArchiveRootPath, jobId, true); + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, archiveStorage); + + fetcher.lazyFetchArchiveProactively(jobId.toString(), archivePath); + waitForArchiveLoaded(archiveMetaInfoCache, jobId.toString()); + + assertThat(archiveMetaInfoCache.get(jobId.toString()).getEventType()).isEqualTo(CREATED); + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + + // without explicit path + jobId = JobID.generate(); + createJobArchive(remoteArchiveRootPath, jobId, true); + + // call without explicit path; the fetcher should locate the archive in refreshDirs. + fetcher.lazyFetchArchiveProactively(jobId.toString(), null); + waitForArchiveLoaded(archiveMetaInfoCache, jobId.toString()); + + assertThat(archiveMetaInfoCache.get(jobId.toString()).getEventType()).isEqualTo(CREATED); + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + "/config.json")).isTrue(); + } + + @TestTemplate + void testCleanUpLazyFetchTaskCancelsRunningFuture() throws Exception { + JobID jobId = JobID.generate(); + createJobArchive(remoteArchiveRootPath, jobId, true); + + HistoryServerTestUtils.BlockingArchiveStorage blockingStorage = + new HistoryServerTestUtils.BlockingArchiveStorage<>( + archiveStorage, "jobs/" + jobId + "/config"); + archiveStorage = blockingStorage; + + HistoryServerArchiveFetcher fetcher = + createArchiveFetcher(remoteArchiveRootPath, false, blockingStorage); + fetcher.fetchArchives(LAZY); + // make sure the async detail task has started + assertThat(blockingStorage.asyncStartLatch.await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(fetcher.getCommonFetchTask(jobId.toString())).isNotNull(); + assertThat(fetcher.getCommonFetchTask(jobId.toString())).isNotDone(); + + // cancel the in-flight task + fetcher.cleanUpLazyFetchTask(jobId.toString()); + + assertThat(fetcher.getCommonFetchTask(jobId.toString())).isNull(); + assertThat(archiveStorage.exists("overviews/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + ".json")).isTrue(); + assertThat(archiveStorage.exists("jobs/" + jobId + "/config.json")).isFalse(); + } +} diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java index 4c7cd990d23496..26fcceedf450a6 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java @@ -52,8 +52,6 @@ import org.apache.flink.util.FlinkException; import org.apache.flink.util.jackson.JacksonMapperFactory; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -68,7 +66,6 @@ import java.io.File; import java.io.IOException; -import java.io.StringWriter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -87,16 +84,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createLegacyArchive; +import static org.apache.flink.runtime.webmonitor.history.HistoryServerTestUtils.createLegacyArchiveWithModifiedDate; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for the HistoryServer. */ class HistoryServerTest { - private static final JsonFactory JACKSON_FACTORY = - new JsonFactory() - .enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) - .disable(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT); private static final ObjectMapper OBJECT_MAPPER = JacksonMapperFactory.createObjectMapper() .enable(DeserializationFeature.FAIL_ON_MISSING_CREATOR_PROPERTIES); @@ -198,7 +193,7 @@ void testRemoveOldestModifiedArchivesBeyondHistorySizeLimit(final boolean versio for (int j = 0; j < numArchivesBeforeHsStarted; j++) { JobID jobId = - createLegacyArchive( + createLegacyArchiveWithModifiedDate( jmDirectory.toPath(), j * oneMinuteSinceEpoch, versionLessThan14); if (j >= numArchivesToRemoveUponHsStart) { expectedJobIdsToKeep.add(jobId); @@ -253,7 +248,7 @@ void testRemoveOldestModifiedArchivesBeyondHistorySizeLimit(final boolean versio j++) { expectedJobIdsToKeep.remove(0); expectedJobIdsToKeep.add( - createLegacyArchive( + createLegacyArchiveWithModifiedDate( jmDirectory.toPath(), j * oneMinuteSinceEpoch, versionLessThan14)); } assertThat(numArchivesCreatedTotal.await(10L, TimeUnit.SECONDS)).isTrue(); @@ -534,62 +529,6 @@ private static void runJob() throws Exception { env.execute(); } - private static JobID createLegacyArchive( - Path directory, long fileModifiedDate, boolean versionLessThan14) throws IOException { - JobID jobId = createLegacyArchive(directory, versionLessThan14); - File jobArchive = directory.resolve(jobId.toString()).toFile(); - jobArchive.setLastModified(fileModifiedDate); - return jobId; - } - - private static JobID createLegacyArchive(Path directory, boolean versionLessThan14) - throws IOException { - JobID jobId = JobID.generate(); - - StringWriter sw = new StringWriter(); - try (JsonGenerator gen = JACKSON_FACTORY.createGenerator(sw)) { - try (JsonObject root = new JsonObject(gen)) { - try (JsonArray finished = new JsonArray(gen, "finished")) { - try (JsonObject job = new JsonObject(gen)) { - gen.writeStringField("jid", jobId.toString()); - gen.writeStringField("name", "testjob"); - gen.writeStringField("state", JobStatus.FINISHED.name()); - - gen.writeNumberField("start-time", 0L); - gen.writeNumberField("end-time", 1L); - gen.writeNumberField("duration", 1L); - gen.writeNumberField("last-modification", 1L); - - try (JsonObject tasks = new JsonObject(gen, "tasks")) { - gen.writeNumberField("total", 0); - - if (versionLessThan14) { - gen.writeNumberField("pending", 0); - } else { - gen.writeNumberField("created", 0); - gen.writeNumberField("deploying", 0); - gen.writeNumberField("scheduled", 0); - } - gen.writeNumberField("running", 0); - gen.writeNumberField("finished", 0); - gen.writeNumberField("canceling", 0); - gen.writeNumberField("canceled", 0); - gen.writeNumberField("failed", 0); - } - } - } - } - } - String json = sw.toString(); - ArchivedJson archivedJson = new ArchivedJson("/joboverview", json); - FsJsonArchivist.writeArchivedJsons( - new org.apache.flink.core.fs.Path( - directory.toAbsolutePath().toString(), jobId.toString()), - Collections.singleton(archivedJson)); - - return jobId; - } - @Test void testApplicationAndJobArchives() throws Exception { int numApplications = 2; @@ -991,39 +930,4 @@ private void deleteApplicationArchiveDir(ApplicationID applicationId) throws IOE .getParent(); applicationArchiveDir.getFileSystem().delete(applicationArchiveDir, true); } - - private static final class JsonObject implements AutoCloseable { - - private final JsonGenerator gen; - - JsonObject(JsonGenerator gen) throws IOException { - this.gen = gen; - gen.writeStartObject(); - } - - private JsonObject(JsonGenerator gen, String name) throws IOException { - this.gen = gen; - gen.writeObjectFieldStart(name); - } - - @Override - public void close() throws IOException { - gen.writeEndObject(); - } - } - - private static final class JsonArray implements AutoCloseable { - - private final JsonGenerator gen; - - JsonArray(JsonGenerator gen, String name) throws IOException { - this.gen = gen; - gen.writeArrayFieldStart(name); - } - - @Override - public void close() throws IOException { - gen.writeEndArray(); - } - } } diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTestUtils.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTestUtils.java new file mode 100644 index 00000000000000..de42e501a711cb --- /dev/null +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTestUtils.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.webmonitor.history; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.Path; +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.history.FsJsonArchivist; +import org.apache.flink.runtime.messages.webmonitor.JobDetails; +import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails; +import org.apache.flink.runtime.rest.messages.JobsOverviewHeaders; +import org.apache.flink.runtime.webmonitor.history.retaining.ArchiveRetainedStrategy; +import org.apache.flink.util.jackson.JacksonMapperFactory; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import javax.annotation.Nullable; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.apache.flink.runtime.webmonitor.history.HistoryServerArchiveFetcher.ArchiveEventType.CREATED; + +/** Common utilities for history server tests. */ +public class HistoryServerTestUtils { + + private static final JsonFactory JACKSON_FACTORY = + new JsonFactory() + .enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) + .disable(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT); + + public static final ObjectMapper OBJECT_MAPPER = JacksonMapperFactory.createObjectMapper(); + public static final ArchiveRetainedStrategy RETAIN_ALL = (file, index) -> true; + + static HistoryServer.RefreshLocation createRefreshLocation(File dir) throws Exception { + Path path = new Path(dir.toURI().toString()); + FileSystem fs = path.getFileSystem(); + return new HistoryServer.RefreshLocation(path, fs); + } + + /** + * Create a job archive at {@code baseDir/}. + * + * @param baseDir base directory under which the archive will be written + * @param jobId job id used as the archive file name + * @param withDetail whether to include the {@code /jobs//config} sub-archive + * @return the archive {@link Path} + */ + static Path createJobArchive(File baseDir, JobID jobId, boolean withDetail) throws Exception { + MultipleJobsDetails overview = + new MultipleJobsDetails( + Collections.singleton( + new JobDetails( + jobId, + "test-job", + 0L, + 1L, + 1L, + JobStatus.FINISHED, + 1L, + new int[ExecutionState.values().length], + 0))); + String overviewJson = OBJECT_MAPPER.writeValueAsString(overview); + ArchivedJson overviewArchive = new ArchivedJson(JobsOverviewHeaders.URL, overviewJson); + + // mock a simple /jobs/.json + String jobJson = "{\"jid\":\"" + jobId + "\"}"; + + List archives = new ArrayList<>(); + archives.add(overviewArchive); + archives.add(new ArchivedJson("/jobs/" + jobId, jobJson)); + + if (withDetail) { + String detailJson = "{\"jid\":\"" + jobId + "\",\"name\":\"test-job\"}"; + archives.add(new ArchivedJson("/jobs/" + jobId + "/config", detailJson)); + } + + Path archivePath = new Path(baseDir.toURI().toString(), jobId.toString()); + FsJsonArchivist.writeArchivedJsons(archivePath, archives); + return archivePath; + } + + /** Create a legacy job archive at {@code directory/} with only {@code /joboverview}. */ + public static JobID createLegacyArchive(java.nio.file.Path directory, boolean versionLessThan14) + throws IOException { + JobID jobId = JobID.generate(); + + StringWriter sw = new StringWriter(); + try (JsonGenerator gen = JACKSON_FACTORY.createGenerator(sw)) { + try (JsonObject root = new JsonObject(gen)) { + try (JsonArray finished = new JsonArray(gen, "finished")) { + try (JsonObject job = new JsonObject(gen)) { + gen.writeStringField("jid", jobId.toString()); + gen.writeStringField("name", "testjob"); + gen.writeStringField("state", JobStatus.FINISHED.name()); + + gen.writeNumberField("start-time", 0L); + gen.writeNumberField("end-time", 1L); + gen.writeNumberField("duration", 1L); + gen.writeNumberField("last-modification", 1L); + + try (JsonObject tasks = new JsonObject(gen, "tasks")) { + gen.writeNumberField("total", 0); + + if (versionLessThan14) { + gen.writeNumberField("pending", 0); + } else { + gen.writeNumberField("created", 0); + gen.writeNumberField("deploying", 0); + gen.writeNumberField("scheduled", 0); + } + gen.writeNumberField("running", 0); + gen.writeNumberField("finished", 0); + gen.writeNumberField("canceling", 0); + gen.writeNumberField("canceled", 0); + gen.writeNumberField("failed", 0); + } + } + } + } + } + String json = sw.toString(); + ArchivedJson archivedJson = new ArchivedJson("/joboverview", json); + FsJsonArchivist.writeArchivedJsons( + new Path(directory.toAbsolutePath().toString(), jobId.toString()), + Collections.singleton(archivedJson)); + + return jobId; + } + + public static JobID createLegacyArchiveWithModifiedDate( + java.nio.file.Path directory, long fileModifiedDate, boolean versionLessThan14) + throws IOException { + JobID jobId = createLegacyArchive(directory, versionLessThan14); + File jobArchive = directory.resolve(jobId.toString()).toFile(); + jobArchive.setLastModified(fileModifiedDate); + return jobId; + } + + /** + * Wait until {@code cache.get(archiveId).eventType == CREATED}. Throws {@link AssertionError} + * if the deadline is exceeded. + */ + static void waitForArchiveLoaded(Map cache, String archiveId) + throws InterruptedException { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline) { + ArchiveMetaInfo metaInfo = cache.get(archiveId); + if (metaInfo != null && metaInfo.getEventType() == CREATED) { + return; + } + Thread.sleep(50); + } + throw new AssertionError( + "Timed out waiting for archive " + archiveId + " to reach CREATED state"); + } + + /** + * Blocking archive storage for testing the LAZY load mode. Calls to {@link + * #putArchiveContent(String, String)} whose key contains {@code blockOnKeyPrefix} will block + * until {@link #releaseLatch} is counted down. + * + *

This is a decorator that delegates to a wrapped {@link ArchiveStorage}, so it can be used + * with any storage backend (e.g. {@link FileArchiveStorage} or {@link RocksDBArchiveStorage}). + */ + public static class BlockingArchiveStorage implements ArchiveStorage { + + public final CountDownLatch asyncStartLatch = new CountDownLatch(1); + public final CountDownLatch releaseLatch = new CountDownLatch(1); + + private final ArchiveStorage delegate; + private final String blockOnKeyPrefix; + + public BlockingArchiveStorage(ArchiveStorage delegate, String blockOnKeyPrefix) { + this.delegate = delegate; + this.blockOnKeyPrefix = blockOnKeyPrefix; + } + + @Override + public boolean exists(String key) throws IOException { + return delegate.exists(key); + } + + @Nullable + @Override + public Entry getEntry(String key) throws IOException { + return delegate.getEntry(key); + } + + @Override + public void putArchiveContent(String key, String archiveContent) throws IOException { + if (key.contains(blockOnKeyPrefix)) { + asyncStartLatch.countDown(); + try { + boolean released = releaseLatch.await(10, TimeUnit.SECONDS); + if (!released) { + throw new IOException( + "BlockingArchiveStorage: timed out waiting for release latch"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("BlockingArchiveStorage: interrupted", e); + } + } + delegate.putArchiveContent(key, archiveContent); + } + + @Override + public void delete(String key) throws IOException { + delegate.delete(key); + } + + @Override + public void deleteEntriesByPrefix(String keyPrefix) throws IOException { + delegate.deleteEntriesByPrefix(keyPrefix); + } + + @Override + public List getEntriesByPrefix(String prefix) throws IOException { + return delegate.getEntriesByPrefix(prefix); + } + + @Override + public String readArchiveContent(Entry entry) throws IOException { + return delegate.readArchiveContent(entry); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + } + + private static final class JsonObject implements AutoCloseable { + + private final JsonGenerator gen; + + JsonObject(JsonGenerator gen) throws IOException { + this.gen = gen; + gen.writeStartObject(); + } + + private JsonObject(JsonGenerator gen, String name) throws IOException { + this.gen = gen; + gen.writeObjectFieldStart(name); + } + + @Override + public void close() throws IOException { + gen.writeEndObject(); + } + } + + private static final class JsonArray implements AutoCloseable { + + private final JsonGenerator gen; + + JsonArray(JsonGenerator gen, String name) throws IOException { + this.gen = gen; + gen.writeArrayFieldStart(name); + } + + @Override + public void close() throws IOException { + gen.writeEndArray(); + } + } +} diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java index 968dcf73a7d17f..6a4bc3f97150b1 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/utils/WebFrontendBootstrapTest.java @@ -24,6 +24,7 @@ import org.apache.flink.runtime.io.network.netty.Prio0InboundChannelHandlerFactory; import org.apache.flink.runtime.io.network.netty.Prio1InboundChannelHandlerFactory; import org.apache.flink.runtime.rest.handler.router.Router; +import org.apache.flink.runtime.webmonitor.history.FileArchiveStorage; import org.apache.flink.runtime.webmonitor.history.HistoryServerStaticFileServerHandler; import org.apache.flink.runtime.webmonitor.testutils.HttpUtils; import org.apache.flink.testutils.junit.extensions.ContextClassLoaderExtension; @@ -34,6 +35,7 @@ import org.junit.jupiter.api.io.TempDir; import org.slf4j.LoggerFactory; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -57,12 +59,16 @@ class WebFrontendBootstrapTest { @Test void testHandlersMustBeLoaded() throws Exception { Path webDir = Files.createDirectories(tmp.resolve("webDir")); + File webDirFile = webDir.toFile(); Configuration configuration = new Configuration(); configuration.set(Prio0InboundChannelHandlerFactory.REDIRECT_FROM_URL, "/nonExisting"); configuration.set(Prio0InboundChannelHandlerFactory.REDIRECT_TO_URL, "/index.html"); Router router = new Router<>() - .addGet("/:*", new HistoryServerStaticFileServerHandler(webDir.toFile())); + .addGet( + "/:*", + new HistoryServerStaticFileServerHandler( + new FileArchiveStorage(webDirFile), webDirFile)); WebFrontendBootstrap webUI = new WebFrontendBootstrap( router, diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java index 58ad00e77273bb..d091128bdb560e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java @@ -71,6 +71,7 @@ import javax.annotation.Nullable; import java.io.Serializable; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.Locale; @@ -395,9 +396,23 @@ public void open() throws Exception { && areInterruptibleTimersConfigured() && getTimeServiceManager().isPresent()) { LOG.info("Interruptible timers enabled for {}", getClass().getSimpleName()); + InternalTimeServiceManager timeServiceManager = getTimeServiceManager().get(); + boolean emitIntermediateWatermarks = + getContainingTask() + .getJobConfiguration() + .get( + CheckpointingOptions + .UNALIGNED_INTERRUPTIBLE_TIMERS_EMIT_INTERMEDIATE_WATERMARKS); + if (emitIntermediateWatermarks) { + timeServiceManager.configureIntermediateWatermarkInterval( + Duration.ofMillis(getExecutionConfig().getAutoWatermarkInterval())); + } this.watermarkProcessor = new MailboxWatermarkProcessor( - output, mailboxExecutor, getTimeServiceManager().get()); + output, + mailboxExecutor, + timeServiceManager, + emitIntermediateWatermarks); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java index b898f439c1bcff..8dfd584351a01e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorV2.java @@ -68,6 +68,7 @@ import javax.annotation.Nullable; +import java.time.Duration; import java.util.Arrays; import java.util.Locale; import java.util.Optional; @@ -249,9 +250,23 @@ public final void initializeState(StreamTaskStateInitializer streamTaskStateMana && areInterruptibleTimersConfigured() && getTimeServiceManager().isPresent()) { LOG.info("Interruptible timers enabled for {}", getClass().getSimpleName()); + InternalTimeServiceManager timeServiceManager = getTimeServiceManager().get(); + boolean emitIntermediateWatermarks = + runtimeContext + .getJobConfiguration() + .get( + CheckpointingOptions + .UNALIGNED_INTERRUPTIBLE_TIMERS_EMIT_INTERMEDIATE_WATERMARKS); + if (emitIntermediateWatermarks) { + timeServiceManager.configureIntermediateWatermarkInterval( + Duration.ofMillis(getExecutionConfig().getAutoWatermarkInterval())); + } watermarkProcessor = new MailboxWatermarkProcessor( - output, mailboxExecutor, getTimeServiceManager().get()); + output, + mailboxExecutor, + timeServiceManager, + emitIntermediateWatermarks); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java index e24e879afaeffe..dcc37bc7bedcc9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java @@ -30,6 +30,7 @@ import org.apache.flink.streaming.runtime.tasks.StreamTaskCancellationContext; import java.io.Serializable; +import java.time.Duration; /** * An entity keeping all the time-related services. @@ -79,6 +80,20 @@ InternalTimerService getInternalTimerService( boolean tryAdvanceWatermark(Watermark watermark, ShouldStopAdvancingFn shouldStopAdvancingFn) throws Exception; + /** + * Configures how often an intermediate watermark should be made available (see {@link + * #getReachedWatermark()}) while a {@link #tryAdvanceWatermark} call is interrupted before + * completing. A {@code interval} of {@link Duration#ZERO zero} disables this. Implementations + * that do not support interrupted watermark advancement may ignore this. + */ + default void configureIntermediateWatermarkInterval(Duration interval) {} + + /** + * Returns the highest watermark for which all managed {@link InternalTimerService timer + * services} are known to have fired all due timers. + */ + long getReachedWatermark(); + /** * Snapshots the timers to raw keyed state. * diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManagerImpl.java index c8510995cbd530..2b7f31c060afcc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManagerImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManagerImpl.java @@ -44,8 +44,11 @@ import java.io.IOException; import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -81,6 +84,11 @@ public class InternalTimeServiceManagerImpl implements InternalTimeServiceMan @Nullable AsyncExecutionController asyncExecutionController; + private long intermediateWatermarkIntervalMs = 0; + private boolean intermediateWatermarkNudgeScheduled = false; + private long reachedWatermark = Long.MIN_VALUE; + private int nextServiceStartIndex = 0; + private InternalTimeServiceManagerImpl( TaskIOMetricGroup taskIOMetricGroup, KeyGroupRange localKeyGroupRange, @@ -215,15 +223,59 @@ public void advanceWatermark(Watermark watermark) throws Exception { } } + @Override + public void configureIntermediateWatermarkInterval(Duration interval) { + this.intermediateWatermarkIntervalMs = interval.toMillis(); + } + @Override public boolean tryAdvanceWatermark( Watermark watermark, ShouldStopAdvancingFn shouldStopAdvancingFn) throws Exception { - for (InternalTimerServiceImpl service : timerServices.values()) { - if (!service.tryAdvanceWatermark(watermark.getTimestamp(), shouldStopAdvancingFn)) { - return false; + maybeScheduleIntermediateWatermarkNudge(); + List> services = new ArrayList<>(timerServices.values()); + boolean fullyAdvanced = true; + for (int i = 0; i < services.size() && fullyAdvanced; i++) { + // Rotate the starting service every call so that a persistently-behind service can't + // permanently starve the ones after it in a fixed iteration order: once one service + // is interrupted, stop attempting to fire on the remaining ones this round, but still + // fold their (possibly stale, from an earlier round) reachedWatermark into the min + // below. + InternalTimerServiceImpl service = + services.get((nextServiceStartIndex + i) % services.size()); + if (fullyAdvanced) { + fullyAdvanced = + service.tryAdvanceWatermark( + watermark.getTimestamp(), shouldStopAdvancingFn); } } - return true; + if (!services.isEmpty()) { + nextServiceStartIndex = (nextServiceStartIndex + 1) % services.size(); + } + long minReachedWatermark = Long.MAX_VALUE; + for (InternalTimerServiceImpl service : services) { + minReachedWatermark = Math.min(minReachedWatermark, service.getReachedWatermark()); + } + reachedWatermark = minReachedWatermark; + return fullyAdvanced; + } + + @Override + public long getReachedWatermark() { + return reachedWatermark; + } + + // A firing loop only yields once the mailbox has other mail waiting; ordinary record/watermark + // traffic doesn't go through the mailbox, so a long, otherwise-idle drain would never yield on + // its own. This periodic no-op mail forces a yield point at roughly the configured interval, + // without adding any per-timer clock check to the firing loop itself. + private void maybeScheduleIntermediateWatermarkNudge() { + if (intermediateWatermarkIntervalMs > 0 && !intermediateWatermarkNudgeScheduled) { + intermediateWatermarkNudgeScheduled = true; + processingTimeService.scheduleWithFixedDelay( + timestamp -> {}, + intermediateWatermarkIntervalMs, + intermediateWatermarkIntervalMs); + } } ////////////////// Fault Tolerance Methods /////////////////// diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceImpl.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceImpl.java index e24e8c9a62dae3..6beb8c87814ced 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceImpl.java @@ -71,6 +71,14 @@ public class InternalTimerServiceImpl implements InternalTimerService { */ protected long currentWatermark = Long.MIN_VALUE; + /** + * Unlike {@link #currentWatermark}, which is set to the requested target watermark before any + * timer fires, this only advances after each timer has actually fired, so {@link + * #getReachedWatermark()} can safely surface it downstream even while {@link + * #tryAdvanceWatermark} is interrupted before reaching its requested target. + */ + private long reachedWatermark = Long.MIN_VALUE; + /** * The one and only Future (if any) registered to execute the next {@link Triggerable} action, * when its (processing) time arrives. @@ -229,6 +237,10 @@ public void initializeWatermark(long watermark) { this.currentWatermark = watermark; } + long getReachedWatermark() { + return reachedWatermark; + } + @Override public void registerProcessingTimeTimer(N namespace, long time) { InternalTimer oldHead = processingTimeTimersQueue.peek(); @@ -339,10 +351,19 @@ public boolean tryAdvanceWatermark( eventTimeTimersQueue.poll(); triggerTarget.onEventTime(timer); taskIOMetricGroup.getNumFiredTimers().inc(); + // Other timers due at exactly this timestamp may still be unfired, so only claim + // progress strictly below it. + reachedWatermark = timer.getTimestamp() - 1; // Check if we should stop advancing after at least one iteration to guarantee progress // and prevent a potential starvation. interrupted = shouldStopAdvancingFn.test(); } + if (!interrupted) { + // The loop above ran to completion: every timer due at or before `time` has fired (or + // none were due), so the full requested watermark has been reached, not just the last + // fired timer's timestamp. + reachedWatermark = time; + } return !interrupted; } diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessor.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessor.java index fb498f65f07ee1..69c58f11097e69 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessor.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessor.java @@ -44,6 +44,7 @@ public class MailboxWatermarkProcessor { private final Output> output; private final MailboxExecutor mailboxExecutor; private final InternalTimeServiceManager internalTimeServiceManager; + private final boolean emitIntermediateWatermarks; /** * Flag to indicate whether a progress watermark is scheduled in the mailbox. This is used to @@ -53,13 +54,17 @@ public class MailboxWatermarkProcessor { private Watermark maxInputWatermark = Watermark.UNINITIALIZED; + private long lastEmittedIntermediateWatermark = Long.MIN_VALUE; + public MailboxWatermarkProcessor( Output> output, MailboxExecutor mailboxExecutor, - InternalTimeServiceManager internalTimeServiceManager) { + InternalTimeServiceManager internalTimeServiceManager, + boolean emitIntermediateWatermarks) { this.output = checkNotNull(output); this.mailboxExecutor = checkNotNull(mailboxExecutor); this.internalTimeServiceManager = checkNotNull(internalTimeServiceManager); + this.emitIntermediateWatermarks = emitIntermediateWatermarks; } public void emitWatermarkInsideMailbox(Watermark mark) throws Exception { @@ -73,8 +78,20 @@ private void emitWatermarkInsideMailbox() throws Exception { if (internalTimeServiceManager.tryAdvanceWatermark( maxInputWatermark, mailboxExecutor::shouldInterrupt)) { // In case output watermark has fully progressed emit it downstream. + lastEmittedIntermediateWatermark = maxInputWatermark.getTimestamp(); output.emitWatermark(maxInputWatermark); - } else if (!progressWatermarkScheduled) { + return; + } + if (emitIntermediateWatermarks) { + long reachedWatermark = internalTimeServiceManager.getReachedWatermark(); + if (reachedWatermark > lastEmittedIntermediateWatermark) { + // Firing was interrupted before completing; surface the progress made so far + // instead of leaving watermark advancement stalled until the whole drain finishes. + lastEmittedIntermediateWatermark = reachedWatermark; + output.emitWatermark(new Watermark(reachedWatermark)); + } + } + if (!progressWatermarkScheduled) { progressWatermarkScheduled = true; // We still have work to do, but we need to let other mails to be processed first. mailboxExecutor.execute( diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/sorted/state/BatchExecutionInternalTimeServiceManager.java b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/sorted/state/BatchExecutionInternalTimeServiceManager.java index df215a56501c33..aaaefc26dec22e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/sorted/state/BatchExecutionInternalTimeServiceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/streaming/api/operators/sorted/state/BatchExecutionInternalTimeServiceManager.java @@ -56,6 +56,8 @@ public class BatchExecutionInternalTimeServiceManager // should perform correctly when the timer fires. private final boolean asyncStateProcessingMode; + private long reachedWatermark = Long.MIN_VALUE; + public BatchExecutionInternalTimeServiceManager( ProcessingTimeService processingTimeService, boolean asyncStateProcessingMode) { this.processingTimeService = checkNotNull(processingTimeService); @@ -89,6 +91,7 @@ public void advanceWatermark(Watermark watermark) { if (watermark.getTimestamp() == Long.MAX_VALUE) { keySelected(null); } + reachedWatermark = watermark.getTimestamp(); } @Override @@ -98,6 +101,11 @@ public boolean tryAdvanceWatermark( return true; } + @Override + public long getReachedWatermark() { + return reachedWatermark; + } + @Override public void snapshotToRawKeyedState( KeyedStateCheckpointOutputStream context, String operatorName) throws Exception { diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessorTest.java b/flink-runtime/src/test/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessorTest.java index e0b76c4123f521..299c511f1661bd 100644 --- a/flink-runtime/src/test/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/streaming/api/operators/MailboxWatermarkProcessorTest.java @@ -53,7 +53,8 @@ void testEmitWatermarkInsideMailbox() throws Exception { new CollectorOutput<>(emittedElements), new MailboxExecutorImpl( mailbox, priority, StreamTaskActionExecutor.IMMEDIATE), - timerService); + timerService, + true); final List expectedOutput = new ArrayList<>(); watermarkProcessor.emitWatermarkInsideMailbox(new Watermark(1)); watermarkProcessor.emitWatermarkInsideMailbox(new Watermark(2)); @@ -84,6 +85,87 @@ void testEmitWatermarkInsideMailbox() throws Exception { assertThat(emittedElements).containsExactlyElementsOf(expectedOutput); } + /** + * An interruption unrelated to the intermediate-watermark nudge (e.g. a checkpoint mail) must + * not surface an intermediate watermark when the feature is disabled. + */ + @Test + void testIntermediateWatermarkNotEmittedWhenDisabled() throws Exception { + int priority = 42; + final List emittedElements = new ArrayList<>(); + final TaskMailboxImpl mailbox = new TaskMailboxImpl(); + final InternalTimeServiceManager timerService = + new NoOpInternalTimeServiceManager() { + @Override + public long getReachedWatermark() { + return 5; + } + }; + + final MailboxWatermarkProcessor> watermarkProcessor = + new MailboxWatermarkProcessor<>( + new CollectorOutput<>(emittedElements), + new MailboxExecutorImpl( + mailbox, priority, StreamTaskActionExecutor.IMMEDIATE), + timerService, + false); + + // Unrelated mail interrupts the firing loop for a reason unrelated to the nudge. + mailbox.put(new Mail(() -> {}, TaskMailbox.MIN_PRIORITY, "checkpoint mail")); + + watermarkProcessor.emitWatermarkInsideMailbox(new Watermark(10)); + + // configureIntermediateWatermarkInterval() was never called on this manager. + assertThat(emittedElements).isEmpty(); + } + + /** + * Once a watermark has been fully emitted via the shortcut branch, a later interrupted advance + * must not emit an intermediate watermark below it. + */ + @Test + void testIntermediateWatermarkNeverBelowAlreadyEmittedWatermark() throws Exception { + int priority = 42; + final List emittedElements = new ArrayList<>(); + final TaskMailboxImpl mailbox = new TaskMailboxImpl(); + final boolean[] fullyAdvancedOnce = new boolean[] {false}; + final InternalTimeServiceManager timerService = + new NoOpInternalTimeServiceManager() { + @Override + public boolean tryAdvanceWatermark( + Watermark watermark, ShouldStopAdvancingFn shouldStopAdvancingFn) { + if (!fullyAdvancedOnce[0]) { + fullyAdvancedOnce[0] = true; + return true; + } + return false; + } + + @Override + public long getReachedWatermark() { + return 5; + } + }; + + final MailboxWatermarkProcessor> watermarkProcessor = + new MailboxWatermarkProcessor<>( + new CollectorOutput<>(emittedElements), + new MailboxExecutorImpl( + mailbox, priority, StreamTaskActionExecutor.IMMEDIATE), + timerService, + true); + + // Fully advances to 10 via the shortcut branch. + watermarkProcessor.emitWatermarkInsideMailbox(new Watermark(10)); + // A new, higher watermark arrives but is interrupted; reachedWatermark(5) is below the + // watermark(10) already emitted above. + watermarkProcessor.emitWatermarkInsideMailbox(new Watermark(20)); + + assertThat(emittedElements) + .as("watermarks must be non-decreasing") + .containsExactly(new Watermark(10)); + } + private static class NoOpInternalTimeServiceManager implements InternalTimeServiceManager { @Override @@ -106,6 +188,11 @@ public boolean tryAdvanceWatermark( return !shouldStopAdvancingFn.test(); } + @Override + public long getReachedWatermark() { + return Long.MIN_VALUE; + } + @Override public void snapshotToRawKeyedState( KeyedStateCheckpointOutputStream stateCheckpointOutputStream, String operatorName) diff --git a/flink-runtime/src/test/resources/arraylist-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/arraylist-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..2bc2cfe5bfbd11 Binary files /dev/null and b/flink-runtime/src/test/resources/arraylist-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/arraylist-serializer-2.3/test-data b/flink-runtime/src/test/resources/arraylist-serializer-2.3/test-data new file mode 100644 index 00000000000000..ac798598cd7d02 Binary files /dev/null and b/flink-runtime/src/test/resources/arraylist-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/buffer-entry-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/buffer-entry-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..4026c41730601e Binary files /dev/null and b/flink-runtime/src/test/resources/buffer-entry-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/buffer-entry-serializer-2.3/test-data b/flink-runtime/src/test/resources/buffer-entry-serializer-2.3/test-data new file mode 100644 index 00000000000000..7072021895f5cb Binary files /dev/null and b/flink-runtime/src/test/resources/buffer-entry-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/global-window-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/global-window-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..ac44c4c40c10bc Binary files /dev/null and b/flink-runtime/src/test/resources/global-window-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/global-window-serializer-2.3/test-data b/flink-runtime/src/test/resources/global-window-serializer-2.3/test-data new file mode 100644 index 00000000000000..f76dd238ade089 Binary files /dev/null and b/flink-runtime/src/test/resources/global-window-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/java-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/java-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..db1abe95eda9ea Binary files /dev/null and b/flink-runtime/src/test/resources/java-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/java-serializer-2.3/test-data b/flink-runtime/src/test/resources/java-serializer-2.3/test-data new file mode 100644 index 00000000000000..22a3c768e29ead Binary files /dev/null and b/flink-runtime/src/test/resources/java-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/stream-element-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/stream-element-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..2a9a9aea83a6d6 Binary files /dev/null and b/flink-runtime/src/test/resources/stream-element-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/stream-element-serializer-2.3/test-data b/flink-runtime/src/test/resources/stream-element-serializer-2.3/test-data new file mode 100644 index 00000000000000..3022a0815fa6ad Binary files /dev/null and b/flink-runtime/src/test/resources/stream-element-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/time-window-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/time-window-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7d2b4b69a6ad7b Binary files /dev/null and b/flink-runtime/src/test/resources/time-window-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/time-window-serializer-2.3/test-data b/flink-runtime/src/test/resources/time-window-serializer-2.3/test-data new file mode 100644 index 00000000000000..bd72696cdd363b Binary files /dev/null and b/flink-runtime/src/test/resources/time-window-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/timer-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/timer-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..67e3a083d81a2d Binary files /dev/null and b/flink-runtime/src/test/resources/timer-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/timer-serializer-2.3/test-data b/flink-runtime/src/test/resources/timer-serializer-2.3/test-data new file mode 100644 index 00000000000000..5e9827d82b43c9 Binary files /dev/null and b/flink-runtime/src/test/resources/timer-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/ttl-aware-serializer-string-value-2.3/serializer-snapshot b/flink-runtime/src/test/resources/ttl-aware-serializer-string-value-2.3/serializer-snapshot new file mode 100644 index 00000000000000..d3e266190e85ef Binary files /dev/null and b/flink-runtime/src/test/resources/ttl-aware-serializer-string-value-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/ttl-aware-serializer-string-value-2.3/test-data b/flink-runtime/src/test/resources/ttl-aware-serializer-string-value-2.3/test-data new file mode 100644 index 00000000000000..aaf5589b658a29 --- /dev/null +++ b/flink-runtime/src/test/resources/ttl-aware-serializer-string-value-2.3/test-data @@ -0,0 +1 @@ + hello Gordon \ No newline at end of file diff --git a/flink-runtime/src/test/resources/ttl-aware-serializer-ttl-value-2.3/serializer-snapshot b/flink-runtime/src/test/resources/ttl-aware-serializer-ttl-value-2.3/serializer-snapshot new file mode 100644 index 00000000000000..a1dde1d34fcd12 Binary files /dev/null and b/flink-runtime/src/test/resources/ttl-aware-serializer-ttl-value-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/ttl-aware-serializer-ttl-value-2.3/test-data b/flink-runtime/src/test/resources/ttl-aware-serializer-ttl-value-2.3/test-data new file mode 100644 index 00000000000000..5596986cd7a085 Binary files /dev/null and b/flink-runtime/src/test/resources/ttl-aware-serializer-ttl-value-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/ttl-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/ttl-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..7668b613bd9a26 Binary files /dev/null and b/flink-runtime/src/test/resources/ttl-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/ttl-serializer-2.3/test-data b/flink-runtime/src/test/resources/ttl-serializer-2.3/test-data new file mode 100644 index 00000000000000..5596986cd7a085 Binary files /dev/null and b/flink-runtime/src/test/resources/ttl-serializer-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/union-serializer-one-2.3/serializer-snapshot b/flink-runtime/src/test/resources/union-serializer-one-2.3/serializer-snapshot new file mode 100644 index 00000000000000..8af8c712a44e1f Binary files /dev/null and b/flink-runtime/src/test/resources/union-serializer-one-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/union-serializer-one-2.3/test-data b/flink-runtime/src/test/resources/union-serializer-one-2.3/test-data new file mode 100644 index 00000000000000..9a2dda7f54c7ee --- /dev/null +++ b/flink-runtime/src/test/resources/union-serializer-one-2.3/test-data @@ -0,0 +1 @@ +flink \ No newline at end of file diff --git a/flink-runtime/src/test/resources/union-serializer-two-2.3/serializer-snapshot b/flink-runtime/src/test/resources/union-serializer-two-2.3/serializer-snapshot new file mode 100644 index 00000000000000..8af8c712a44e1f Binary files /dev/null and b/flink-runtime/src/test/resources/union-serializer-two-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/union-serializer-two-2.3/test-data b/flink-runtime/src/test/resources/union-serializer-two-2.3/test-data new file mode 100644 index 00000000000000..564b7c9efdc9df Binary files /dev/null and b/flink-runtime/src/test/resources/union-serializer-two-2.3/test-data differ diff --git a/flink-runtime/src/test/resources/void-namespace-serializer-2.3/serializer-snapshot b/flink-runtime/src/test/resources/void-namespace-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..9df815a3705c3b Binary files /dev/null and b/flink-runtime/src/test/resources/void-namespace-serializer-2.3/serializer-snapshot differ diff --git a/flink-runtime/src/test/resources/void-namespace-serializer-2.3/test-data b/flink-runtime/src/test/resources/void-namespace-serializer-2.3/test-data new file mode 100644 index 00000000000000..f76dd238ade089 Binary files /dev/null and b/flink-runtime/src/test/resources/void-namespace-serializer-2.3/test-data differ diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/UnalignedCheckpointsInterruptibleTimersTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/UnalignedCheckpointsInterruptibleTimersTest.java index 47bfbe84592d82..1750c0e20b733c 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/UnalignedCheckpointsInterruptibleTimersTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/UnalignedCheckpointsInterruptibleTimersTest.java @@ -98,11 +98,15 @@ void testSingleWatermarkHoldingOperatorInTheChain() throws Exception { assertThat(harness.getOutput()) .containsExactly( asFiredRecord("key-0"), + // Intermediate watermark surfacing progress after firing the first of + // the 2 timers due at firstWindowEnd, before the drain is interrupted. + asWatermark(Instant.ofEpochMilli(firstWindowEnd.toEpochMilli() - 1)), asMailRecord("key-0"), asFiredRecord("key-1"), asMailRecord("key-1"), asWatermark(firstWindowEnd), asFiredRecord("key-0"), + asWatermark(Instant.ofEpochMilli(secondWindowEnd.toEpochMilli() - 1)), asMailRecord("key-0"), asFiredRecord("key-1"), asMailRecord("key-1"), @@ -195,6 +199,9 @@ void testDeferredWatermarkIsEmittedBeforeEndOfData() throws Exception { assertThat(harness.getOutput()) .containsExactly( asFiredRecord("key-0"), + // Intermediate watermark surfacing progress after firing the first of + // the 2 timers due at windowEnd, before the drain is interrupted. + asWatermark(Instant.ofEpochMilli(windowEnd.toEpochMilli() - 1)), asMailRecord("key-0"), asFiredRecord("key-1"), asMailRecord("key-1"), @@ -203,6 +210,120 @@ void testDeferredWatermarkIsEmittedBeforeEndOfData() throws Exception { } } + @Test + void testIntermediateWatermarksEmittedDuringLongDrain() throws Exception { + final Instant t1 = Instant.ofEpochMilli(100L); + final Instant t2 = Instant.ofEpochMilli(200L); + final Instant t3 = Instant.ofEpochMilli(300L); + + try (final StreamTaskMailboxTestHarness harness = + new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, Types.STRING) + .addJobConfig( + CheckpointingOptions.CHECKPOINTING_INTERVAL, Duration.ofSeconds(1)) + .addJobConfig(CheckpointingOptions.ENABLE_UNALIGNED, true) + .addJobConfig( + CheckpointingOptions.ENABLE_UNALIGNED_INTERRUPTIBLE_TIMERS, true) + .modifyStreamConfig( + UnalignedCheckpointsInterruptibleTimersTest::setupStreamConfig) + .addInput(Types.STRING) + .setupOperatorChain( + SimpleOperatorFactory.of( + new MultipleTimersAtTheSameTimestamp() + .withTimers(t1, 1) + .withTimers(t2, 1) + .withTimers(t3, 1))) + .name("first") + .finishForSingletonOperatorChain(StringSerializer.INSTANCE) + .build()) { + harness.setAutoProcess(false); + harness.processElement(new StreamRecord<>("register timers")); + harness.processAll(); + // A single watermark whose drain requires firing multiple, individually-interrupted + // timers (each fired timer schedules a mailbox mail, forcing an interruption). + harness.processElement(asWatermark(t3)); + + final List seenWatermarks = new ArrayList<>(); + while (seenWatermarks.isEmpty() + || seenWatermarks.get(seenWatermarks.size() - 1).getTimestamp() + < t3.toEpochMilli()) { + harness.processSingleStep(); + Object outputElement; + while ((outputElement = harness.getOutput().poll()) != null) { + if (outputElement instanceof Watermark) { + seenWatermarks.add((Watermark) outputElement); + } + } + } + + // The drain is interrupted after firing each of the 3 timers. Progress made before + // the final interruption should be visible downstream as intermediate watermarks, + // not only as the single final watermark once the whole drain completes. + assertThat(seenWatermarks).hasSizeGreaterThan(1); + assertThat(seenWatermarks.get(0).getTimestamp()).isLessThan(t3.toEpochMilli()); + assertThat(seenWatermarks.get(seenWatermarks.size() - 1).getTimestamp()) + .isEqualTo(t3.toEpochMilli()); + assertThat(seenWatermarks).extracting(Watermark::getTimestamp).isSorted(); + } + } + + /** + * Once one timer service is interrupted, {@link + * org.apache.flink.streaming.api.operators.InternalTimeServiceManagerImpl#tryAdvanceWatermark} + * never even attempts the other services this round, so a persistently-behind service can + * starve the others' contribution to the reported intermediate watermark for as long as it + * itself keeps getting interrupted. + */ + @Test + void testStarvedTimerServiceDelaysIntermediateWatermark() throws Exception { + final int timersPerService = 20; + final Instant watermark = Instant.ofEpochMilli(timersPerService + 10L); + + try (final StreamTaskMailboxTestHarness harness = + new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, Types.STRING) + .addJobConfig( + CheckpointingOptions.CHECKPOINTING_INTERVAL, Duration.ofSeconds(1)) + .addJobConfig(CheckpointingOptions.ENABLE_UNALIGNED, true) + .addJobConfig( + CheckpointingOptions.ENABLE_UNALIGNED_INTERRUPTIBLE_TIMERS, true) + .modifyStreamConfig( + UnalignedCheckpointsInterruptibleTimersTest::setupStreamConfig) + .addInput(Types.LONG) + .setupOperatorChain( + SimpleOperatorFactory.of(new TwoTimerServicesWithEqualBacklogs())) + .name("first") + .finishForSingletonOperatorChain(StringSerializer.INSTANCE) + .build()) { + harness.setAutoProcess(false); + for (long ts = 1; ts <= timersPerService; ts++) { + harness.processElement(new StreamRecord<>(ts)); + } + harness.processAll(); + harness.processElement(asWatermark(watermark)); + + int firedCount = 0; + Watermark firstWatermark = null; + while (firstWatermark == null && firedCount < 2 * timersPerService) { + harness.processSingleStep(); + Object outputElement; + while ((outputElement = harness.getOutput().poll()) != null) { + if (outputElement instanceof Watermark) { + firstWatermark = (Watermark) outputElement; + break; + } + firedCount++; + } + } + + assertThat(firstWatermark).as("a watermark should eventually appear").isNotNull(); + // With two equally-backlogged services, an intermediate watermark should surface well + // before either service's entire backlog has drained -- not only once one of them + // (whichever happens to be attempted first) has completely finished firing. + assertThat(firedCount) + .as("first watermark should appear before either backlog is drained") + .isLessThan(timersPerService); + } + } + private static Watermark asWatermark(Instant timestamp) { return new Watermark(timestamp.toEpochMilli()); } @@ -280,6 +401,56 @@ MultipleTimersAtTheSameTimestamp withTimers(Instant timestamp, int count) { } } + /** + * Registers one timer per element (at the timestamp given by the element's value) on each of + * two independently-named timer services, each firing one timer at a time (interrupted via a + * scheduled mail). + */ + private static class TwoTimerServicesWithEqualBacklogs extends AbstractStreamOperator + implements OneInputStreamOperator, + Triggerable, + YieldingOperator { + + private transient @Nullable MailboxExecutor mailboxExecutor; + private transient InternalTimerService serviceA; + private transient InternalTimerService serviceB; + + @Override + public boolean useInterruptibleTimers(ReadableConfig config) { + return true; + } + + @Override + public void setMailboxExecutor(MailboxExecutor mailboxExecutor) { + super.setMailboxExecutor(mailboxExecutor); + this.mailboxExecutor = mailboxExecutor; + } + + @Override + public void open() throws Exception { + super.open(); + serviceA = getInternalTimerService("serviceA", StringSerializer.INSTANCE, this); + serviceB = getInternalTimerService("serviceB", StringSerializer.INSTANCE, this); + } + + @Override + public void processElement(StreamRecord element) { + setCurrentKey("key"); + long ts = element.getValue(); + serviceA.registerEventTimeTimer("A", ts); + serviceB.registerEventTimeTimer("B", ts); + } + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + mailboxExecutor.execute(() -> {}, "mail"); + output.collect(asFiredRecord(timer.getNamespace() + "-" + timer.getTimestamp())); + } + + @Override + public void onProcessingTime(InternalTimer timer) throws Exception {} + } + /** * Registers a timer for the current key on every element; every fired timer enqueues a mail, so * that firing is interrupted after each timer. diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink2.3-snapshot new file mode 100644 index 00000000000000..bdb12e5f995040 Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink2.3-snapshot differ diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink2.3-snapshot new file mode 100644 index 00000000000000..afe1e9a6a438f4 Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink2.3-snapshot differ diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink2.3-snapshot new file mode 100644 index 00000000000000..ef67e59f01ef22 Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink2.3-snapshot differ diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink2.3-snapshot new file mode 100644 index 00000000000000..728a1473730718 Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink2.3-snapshot differ diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink2.3-snapshot new file mode 100644 index 00000000000000..34ee05b8eb88bb Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink2.3-snapshot differ diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink2.3-snapshot new file mode 100644 index 00000000000000..17cdda08dedab1 Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink2.3-snapshot differ diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink2.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink2.3-snapshot new file mode 100644 index 00000000000000..bdbfae3a80a521 Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink2.3-snapshot differ diff --git a/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj b/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj index cb9121e97d183a..a031434e8ca709 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj +++ b/flink-table/flink-sql-parser/src/main/codegen/templates/Parser.jj @@ -1658,7 +1658,8 @@ SqlNode PartitionedQueryOrQueryOrExpr(ExprContext exprContext) : SqlNode e; } { - e = OrderedQueryOrExpr(exprContext) + // QueryOrExpr, not OrderedQueryOrExpr: ORDER BY is handled by PartitionedByAndOrderBy below. + e = QueryOrExpr(exprContext) e = PartitionedByAndOrderBy(e) { return e; } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/EarlyFireJoinHintOptions.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/EarlyFireJoinHintOptions.java new file mode 100644 index 00000000000000..6c919f318f2469 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/config/EarlyFireJoinHintOptions.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.api.config; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; + +import org.apache.flink.shaded.guava33.com.google.common.collect.ImmutableSet; + +import java.time.Duration; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.flink.configuration.ConfigOptions.key; + +/** + * This class holds hint option name definitions for EARLY_FIRE join hints based on {@link + * org.apache.flink.configuration.ConfigOption}. + */ +@PublicEvolving +public class EarlyFireJoinHintOptions { + + public static final ConfigOption DELAY = + key("delay") + .durationType() + .noDefaultValue() + .withDescription( + "The delay between the time an unmatched outer row becomes eligible to" + + " be emitted with null padding and the time it is actually" + + " emitted. Must be at least 1 millisecond."); + + public static final ConfigOption TIME_MODE = + key("time-mode") + .enumType(TimeMode.class) + .noDefaultValue() + .withDescription( + "The time domain that drives the early-fire delay, can be 'rowtime' or" + + " 'proctime'. If not set, it defaults to the time domain of" + + " the interval join."); + + private static final Set> requiredKeys = new HashSet<>(); + private static final Set> supportedKeys = new HashSet<>(); + + static { + requiredKeys.add(DELAY); + + supportedKeys.add(DELAY); + supportedKeys.add(TIME_MODE); + } + + public static ImmutableSet getRequiredOptions() { + return ImmutableSet.copyOf(requiredKeys); + } + + public static ImmutableSet getSupportedOptions() { + return ImmutableSet.copyOf(supportedKeys); + } + + /** The time domain that drives the early-fire delay. */ + @PublicEvolving + public enum TimeMode { + ROWTIME, + PROCTIME + } +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/CatalogManager.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/CatalogManager.java index ca31e17ebf4e48..fa9db8cb50dbda 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/CatalogManager.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/CatalogManager.java @@ -1114,7 +1114,21 @@ private boolean temporaryDatabaseExists(String catalogName, String databaseName) } private boolean permanentDatabaseExists(String catalogName, String databaseName) { - return getCatalog(catalogName).map(c -> c.databaseExists(databaseName)).orElse(false); + return getCatalog(catalogName) + .map( + c -> { + try { + return c.databaseExists(databaseName); + } catch (CatalogException e) { + LOG.warn( + "Unable to check whether database '{}' exists in catalog '{}'.", + databaseName, + catalogName, + e); + return false; + } + }) + .orElse(false); } /** diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java index 5e4a02cb9fb678..7364536099a7cc 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/resolver/rules/ResolveCallByArgumentsRule.java @@ -299,6 +299,9 @@ private UnresolvedCallExpression executeAssignment( functionName)); } + SystemTypeInference.checkNoSystemArguments( + inference.disableSystemArguments(), namedArgs.keySet(), functionName); + fillInDefaultNamedArguments(declaredArgs, namedArgs); fillInPtfSpecificNamedArguments( functionName, definition, declaredArgs, namedArgs, actualArgs); diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/ddl/CreateConnectionOperation.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/ddl/CreateConnectionOperation.java new file mode 100644 index 00000000000000..b853150b955937 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/ddl/CreateConnectionOperation.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.operations.ddl; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.internal.TableResultImpl; +import org.apache.flink.table.api.internal.TableResultInternal; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.SensitiveConnection; +import org.apache.flink.table.operations.Operation; +import org.apache.flink.table.operations.OperationUtils; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** Operation to describe a CREATE CONNECTION statement. */ +@Internal +public class CreateConnectionOperation implements CreateOperation { + + private static final String MASKED_VALUE = "****"; + + private final ObjectIdentifier connectionIdentifier; + private final SensitiveConnection sensitiveConnection; + private final boolean ignoreIfExists; + private final boolean isTemporary; + + public CreateConnectionOperation( + ObjectIdentifier connectionIdentifier, + SensitiveConnection sensitiveConnection, + boolean ignoreIfExists, + boolean isTemporary) { + this.connectionIdentifier = connectionIdentifier; + this.sensitiveConnection = sensitiveConnection; + this.ignoreIfExists = ignoreIfExists; + this.isTemporary = isTemporary; + } + + public ObjectIdentifier getConnectionIdentifier() { + return connectionIdentifier; + } + + public SensitiveConnection getSensitiveConnection() { + return sensitiveConnection; + } + + public boolean isIgnoreIfExists() { + return ignoreIfExists; + } + + public boolean isTemporary() { + return isTemporary; + } + + @Override + public String asSummaryString() { + Map maskedOptions = + sensitiveConnection.getOptions().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> MASKED_VALUE, + (a, b) -> a, + LinkedHashMap::new)); + Map params = new LinkedHashMap<>(); + params.put("connectionOptions", maskedOptions); + params.put("identifier", connectionIdentifier); + params.put("ignoreIfExists", ignoreIfExists); + params.put("isTemporary", isTemporary); + + return OperationUtils.formatWithChildren( + "CREATE CONNECTION", params, List.of(), Operation::asSummaryString); + } + + @Override + public TableResultInternal execute(Context ctx) { + if (isTemporary) { + ctx.getCatalogManager() + .createTemporaryConnection( + sensitiveConnection, connectionIdentifier, ignoreIfExists); + } else { + ctx.getCatalogManager() + .createConnection(sensitiveConnection, connectionIdentifier, ignoreIfExists); + } + return TableResultImpl.TABLE_RESULT_OK; + } +} diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/CatalogManagerTest.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/CatalogManagerTest.java index 907d03dfcf8d1a..51774ffb6f6b17 100644 --- a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/CatalogManagerTest.java +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/CatalogManagerTest.java @@ -245,6 +245,43 @@ void testTableModificationListener() throws Exception { assertThat(dropTemporaryEvent.identifier().getObjectName()).isEqualTo("table2"); } + @Test + void testSchemaExistsSwallowsCatalogExceptionFromDatabaseExists() { + CatalogManager catalogManager = + CatalogManagerMocks.preparedCatalogManager() + .defaultCatalog("broken", new UnreachableCatalog("broken")) + .classLoader(CatalogManagerTest.class.getClassLoader()) + .config(new Configuration()) + .catalogStoreHolder( + CatalogStoreHolder.newBuilder() + .classloader(CatalogManagerTest.class.getClassLoader()) + .catalogStore(new GenericInMemoryCatalogStore()) + .config(new Configuration()) + .build()) + .build(); + assertThat(catalogManager.schemaExists("broken", "default")).isFalse(); + } + + /** + * A catalog whose {@link #databaseExists(String)} always fails, simulating a connectivity + * problem with an unreachable destination. + */ + private static class UnreachableCatalog extends GenericInMemoryCatalog { + UnreachableCatalog(String name) { + super(name, "default"); + } + + @Override + public boolean databaseExists(String databaseName) { + throw new CatalogException( + "Failed to connect to database '" + + databaseName + + "' of catalog '" + + getName() + + "'."); + } + } + @Test public void testDropCurrentDatabase() throws Exception { CatalogManager catalogManager = createCatalogManager(null); diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-case-class-serializer-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/scala-case-class-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..10d9840a6ce40b Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-case-class-serializer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-case-class-serializer-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/scala-case-class-serializer-2.3/test-data new file mode 100644 index 00000000000000..ddb6ef533cd3ed Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-case-class-serializer-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-either-serializer-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/scala-either-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..ba47931818671b Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-either-serializer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-either-serializer-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/scala-either-serializer-2.3/test-data new file mode 100644 index 00000000000000..70cd7bf9fb3448 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-either-serializer-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-enum-serializer-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/scala-enum-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..a2fab54f78327b Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-enum-serializer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-enum-serializer-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/scala-enum-serializer-2.3/test-data new file mode 100644 index 00000000000000..593f4708db84ac Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-enum-serializer-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-option-serializer-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/scala-option-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..1a4c1ab5d74378 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-option-serializer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-option-serializer-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/scala-option-serializer-2.3/test-data new file mode 100644 index 00000000000000..f76dd238ade089 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-option-serializer-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-try-serializer-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/scala-try-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..148cf99ef2b207 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-try-serializer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/scala-try-serializer-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/scala-try-serializer-2.3/test-data new file mode 100644 index 00000000000000..a413c04a650534 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/scala-try-serializer-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-bitset-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-bitset-2.3/serializer-snapshot new file mode 100644 index 00000000000000..fe5281ef4d83b7 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-bitset-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-bitset-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-bitset-2.3/test-data new file mode 100644 index 00000000000000..5eb5107ad6c88c Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-bitset-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-indexedseq-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-indexedseq-2.3/serializer-snapshot new file mode 100644 index 00000000000000..367b4a8e86d596 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-indexedseq-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-indexedseq-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-indexedseq-2.3/test-data new file mode 100644 index 00000000000000..ca6108ccd17bf5 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-indexedseq-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-linearseq-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-linearseq-2.3/serializer-snapshot new file mode 100644 index 00000000000000..52902d19346bbd Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-linearseq-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-linearseq-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-linearseq-2.3/test-data new file mode 100644 index 00000000000000..3b0d0b065b0b1d Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-linearseq-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..976c3c220a4043 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.3/test-data new file mode 100644 index 00000000000000..ca6108ccd17bf5 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-map-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-map-2.3/serializer-snapshot new file mode 100644 index 00000000000000..88e4061ef6da64 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-map-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-map-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-map-2.3/test-data new file mode 100644 index 00000000000000..f209ff5f4fe111 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-map-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-seq-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-seq-2.3/serializer-snapshot new file mode 100644 index 00000000000000..94a2b25dcf8fbd Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-seq-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-seq-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-seq-2.3/test-data new file mode 100644 index 00000000000000..ca6108ccd17bf5 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-seq-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-set-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-set-2.3/serializer-snapshot new file mode 100644 index 00000000000000..af3fea13ded844 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-set-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-set-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-set-2.3/test-data new file mode 100644 index 00000000000000..3b0d0b065b0b1d Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-set-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-case-class-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-case-class-2.3/serializer-snapshot new file mode 100644 index 00000000000000..588c8b45a73d23 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-case-class-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-case-class-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-case-class-2.3/test-data new file mode 100644 index 00000000000000..7adaf0059e94f3 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-case-class-2.3/test-data differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-pojo-2.3/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-pojo-2.3/serializer-snapshot new file mode 100644 index 00000000000000..c31c582e21e2de Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-pojo-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-pojo-2.3/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-pojo-2.3/test-data new file mode 100644 index 00000000000000..ebc7236c833187 Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-with-pojo-2.3/test-data differ diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/SystemTypeInference.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/SystemTypeInference.java index 971831d6ac442b..a1f9fbc5a9fdd9 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/SystemTypeInference.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/SystemTypeInference.java @@ -46,6 +46,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -126,6 +127,33 @@ public static boolean isInvalidUidForProcessTableFunction(String uid) { return !UID_FORMAT.test(uid); } + /** + * Rejects the implicit system arguments ({@code on_time}, {@code uid}) for a function that + * disables them via {@link TypeInference#disableSystemArguments()}. + * + *

The system arguments are not part of such a function's signature. Enforcing this from + * every translation path (SQL operand checking and Table API call resolution) rejects them + * consistently, regardless of whether the function is processed by the generic PTF rule or a + * dedicated optimizer rule (e.g. ML_PREDICT, LATERAL SNAPSHOT). + */ + public static void checkNoSystemArguments( + boolean sysArgsDisabled, + Collection suppliedArgumentNames, + String functionName) { + if (!sysArgsDisabled) { + return; + } + for (StaticArgument systemArg : PROCESS_TABLE_FUNCTION_SYSTEM_ARGS) { + if (suppliedArgumentNames.contains(systemArg.getName())) { + throw new ValidationException( + String.format( + "Invalid function call. The '%s' argument is not supported " + + "because function '%s' does not use system arguments.", + systemArg.getName(), functionName)); + } + } + } + // -------------------------------------------------------------------------------------------- private static void checkScalarArgsOnly(List defaultArgs) { diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index b8f76f7f969929..450dc204ec69ed 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -158,6 +158,12 @@ under the License. flink-table-runtime ${project.version} + + org.apache.flink + flink-table-type-utils + ${project.version} + ${flink.markBundledAsOptional} + diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java new file mode 100644 index 00000000000000..385b866c724c1b --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/SqlSetSemanticsTableOperator.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Copied from Calcite because of CALCITE-7660. Should be removed after upgrade to Calcite 1.43.0. + */ +public class SqlSetSemanticsTableOperator extends SqlInternalOperator { + + // ~ Constructors ----------------------------------------------------------- + + public SqlSetSemanticsTableOperator() { + super("SET_SEMANTICS_TABLE", SqlKind.SET_SEMANTICS_TABLE); + } + + @Override + public SqlCall createCall( + @Nullable SqlLiteral functionQualifier, + SqlParserPos pos, + @Nullable SqlNode... operands) { + assert operands.length == 3; + SqlNode partitionList = operands[1]; + SqlNode orderList = operands[2]; + assert (partitionList != null && !SqlNodeList.isEmptyList(partitionList)) + || (orderList != null && !SqlNodeList.isEmptyList(orderList)); + return super.createCall(functionQualifier, pos, operands); + } + + @Override + public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { + call.operand(0).unparse(writer, 0, 0); + + SqlNodeList partitionList = call.operand(1); + if (!partitionList.isEmpty()) { + writer.sep("PARTITION BY"); + // FLINK MODIFICATION BEGIN + final SqlWriter.Frame partitionFrame = + partitionList.size() == 1 + ? writer.startList("", "") + : writer.startList("(", ")"); + // FLINK MODIFICATION END + partitionList.unparse(writer, 0, 0); + writer.endList(partitionFrame); + } + SqlNodeList orderList = call.operand(2); + if (!orderList.isEmpty()) { + writer.sep("ORDER BY"); + // FLINK MODIFICATION BEGIN + final SqlWriter.Frame orderFrame = + orderList.size() == 1 ? writer.startList("", "") : writer.startList("(", ")"); + orderList.unparse(writer, 0, 0); + writer.endList(orderFrame); + // FLINK MODIFICATION END + } + } + + @Override + public RelDataType deriveType(SqlValidator validator, SqlValidatorScope scope, SqlCall call) { + final List operands = call.getOperandList(); + return requireNonNull(validator.deriveType(scope, operands.get(0))); + } + + @Override + public boolean argumentMustBeScalar(int ordinal) { + return false; + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java index 651a86925c373f..0eb925352f0791 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidator.java @@ -30,9 +30,11 @@ import org.apache.flink.table.functions.FunctionKind; import org.apache.flink.table.planner.catalog.CatalogSchemaModel; import org.apache.flink.table.planner.catalog.CatalogSchemaTable; +import org.apache.flink.table.planner.functions.bridging.BridgingSqlFunction; import org.apache.flink.table.planner.plan.FlinkCalciteCatalogReader; import org.apache.flink.table.planner.plan.utils.FlinkRexUtil; import org.apache.flink.table.planner.utils.ShortcutUtils; +import org.apache.flink.table.types.inference.SystemTypeInference; import org.apache.flink.table.types.logical.DecimalType; import org.apache.calcite.plan.RelOptCluster; @@ -89,6 +91,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -383,6 +386,7 @@ protected void addToSelectList( final SqlBasicCall call = (SqlBasicCall) node; checkNoNamedAndPositionalMixedArgs(call); + checkDisabledSystemArgs(call); // Special case for MODEL if (node instanceof SqlExplicitModelCall) { @@ -456,6 +460,32 @@ private static void checkNoNamedAndPositionalMixedArgs(SqlBasicCall call) { } } + /** + * Rejects the implicit PTF system arguments (on_time, uid) for functions that disable them. + * + *

This must happen before Calcite permutes named arguments, because unknown named arguments + * are silently dropped during permutation and would otherwise be lost. The actual rule and + * error message live in {@link SystemTypeInference#checkNoSystemArguments} so that the Table + * API path (which resolves calls without this validator) enforces it identically. + */ + private static void checkDisabledSystemArgs(SqlBasicCall call) { + final SqlOperator operator = call.getOperator(); + if (!(operator instanceof BridgingSqlFunction) + || !((BridgingSqlFunction) operator).getTypeInference().disableSystemArguments()) { + return; + } + final Set suppliedArgNames = new HashSet<>(); + for (SqlNode operand : call.getOperandList()) { + if (operand != null && operand.getKind() == SqlKind.ARGUMENT_ASSIGNMENT) { + final SqlNode nameNode = ((SqlCall) operand).operand(1); + if (nameNode instanceof SqlIdentifier) { + suppliedArgNames.add(((SqlIdentifier) nameNode).getSimple()); + } + } + } + SystemTypeInference.checkNoSystemArguments(true, suppliedArgNames, operator.getName()); + } + @Override public SqlNode maybeCast(SqlNode node, RelDataType currentType, RelDataType desiredType) { return super.maybeCast(node, currentType, desiredType); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToArrayCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToArrayCastRule.java index df687fdb9196c2..d3877160cb7c50 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToArrayCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToArrayCastRule.java @@ -55,7 +55,6 @@ private static boolean isValidArrayCasting( return CastRuleProvider.resolve(innerInputType, innerTargetType) != null; } - @SuppressWarnings("rawtypes") @Override protected String generateCodeBlockInternal( CodeGeneratorCastRule.Context context, @@ -67,16 +66,17 @@ protected String generateCodeBlockInternal( final LogicalType innerTargetType = ((ArrayType) targetLogicalType).getElementType(); final String innerTargetTypeTerm = arrayElementType(innerTargetType); - final String arraySize = methodCall(inputTerm, "size"); + final String arraySizeTerm = newName(context.getCodeGeneratorContext(), "arraySize"); final String objArrayTerm = newName(context.getCodeGeneratorContext(), "objArray"); return new CastRuleUtils.CodeWriter() + .declStmt(int.class, arraySizeTerm, methodCall(inputTerm, "size")) .declStmt( innerTargetTypeTerm + "[]", objArrayTerm, - newArray(innerTargetTypeTerm, arraySize)) + newArray(innerTargetTypeTerm, arraySizeTerm)) .forStmt( - arraySize, + arraySizeTerm, (index, loopWriter) -> { CastCodeBlock codeBlock = // Null check is done at the array access level @@ -97,10 +97,18 @@ protected String generateCodeBlockInternal( index, codeBlock.getReturnTerm())); } else { - loopWriter - .append(codeBlock) - .assignArrayStmt( - objArrayTerm, index, codeBlock.getReturnTerm()); + loopWriter.ifStmt( + "!" + methodCall(inputTerm, "isNullAt", index), + thenWriter -> + thenWriter + .append(codeBlock) + .assignArrayStmt( + objArrayTerm, + index, + codeBlock.getReturnTerm()), + elseWriter -> + elseWriter.throwStmt( + "new org.apache.flink.table.api.TableRuntimeException(\"Target is not nullable but a NULL was found.\")")); } }, context.getCodeGeneratorContext()) diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java index 378060a4d44519..4c32ce9b953dd7 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/ArrayToStringCastRule.java @@ -60,54 +60,53 @@ private ArrayToStringCastRule() { isNull$0 = _myInputIsNull; if (!isNull$0) { - builder$1.setLength(0); - builder$1.append("["); - for (int i$3 = 0; i$3 < _myInput.size(); i$3++) { - if (builder$1.length() > 10) { + int size$2 = _myInput.size(); + builder$0.setLength(0); + builder$0.append("["); + for (int i$3 = 0; i$3 < size$2; i$3++) { + if (builder$0.length() > 10) { break; } if (i$3 != 0) { - builder$1.append(", "); + builder$0.append(", "); } int element$4 = -1; boolean elementIsNull$5 = _myInput.isNullAt(i$3); if (!elementIsNull$5) { - element$4 = _myInput.getInt(i$3); - isNull$2 = false; - if (!isNull$2) { - result$3 = org.apache.flink.table.data.binary.BinaryStringData.fromString("" + element$4); - isNull$2 = result$3 == null; - } else { - result$3 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; - } - builder$1.append(result$3); + element$4 = _myInput.getInt(i$3); + isNull$2 = false; + if (!isNull$2) { + result$3 = org.apache.flink.table.data.binary.BinaryStringData.fromString("" + element$4); + isNull$2 = result$3 == null; } else { - builder$1.append("NULL"); + result$3 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; + } + builder$0.append(result$3); + } else { + builder$0.append("NULL"); } } - builder$1.append("]"); - java.lang.String resultString$2; - resultString$2 = builder$1.toString(); - if (builder$1.length() > 10) { - resultString$2 = builder$1.substring(0, java.lang.Math.min(builder$1.length(), 10)); + builder$0.append("]"); + java.lang.String resultString$1; + if (builder$0.length() > 10) { + resultString$1 = builder$0.substring(0, 10); } else { - if (resultString$2.length() < 10) { + resultString$1 = builder$0.toString(); + if (builder$0.length() < 10) { int padLength$6; - padLength$6 = 10 - resultString$2.length(); - java.lang.StringBuilder sbPadding$7; - sbPadding$7 = new java.lang.StringBuilder(); - for (int i$8 = 0; i$8 < padLength$6; i$8++) { - sbPadding$7.append(" "); - } - resultString$2 = resultString$2 + sbPadding$7.toString(); + padLength$6 = 10 - builder$0.length(); + resultString$1 = resultString$1 + " ".repeat(padLength$6); } } - result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$2); + result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$1); isNull$0 = result$1 == null; } else { result$1 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; } + returnTerm = result$1 + isNullTerm = isNull$0 + */ @Override protected String generateCodeBlockInternal( @@ -124,14 +123,16 @@ protected String generateCodeBlockInternal( className(StringBuilder.class), builderTerm, constructorCall(StringBuilder.class)); final String resultStringTerm = newName(codeGeneratorContext, "resultString"); + final String sizeTerm = newName(codeGeneratorContext, "size"); final int length = LogicalTypeChecks.getLength(targetLogicalType); CastRuleUtils.CodeWriter writer = new CastRuleUtils.CodeWriter() + .declStmt(int.class, sizeTerm, methodCall(inputTerm, "size")) .stmt(methodCall(builderTerm, "setLength", 0)) .stmt(methodCall(builderTerm, "append", strLiteral("["))) .forStmt( - methodCall(inputTerm, "size"), + sizeTerm, (indexTerm, loopBodyWriter) -> { String elementTerm = newName(codeGeneratorContext, "element"); String elementIsNullTerm = diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CharVarCharTrimPadCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CharVarCharTrimPadCastRule.java index b3d0b46b933e2b..1c60f26eac3290 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CharVarCharTrimPadCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CharVarCharTrimPadCastRule.java @@ -29,9 +29,9 @@ import org.apache.flink.table.types.logical.utils.LogicalTypeChecks; import static org.apache.flink.table.planner.codegen.CodeGenUtils.newName; -import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.constructorCall; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.methodCall; import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall; +import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.strLiteral; import static org.apache.flink.table.types.logical.VarCharType.STRING_TYPE; /** @@ -179,10 +179,6 @@ static String stringExceedsLength(String strTerm, int targetLength) { return methodCall(strTerm, "length") + " > " + targetLength; } - static String stringShouldPad(String strTerm, int targetLength) { - return methodCall(strTerm, "length") + " < " + targetLength; - } - static boolean couldTrim(int targetLength) { return targetLength < VarCharType.MAX_LENGTH; } @@ -199,8 +195,7 @@ static CastRuleUtils.CodeWriter padAndTrimStringIfNeeded( String resultStringTerm, String builderTerm, CodeGeneratorContext codeGeneratorContext) { - writer.declStmt(String.class, resultStringTerm) - .assignStmt(resultStringTerm, methodCall(builderTerm, "toString")); + writer.declStmt(String.class, resultStringTerm); // Trim and Pad if needed if (!legacyBehaviour && (couldTrim(length) || couldPad(targetType, length))) { @@ -209,23 +204,21 @@ static CastRuleUtils.CodeWriter padAndTrimStringIfNeeded( thenWriter -> thenWriter.assignStmt( resultStringTerm, - methodCall( - builderTerm, - "substring", - 0, - staticCall( - Math.class, - "min", - methodCall(builderTerm, "length"), - length))), - elseWriter -> - padStringIfNeeded( - elseWriter, - targetType, - legacyBehaviour, - length, - resultStringTerm, - codeGeneratorContext)); + methodCall(builderTerm, "substring", 0, length)), + elseWriter -> { + elseWriter.assignStmt( + resultStringTerm, methodCall(builderTerm, "toString")); + padStringIfNeeded( + elseWriter, + targetType, + legacyBehaviour, + length, + resultStringTerm, + methodCall(builderTerm, "length"), + codeGeneratorContext); + }); + } else { + writer.assignStmt(resultStringTerm, methodCall(builderTerm, "toString")); } return writer; } @@ -236,34 +229,24 @@ static void padStringIfNeeded( boolean legacyBehaviour, int length, String returnTerm, + String currentLengthTerm, CodeGeneratorContext codeGeneratorContext) { // Pad if needed if (!legacyBehaviour && couldPad(targetType, length)) { final String padLength = newName(codeGeneratorContext, "padLength"); - final String sbPadding = newName(codeGeneratorContext, "sbPadding"); writer.ifStmt( - stringShouldPad(returnTerm, length), + currentLengthTerm + " < " + length, thenWriter -> thenWriter .declStmt(int.class, padLength) - .assignStmt( - padLength, - length + " - " + methodCall(returnTerm, "length")) - .declStmt(StringBuilder.class, sbPadding) - .assignStmt(sbPadding, constructorCall(StringBuilder.class)) - .forStmt( - padLength, - (idx, loopWriter) -> - loopWriter.stmt( - methodCall( - sbPadding, "append", "\" \"")), - codeGeneratorContext) + .assignStmt(padLength, length + " - " + currentLengthTerm) .assignStmt( returnTerm, returnTerm + " + " - + methodCall(sbPadding, "toString"))); + + methodCall( + strLiteral(" "), "repeat", padLength))); } } } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java index a315fe2bbbbd3f..2e9819b4e9f75d 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapAndMultisetToStringCastRule.java @@ -73,65 +73,63 @@ private static boolean isMapOrMultiset(LogicalType input, LogicalType target) { isNull$0 = _myInputIsNull; if (!isNull$0) { - org.apache.flink.table.data.ArrayData keys$2 = _myInput.keyArray(); - org.apache.flink.table.data.ArrayData values$3 = _myInput.valueArray(); - builder$1.setLength(0); - builder$1.append("{"); - for (int i$5 = 0; i$5 < _myInput.size(); i$5++) { - if (builder$1.length() > 12) { + int size$4 = _myInput.size(); + org.apache.flink.table.data.ArrayData keys$1 = _myInput.keyArray(); + org.apache.flink.table.data.ArrayData values$2 = _myInput.valueArray(); + builder$0.setLength(0); + builder$0.append("{"); + for (int i$5 = 0; i$5 < size$4; i$5++) { + if (builder$0.length() > 12) { break; } if (i$5 != 0) { - builder$1.append(", "); + builder$0.append(", "); } org.apache.flink.table.data.binary.BinaryStringData key$6 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; - boolean keyIsNull$7 = keys$2.isNullAt(i$5); + boolean keyIsNull$7 = keys$1.isNullAt(i$5); int value$8 = -1; - boolean valueIsNull$9 = values$3.isNullAt(i$5); + boolean valueIsNull$9 = values$2.isNullAt(i$5); if (!keyIsNull$7) { - key$6 = ((org.apache.flink.table.data.binary.BinaryStringData) keys$2.getString(i$5)); - builder$1.append(key$6); + key$6 = ((org.apache.flink.table.data.binary.BinaryStringData) keys$1.getString(i$5)); + builder$0.append(key$6); } else { - builder$1.append("NULL"); + builder$0.append("NULL"); } - builder$1.append("="); + builder$0.append("="); if (!valueIsNull$9) { - value$8 = values$3.getInt(i$5); - isNull$2 = valueIsNull$9; + value$8 = values$2.getInt(i$5); + isNull$2 = false; if (!isNull$2) { - result$3 = org.apache.flink.table.data.binary.BinaryStringData.fromString("" + value$8); + result$3 = org.apache.flink.table.data.binary.BinaryStringData.fromString(org.apache.flink.table.utils.DateTimeUtils.formatIntervalYearMonth(value$8)); isNull$2 = result$3 == null; } else { result$3 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; } - builder$1.append(result$3); - } else { - builder$1.append("NULL"); + builder$0.append(result$3); + } else { + builder$0.append("NULL"); + } } - } - builder$1.append("}"); - java.lang.String resultString$4; - resultString$4 = builder$1.toString(); - if (builder$1.length() > 12) { - resultString$4 = builder$1.substring(0, java.lang.Math.min(builder$1.length(), 12)); - } else { - if (resultString$.length() < 12) { + builder$0.append("}"); + java.lang.String resultString$3; + if (builder$0.length() > 12) { + resultString$3 = builder$0.substring(0, 12); + } else { + resultString$3 = builder$0.toString(); + if (builder$0.length() < 12) { int padLength$10; - padLength$10 = 12 - resultString$.length(); - java.lang.StringBuilder sbPadding$11; - sbPadding$11 = new java.lang.StringBuilder(); - for (int i$12 = 0; i$12 < padLength$10; i$12++) { - sbPadding$11.append(" "); - } - resultString$4 = resultString$4 + sbPadding$11.toString(); + padLength$10 = 12 - builder$0.length(); + resultString$3 = resultString$3 + " ".repeat(padLength$10); } } - result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$4); + result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$3); isNull$0 = result$1 == null; } else { result$1 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; } + returnTerm = result$1 + isNullTerm = isNull$0 */ @Override protected String generateCodeBlockInternal( @@ -157,10 +155,12 @@ protected String generateCodeBlockInternal( final String valueArrayTerm = newName(codeGeneratorContext, "values"); final String resultStringTerm = newName(codeGeneratorContext, "resultString"); + final String sizeTerm = newName(codeGeneratorContext, "size"); final int length = LogicalTypeChecks.getLength(targetLogicalType); CastRuleUtils.CodeWriter writer = new CastRuleUtils.CodeWriter() + .declStmt(int.class, sizeTerm, methodCall(inputTerm, "size")) .declStmt(ArrayData.class, keyArrayTerm, methodCall(inputTerm, "keyArray")) .declStmt( ArrayData.class, @@ -169,7 +169,7 @@ protected String generateCodeBlockInternal( .stmt(methodCall(builderTerm, "setLength", 0)) .stmt(methodCall(builderTerm, "append", strLiteral("{"))) .forStmt( - methodCall(inputTerm, "size"), + sizeTerm, (indexTerm, loopBodyWriter) -> { String keyTerm = newName(codeGeneratorContext, "key"); String keyIsNullTerm = diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapToMapAndMultisetToMultisetCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapToMapAndMultisetToMultisetCastRule.java index e4a15d17634168..b0795760a5523e 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapToMapAndMultisetToMultisetCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/MapToMapAndMultisetToMultisetCastRule.java @@ -18,6 +18,7 @@ package org.apache.flink.table.planner.functions.casting; +import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.GenericMapData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.planner.codegen.CodeGeneratorContext; @@ -85,23 +86,27 @@ private static boolean isValidMapToMapOrMultisetToMultisetCasting( float result$2; isNull$0 = _myInputIsNull; if (!isNull$0) { - java.util.Map map$838 = new java.util.HashMap(); - for (int i$841 = 0; i$841 < _myInput.size(); i$841++) { - java.lang.Float key$839 = null; - java.lang.Integer value$840 = null; - if (!_myInput.keyArray().isNullAt(i$841)) { - result$2 = ((float)(_myInput.keyArray().getInt(i$841))); - key$839 = result$2; + int size$2 = _myInput.size(); + org.apache.flink.table.data.ArrayData keyArray$0 = _myInput.keyArray(); + org.apache.flink.table.data.ArrayData valueArray$1 = _myInput.valueArray(); + java.util.Map map$3 = new java.util.HashMap(size$2); + for (int i$6 = 0; i$6 < size$2; i$6++) { + java.lang.Float key$4 = null; + java.lang.Integer value$5 = null; + if (!keyArray$0.isNullAt(i$6)) { + result$2 = ((float)(keyArray$0.getInt(i$6))); + key$4 = result$2; } - value$840 = _myInput.valueArray().getInt(i$841); - map$838.put(key$839, value$840); + if (!valueArray$1.isNullAt(i$6)) { + value$5 = valueArray$1.getInt(i$6); + } + map$3.put(key$4, value$5); } - result$1 = new org.apache.flink.table.data.GenericMapData(map$838); + result$1 = new org.apache.flink.table.data.GenericMapData(map$3); isNull$0 = result$1 == null; } else { result$1 = null; } - return result$1; */ @Override @@ -132,23 +137,25 @@ protected String generateCodeBlockInternal( final String innerTargetKeyTypeTerm = boxedTypeTermForType(innerTargetKeyType); final String innerTargetValueTypeTerm = boxedTypeTermForType(innerTargetValueType); - final String keyArrayTerm = methodCall(inputTerm, "keyArray"); - final String valueArrayTerm = methodCall(inputTerm, "valueArray"); - final String size = methodCall(inputTerm, "size"); + final String keyArray = newName(codeGeneratorContext, "keyArray"); + final String valueArray = newName(codeGeneratorContext, "valueArray"); + final String size = newName(codeGeneratorContext, "size"); final String map = newName(codeGeneratorContext, "map"); final String key = newName(codeGeneratorContext, "key"); final String value = newName(codeGeneratorContext, "value"); return new CastRuleUtils.CodeWriter() - .declStmt(className(Map.class), map, constructorCall(HashMap.class)) + .declStmt(int.class, size, methodCall(inputTerm, "size")) + .declStmt(ArrayData.class, keyArray, methodCall(inputTerm, "keyArray")) + .declStmt(ArrayData.class, valueArray, methodCall(inputTerm, "valueArray")) + .declStmt(className(Map.class), map, constructorCall(HashMap.class, size)) .forStmt( size, (index, codeWriter) -> { final CastCodeBlock keyCodeBlock = CastRuleProvider.generateAlwaysNonNullCodeBlock( context, - rowFieldReadAccess( - index, keyArrayTerm, innerInputKeyType), + rowFieldReadAccess(index, keyArray, innerInputKeyType), innerInputKeyType, innerTargetKeyType); assert keyCodeBlock != null; @@ -157,7 +164,7 @@ protected String generateCodeBlockInternal( CastRuleProvider.generateAlwaysNonNullCodeBlock( context, rowFieldReadAccess( - index, valueArrayTerm, innerInputValueType), + index, valueArray, innerInputValueType), innerInputValueType, innerTargetValueType); assert valueCodeBlock != null; @@ -165,39 +172,53 @@ protected String generateCodeBlockInternal( codeWriter .declStmt(innerTargetKeyTypeTerm, key, null) .declStmt(innerTargetValueTypeTerm, value, null); - if (innerTargetKeyType.isNullable()) { - codeWriter.ifStmt( - "!" + methodCall(keyArrayTerm, "isNullAt", index), - thenWriter -> - thenWriter - .append(keyCodeBlock) - .assignStmt( - key, keyCodeBlock.getReturnTerm())); - } else { - codeWriter - .append(keyCodeBlock) - .assignStmt(key, keyCodeBlock.getReturnTerm()); - } - - if (inputLogicalType.is(LogicalTypeRoot.MAP) - && innerTargetValueType.isNullable()) { - codeWriter.ifStmt( - "!" + methodCall(valueArrayTerm, "isNullAt", index), - thenWriter -> - thenWriter - .append(valueCodeBlock) - .assignStmt( - value, - valueCodeBlock.getReturnTerm())); - } else { - codeWriter - .append(valueCodeBlock) - .assignStmt(value, valueCodeBlock.getReturnTerm()); - } + iterateOverElements( + index, + codeWriter, + keyArray, + keyCodeBlock, + key, + !innerTargetKeyType.isNullable()); + + iterateOverElements( + index, + codeWriter, + valueArray, + valueCodeBlock, + value, + !inputLogicalType.is(LogicalTypeRoot.MAP) + || !innerTargetValueType.isNullable()); codeWriter.stmt(methodCall(map, "put", key, value)); }, codeGeneratorContext) .assignStmt(returnVariable, constructorCall(GenericMapData.class, map)) .toString(); } + + private static void iterateOverElements( + String index, + CastRuleUtils.CodeWriter codeWriter, + String keyArray, + CastCodeBlock keyCodeBlock, + String key, + boolean throwIfNull) { + if (throwIfNull) { + codeWriter.ifStmt( + "!" + methodCall(keyArray, "isNullAt", index), + thenWriter -> + thenWriter + .append(keyCodeBlock) + .assignStmt(key, keyCodeBlock.getReturnTerm()), + elseWriter -> + elseWriter.throwStmt( + "new org.apache.flink.table.api.TableRuntimeException(\"Target is not nullable but a NULL was found.\")")); + } else { + codeWriter.ifStmt( + "!" + methodCall(keyArray, "isNullAt", index), + thenWriter -> + thenWriter + .append(keyCodeBlock) + .assignStmt(key, keyCodeBlock.getReturnTerm())); + } + } } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RawToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RawToStringCastRule.java index c80ddbb590100d..58e412fb381916 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RawToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RawToStringCastRule.java @@ -47,20 +47,16 @@ private RawToStringCastRule() { if (!isNull$0) { java.lang.Object deserializedObj$0 = _myInput.toObject(typeSerializer$2); if (deserializedObj$0 != null) { + java.lang.String deserializedObjString$2 = deserializedObj$0.toString(); java.lang.String resultString$1; - resultString$1 = deserializedObj$0.toString().toString(); - if (deserializedObj$0.toString().length() > 12) { - resultString$1 = deserializedObj$0.toString().substring(0, java.lang.Math.min(deserializedObj$0.toString().length(), 12)); + if (deserializedObjString$2.length() > 12) { + resultString$1 = deserializedObjString$2.substring(0, 12); } else { - if (resultString$1.length() < 12) { - int padLength$2; - padLength$2 = 12 - resultString$1.length(); - java.lang.StringBuilder sbPadding$3; - sbPadding$3 = new java.lang.StringBuilder(); - for (int i$4 = 0; i$4 < padLength$2; i$4++) { - sbPadding$3.append(" "); - } - resultString$1 = resultString$1 + sbPadding$3.toString(); + resultString$1 = deserializedObjString$2.toString(); + if (deserializedObjString$2.length() < 12) { + int padLength$3; + padLength$3 = 12 - deserializedObjString$2.length(); + resultString$1 = resultString$1 + " ".repeat(padLength$3); } } result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$1); @@ -72,6 +68,9 @@ private RawToStringCastRule() { result$1 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; } + returnTerm = result$1 + isNullTerm = isNull$0 + */ @Override protected String generateCodeBlockInternal( @@ -86,6 +85,8 @@ protected String generateCodeBlockInternal( CodeGenUtils.newName(codeGeneratorContext, "deserializedObj"); final String resultStringTerm = CodeGenUtils.newName(codeGeneratorContext, "resultString"); + final String deserializedObjStringTerm = + CodeGenUtils.newName(codeGeneratorContext, "deserializedObjString"); final int length = LogicalTypeChecks.getLength(targetLogicalType); return new CastRuleUtils.CodeWriter() @@ -97,12 +98,19 @@ protected String generateCodeBlockInternal( deserializedObjTerm + " != null", thenWriter -> CharVarCharTrimPadCastRule.padAndTrimStringIfNeeded( - thenWriter, + // toString() on a deserialized RAW value is + // user-defined and can be arbitrarily expensive, + // so it's computed once here. + thenWriter.declStmt( + String.class, + deserializedObjStringTerm, + methodCall( + deserializedObjTerm, "toString")), targetLogicalType, context.legacyBehaviour(), length, resultStringTerm, - methodCall(deserializedObjTerm, "toString"), + deserializedObjStringTerm, context.getCodeGeneratorContext()) .assignStmt( returnVariable, diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RowToStringCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RowToStringCastRule.java index 358fdfea7f2feb..da50c0e550cf15 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RowToStringCastRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/RowToStringCastRule.java @@ -57,55 +57,53 @@ private static boolean matches(LogicalType input, LogicalType target) { isNull$0 = _myInputIsNull; if (!isNull$0) { - builder$1.setLength(0); - builder$1.append("("); - int f0Value$3 = -1; - boolean f0IsNull$4 = _myInput.isNullAt(0); - if (!f0IsNull$4) { - f0Value$3 = _myInput.getInt(0); - isNull$2 = f0IsNull$4; + builder$0.setLength(0); + builder$0.append("("); + int f0Value$2 = -1; + boolean f0IsNull$3 = _myInput.isNullAt(0); + if (!f0IsNull$3) { + f0Value$2 = _myInput.getInt(0); + isNull$2 = false; if (!isNull$2) { - result$3 = org.apache.flink.table.data.binary.BinaryStringData.fromString("" + f0Value$3); + result$3 = org.apache.flink.table.data.binary.BinaryStringData.fromString("" + f0Value$2); isNull$2 = result$3 == null; } else { result$3 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; } - builder$1.append(result$3); + builder$0.append(result$3); } else { - builder$1.append("NULL"); + builder$0.append("NULL"); } - builder$1.append(", "); - org.apache.flink.table.data.binary.BinaryStringData f1Value$5 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; - boolean f1IsNull$6 = _myInput.isNullAt(1); - if (!f1IsNull$6) { - f1Value$5 = ((org.apache.flink.table.data.binary.BinaryStringData) _myInput.getString(1)); - builder$1.append(f1Value$5); + builder$0.append(", "); + org.apache.flink.table.data.binary.BinaryStringData f1Value$4 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; + boolean f1IsNull$5 = _myInput.isNullAt(1); + if (!f1IsNull$5) { + f1Value$4 = ((org.apache.flink.table.data.binary.BinaryStringData) _myInput.getString(1)); + builder$0.append(f1Value$4); } else { - builder$1.append("NULL"); + builder$0.append("NULL"); } - builder$1.append(")"); - java.lang.String resultString$2; - resultString$2 = builder$1.toString(); - if (builder$1.length() > 12) { - resultString$2 = builder$1.substring(0, java.lang.Math.min(builder$1.length(), 12)); + builder$0.append(")"); + java.lang.String resultString$1; + if (builder$0.length() > 12) { + resultString$1 = builder$0.substring(0, 12); } else { - if (resultString$2.length() < 12) { - int padLength$7; - padLength$7 = 12 - resultString$2.length(); - java.lang.StringBuilder sbPadding$8; - sbPadding$8 = new java.lang.StringBuilder(); - for (int i$9 = 0; i$9 < padLength$7; i$9++) { - sbPadding$8.append(" "); - } - resultString$2 = resultString$2 + sbPadding$8.toString(); + resultString$1 = builder$0.toString(); + if (builder$0.length() < 12) { + int padLength$6; + padLength$6 = 12 - builder$0.length(); + resultString$1 = resultString$1 + " ".repeat(padLength$6); } } - result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$2); + result$1 = org.apache.flink.table.data.binary.BinaryStringData.fromString(resultString$1); isNull$0 = result$1 == null; } else { result$1 = org.apache.flink.table.data.binary.BinaryStringData.EMPTY_UTF8; } + returnTerm = result$1 + isNullTerm = isNull$0 + */ @Override protected String generateCodeBlockInternal( diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/CapitalizeQueryHintsShuttle.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/CapitalizeQueryHintsShuttle.java index 006dab4b832a7f..6da4452010d380 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/CapitalizeQueryHintsShuttle.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/CapitalizeQueryHintsShuttle.java @@ -46,7 +46,8 @@ protected RelNode doVisit(RelNode node) { changed.set(true); if (JoinStrategy.isJoinStrategy(capitalHintName)) { - if (JoinStrategy.isLookupHint(hint.hintName)) { + if (JoinStrategy.isLookupHint(hint.hintName) + || JoinStrategy.isEarlyFireHint(hint.hintName)) { return RelHint.builder(capitalHintName) .hintOptions(hint.kvOptions) .inheritPath(hint.inheritPath) diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java index 5978f98017d91d..204c23a97d84a5 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java @@ -20,6 +20,7 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.api.config.EarlyFireJoinHintOptions; import org.apache.flink.table.api.config.LookupJoinHintOptions; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.planner.plan.rules.logical.WrapJsonAggFunctionArgumentsRule; @@ -36,6 +37,9 @@ import java.time.Duration; import java.util.Collections; import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; /** * A collection of Flink style {@link HintStrategy}s. @@ -135,6 +139,11 @@ public static HintStrategyTable createHintStrategyTable() { HintPredicates.JOIN, HintPredicates.AGGREGATE)) .optionChecker(STATE_TTL_NON_EMPTY_KV_OPTION_CHECKER) .build()) + .hintStrategy( + JoinStrategy.EARLY_FIRE.getJoinHintName(), + HintStrategy.builder(HintPredicates.JOIN) + .optionChecker(EARLY_FIRE_KV_OPTION_CHECKER) + .build()) .build(); } @@ -253,6 +262,55 @@ private static HintOptionChecker fixedSizeListOptionChecker(int size) { return true; }; + private static final HintOptionChecker EARLY_FIRE_KV_OPTION_CHECKER = + (earlyFireHint, litmus) -> { + litmus.check( + earlyFireHint.listOptions.size() == 0, + "Invalid list options in EARLY_FIRE hint, only support key-value options."); + + Configuration conf = Configuration.fromMap(earlyFireHint.kvOptions); + ImmutableSet requiredKeys = + EarlyFireJoinHintOptions.getRequiredOptions(); + litmus.check( + requiredKeys.stream().allMatch(conf::contains), + "Invalid EARLY_FIRE hint: incomplete required option(s): {}", + requiredKeys); + + ImmutableSet supportedKeys = + EarlyFireJoinHintOptions.getSupportedOptions(); + Set supportedKeyNames = + supportedKeys.stream().map(ConfigOption::key).collect(Collectors.toSet()); + Set unknownKeys = + earlyFireHint.kvOptions.keySet().stream() + .filter(key -> !supportedKeyNames.contains(key)) + .collect(Collectors.toCollection(TreeSet::new)); + litmus.check( + unknownKeys.isEmpty(), + "Unsupported EARLY_FIRE hint option(s) {}, supported options are {}.", + unknownKeys, + new TreeSet<>(supportedKeyNames)); + litmus.check( + earlyFireHint.kvOptions.size() <= supportedKeys.size(), + "Too many EARLY_FIRE hint options {} beyond max number of supported options {}", + earlyFireHint.kvOptions.size(), + supportedKeys.size()); + + try { + // try to validate all hint options by parsing them + supportedKeys.forEach(conf::get); + } catch (IllegalArgumentException e) { + litmus.fail("Invalid EARLY_FIRE hint options: {}", e.getMessage()); + } + + Duration delay = conf.get(EarlyFireJoinHintOptions.DELAY); + litmus.check( + null != delay && delay.toMillis() > 0, + "Invalid EARLY_FIRE hint option: {} value should be at least 1 millisecond but was {}", + EarlyFireJoinHintOptions.DELAY.key(), + delay); + return true; + }; + private static final HintOptionChecker STATE_TTL_NON_EMPTY_KV_OPTION_CHECKER = (ttlHint, litmus) -> { litmus.check( diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/JoinStrategy.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/JoinStrategy.java index 0ded47cb22230e..d9e3acff7a248b 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/JoinStrategy.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/JoinStrategy.java @@ -50,6 +50,12 @@ public enum JoinStrategy { /** Instructs the optimizer to use lookup join strategy. Only accept key-value hint options. */ LOOKUP("LOOKUP"), + /** + * Instructs an outer interval join to emit unmatched outer rows with null padding after a + * configurable delay. Only accept key-value hint options. + */ + EARLY_FIRE("EARLY_FIRE"), + /** * Instructs the optimizer to use multi-way join strategy for streaming queries. This hint * allows specifying multiple tables to be joined together in a single {@link @@ -89,6 +95,7 @@ public static boolean validOptions(String hintName, List options) { case NEST_LOOP: return options.size() > 0; case LOOKUP: + case EARLY_FIRE: return null == options || options.size() == 0; case MULTI_JOIN: return options.size() > 0; @@ -101,4 +108,10 @@ public static boolean isLookupHint(String hintName) { return isJoinStrategy(formalizedHintName) && JoinStrategy.valueOf(formalizedHintName) == LOOKUP; } + + public static boolean isEarlyFireHint(String hintName) { + String formalizedHintName = hintName.toUpperCase(Locale.ROOT); + return isJoinStrategy(formalizedHintName) + && JoinStrategy.valueOf(formalizedHintName) == EARLY_FIRE; + } } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlCreateConnectionConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlCreateConnectionConverter.java new file mode 100644 index 00000000000000..f1bb4efa9ea2ae --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlCreateConnectionConverter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.operations.converters; + +import org.apache.flink.sql.parser.ddl.connection.SqlCreateConnection; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.SensitiveConnection; +import org.apache.flink.table.catalog.UnresolvedIdentifier; +import org.apache.flink.table.operations.Operation; +import org.apache.flink.table.operations.ddl.CreateConnectionOperation; + +import java.util.Map; + +/** A converter for {@link SqlCreateConnection}. */ +public class SqlCreateConnectionConverter implements SqlNodeConverter { + + @Override + public Operation convertSqlNode( + SqlCreateConnection sqlCreateConnection, ConvertContext context) { + UnresolvedIdentifier unresolvedIdentifier = + UnresolvedIdentifier.of(sqlCreateConnection.getFullName()); + ObjectIdentifier identifier = + context.getCatalogManager().qualifyIdentifier(unresolvedIdentifier); + + Map options = sqlCreateConnection.getProperties(); + String comment = sqlCreateConnection.getComment(); + + SensitiveConnection sensitiveConnection = SensitiveConnection.of(options, comment); + + return new CreateConnectionOperation( + identifier, + sensitiveConnection, + sqlCreateConnection.isIfNotExists(), + sqlCreateConnection.isTemporary()); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlNodeConverters.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlNodeConverters.java index 7e03c2036641cd..ce8f41df992295 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlNodeConverters.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/SqlNodeConverters.java @@ -90,6 +90,7 @@ public class SqlNodeConverters { register(new SqlShowProcedureConverter()); registerCatalogConverters(); + registerConnectionConverters(); registerMaterializedTableConverters(); registerModelConverters(); registerTableConverters(); @@ -138,6 +139,10 @@ private static void registerCatalogConverters() { register(new SqlShowCreateCatalogConverter()); } + private static void registerConnectionConverters() { + register(new SqlCreateConnectionConverter()); + } + private static void registerMaterializedTableConverters() { register(new SqlAlterMaterializedTableAddDistributionConverter()); register(new SqlAlterMaterializedTableAddSchemaConverter()); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecIntervalJoin.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecIntervalJoin.java index 2ac0af781e0c66..20b676af78562e 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecIntervalJoin.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecIntervalJoin.java @@ -28,6 +28,7 @@ import org.apache.flink.streaming.api.transformations.TwoInputTransformation; import org.apache.flink.streaming.api.transformations.UnionTransformation; import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.config.EarlyFireJoinHintOptions; import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.table.data.RowData; import org.apache.flink.table.planner.delegation.PlannerBase; @@ -59,11 +60,14 @@ import org.apache.flink.shaded.guava33.com.google.common.collect.Lists; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.util.List; /** {@link StreamExecNode} for a time interval stream join. */ @@ -91,13 +95,27 @@ public class StreamExecIntervalJoin extends ExecNodeBase public static final String INTERVAL_JOIN_TRANSFORMATION = "interval-join"; public static final String FIELD_NAME_INTERVAL_JOIN_SPEC = "intervalJoinSpec"; + public static final String FIELD_NAME_EARLY_FIRE_DELAY = "earlyFireDelay"; + public static final String FIELD_NAME_EARLY_FIRE_TIME_MODE = "earlyFireTimeMode"; @JsonProperty(FIELD_NAME_INTERVAL_JOIN_SPEC) private final IntervalJoinSpec intervalJoinSpec; + @Nullable + @JsonProperty(FIELD_NAME_EARLY_FIRE_DELAY) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final Long earlyFireDelay; + + @Nullable + @JsonProperty(FIELD_NAME_EARLY_FIRE_TIME_MODE) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final EarlyFireJoinHintOptions.TimeMode earlyFireTimeMode; + public StreamExecIntervalJoin( ReadableConfig tableConfig, IntervalJoinSpec intervalJoinSpec, + @Nullable Long earlyFireDelay, + @Nullable EarlyFireJoinHintOptions.TimeMode earlyFireTimeMode, InputProperty leftInputProperty, InputProperty rightInputProperty, RowType outputType, @@ -107,6 +125,8 @@ public StreamExecIntervalJoin( ExecNodeContext.newContext(StreamExecIntervalJoin.class), ExecNodeContext.newPersistedConfig(StreamExecIntervalJoin.class, tableConfig), intervalJoinSpec, + earlyFireDelay, + earlyFireTimeMode, Lists.newArrayList(leftInputProperty, rightInputProperty), outputType, description); @@ -118,12 +138,17 @@ public StreamExecIntervalJoin( @JsonProperty(FIELD_NAME_TYPE) ExecNodeContext context, @JsonProperty(FIELD_NAME_CONFIGURATION) ReadableConfig persistedConfig, @JsonProperty(FIELD_NAME_INTERVAL_JOIN_SPEC) IntervalJoinSpec intervalJoinSpec, + @Nullable @JsonProperty(FIELD_NAME_EARLY_FIRE_DELAY) Long earlyFireDelay, + @Nullable @JsonProperty(FIELD_NAME_EARLY_FIRE_TIME_MODE) + EarlyFireJoinHintOptions.TimeMode earlyFireTimeMode, @JsonProperty(FIELD_NAME_INPUT_PROPERTIES) List inputProperties, @JsonProperty(FIELD_NAME_OUTPUT_TYPE) RowType outputType, @JsonProperty(FIELD_NAME_DESCRIPTION) String description) { super(id, context, persistedConfig, inputProperties, outputType, description); Preconditions.checkArgument(inputProperties.size() == 2); this.intervalJoinSpec = Preconditions.checkNotNull(intervalJoinSpec); + this.earlyFireDelay = earlyFireDelay; + this.earlyFireTimeMode = earlyFireTimeMode; } @Override diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/QueryHintsResolver.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/QueryHintsResolver.java index b288f111e30cc4..b153ef2b885514 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/QueryHintsResolver.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/optimize/QueryHintsResolver.java @@ -146,6 +146,13 @@ private List validateAndGetNewHints( updateInfoForOptionCheck(hint.hintName, rightName); newHints.add(hint); } + } else if (JoinStrategy.isEarlyFireHint(hint.hintName)) { + // EARLY_FIRE carries only key-value options and is not bound to a specific input + // side, so it is passed through unchanged once its options are validated by the + // hint option checker. + allHints.add(trimInheritPath(hint)); + validHints.add(trimInheritPath(hint)); + newHints.add(hint); } else if (JoinStrategy.isJoinStrategy(hint.hintName)) { allHints.add(trimInheritPath(hint)); // add options about this hint for finally checking diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java index f76154803a49ca..c4b1012df9729a 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java @@ -28,6 +28,7 @@ import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableFunctionScan; import org.apache.flink.table.planner.plan.utils.FlinkRexUtil; import org.apache.flink.table.planner.plan.utils.LateralSnapshotJoinUtil; +import org.apache.flink.table.planner.utils.ShortcutUtils; import org.apache.flink.table.types.inference.strategies.LateralSnapshotTypeStrategy; import org.apache.calcite.plan.RelOptRuleCall; @@ -123,22 +124,27 @@ public void onMatch(RelOptRuleCall call) { "Could not resolve the TABLE input of the SNAPSHOT scan on the build side of " + "a LATERAL SNAPSHOT join. This is a bug, please file an issue."); } - // The build-side input must declare exactly one watermark, otherwise the operator cannot - // determine when the LOAD phase is complete. - final long rowtimeCount = - rawTableInput.getRowType().getFieldList().stream() - .filter(f -> FlinkTypeFactory.isRowtimeIndicatorType(f.getType())) - .count(); - if (rowtimeCount == 0) { - throw new ValidationException( - "LATERAL SNAPSHOT requires a watermark on the build-side input."); - } - if (rowtimeCount > 1) { - throw new ValidationException( - String.format( - "The build-side input of a LATERAL SNAPSHOT join must not have more than one " - + "row-time attribute, but found %d.", - rowtimeCount)); + // The build-side row-time attribute drives the streaming operator's LOAD phase. In batch + // all input is bounded and the join degrades to a regular join (see + // BatchPhysicalLateralSnapshotJoinRule), so no watermark is required. + if (!ShortcutUtils.unwrapContext(join).isBatchMode()) { + // The build-side input must declare exactly one watermark, otherwise the operator + // cannot determine when the LOAD phase is complete. + final long rowtimeCount = + rawTableInput.getRowType().getFieldList().stream() + .filter(f -> FlinkTypeFactory.isRowtimeIndicatorType(f.getType())) + .count(); + if (rowtimeCount == 0) { + throw new ValidationException( + "LATERAL SNAPSHOT requires a watermark on the build-side input."); + } + if (rowtimeCount > 1) { + throw new ValidationException( + String.format( + "The build-side input of a LATERAL SNAPSHOT join must not have more than one " + + "row-time attribute, but found %d.", + rowtimeCount)); + } } // Replace the SNAPSHOT TableFunctionScan with its input, preserving any FlinkLogicalCalc diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalLateralSnapshotJoinRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalLateralSnapshotJoinRule.java new file mode 100644 index 00000000000000..9338ffb8feb86a --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalLateralSnapshotJoinRule.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.physical.batch; + +import org.apache.flink.table.planner.plan.nodes.FlinkConventions; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLateralSnapshotJoin; +import org.apache.flink.table.planner.plan.nodes.physical.batch.BatchPhysicalHashJoin; +import org.apache.flink.table.planner.plan.rules.logical.LogicalJoinToLateralSnapshotJoinRule; +import org.apache.flink.table.planner.plan.rules.physical.stream.StreamPhysicalLateralSnapshotJoinRule; +import org.apache.flink.table.planner.plan.trait.FlinkRelDistribution; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.util.ImmutableIntList; + +/** + * Converts a {@link FlinkLogicalLateralSnapshotJoin} (created by {@link + * LogicalJoinToLateralSnapshotJoinRule}) into a regular batch {@link BatchPhysicalHashJoin} for + * batch execution. + * + *

In batch all input is bounded and append-only (batch rejects or in the future materializes + * non-insert-only sources up front), so the processing-time {@code LATERAL SNAPSHOT} join + * degenerates to a regular join of the probe side against the (final) build side; the + * SNAPSHOT-specific arguments are irrelevant and dropped. This rule mirrors the streaming {@link + * StreamPhysicalLateralSnapshotJoinRule}, which converts the same logical node to a dedicated + * stream operator. + * + *

The SNAPSHOT input is the build (right) side of the LATERAL join and is the dimension-like + * side, expected to be smaller than the probe (left) side. The join is therefore emitted as a + * shuffle hash join that builds on the right input. + */ +public class BatchPhysicalLateralSnapshotJoinRule extends ConverterRule { + + public static final BatchPhysicalLateralSnapshotJoinRule INSTANCE = + new BatchPhysicalLateralSnapshotJoinRule( + Config.INSTANCE.withConversion( + FlinkLogicalLateralSnapshotJoin.class, + FlinkConventions.LOGICAL(), + FlinkConventions.BATCH_PHYSICAL(), + "BatchPhysicalLateralSnapshotJoinRule")); + + private BatchPhysicalLateralSnapshotJoinRule(Config config) { + super(config); + } + + @Override + public RelNode convert(RelNode rel) { + final FlinkLogicalLateralSnapshotJoin join = (FlinkLogicalLateralSnapshotJoin) rel; + final RelTraitSet providedTraitSet = + rel.getTraitSet().replace(FlinkConventions.BATCH_PHYSICAL()); + + // Both inputs are hash-partitioned on their join keys (shuffle hash join). + final JoinInfo joinInfo = join.analyzeCondition(); + final RelNode newLeft = convertInput(join.getLeft(), joinInfo.leftKeys); + final RelNode newRight = convertInput(join.getRight(), joinInfo.rightKeys); + + return new BatchPhysicalHashJoin( + join.getCluster(), + providedTraitSet, + newLeft, + newRight, + join.getCondition(), + join.getJoinType(), + // leftIsBuild = false: build the right (SNAPSHOT) side, the smaller dimension side. + false, + // isBroadcast = false: shuffle hash join. + false, + // tryDistinctBuildRow = false: only relevant for semi/anti joins. + false, + // withJobStrategyHint = false: not driven by a user join hint. + false); + } + + /** + * Converts a join input to the batch-physical convention and requires it to be hash-partitioned + * on the given join {@code keys} (or a singleton distribution if there are none). + */ + private static RelNode convertInput(RelNode input, ImmutableIntList keys) { + final FlinkRelDistribution distribution = + keys.isEmpty() + ? FlinkRelDistribution.SINGLETON() + : FlinkRelDistribution.hash(keys.toIntArray(), true); + final RelTraitSet requiredTraitSet = + input.getTraitSet() + .replace(FlinkConventions.BATCH_PHYSICAL()) + .replace(distribution); + return RelOptRule.convert(input, requiredTraitSet); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalIntervalJoinRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalIntervalJoinRule.java index 6354f04de07485..d7cbad27d0c5aa 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalIntervalJoinRule.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalIntervalJoinRule.java @@ -19,9 +19,13 @@ package org.apache.flink.table.planner.plan.rules.physical.stream; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.Configuration; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.config.EarlyFireJoinHintOptions; +import org.apache.flink.table.api.config.EarlyFireJoinHintOptions.TimeMode; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; +import org.apache.flink.table.planner.hint.JoinStrategy; import org.apache.flink.table.planner.plan.nodes.FlinkRelNode; import org.apache.flink.table.planner.plan.nodes.exec.spec.IntervalJoinSpec; import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin; @@ -32,11 +36,16 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.immutables.value.Value; +import javax.annotation.Nullable; + +import java.time.Duration; import java.util.Collection; +import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; @@ -133,6 +142,8 @@ public FlinkRelNode transform( RelTraitSet providedTraitSet) { Tuple2, Option> tuple2 = extractWindowBounds(join); + boolean isEventTime = tuple2.f0.get().isEventTime(); + EarlyFire earlyFire = extractEarlyFire(join.getHints(), isEventTime); return new StreamPhysicalIntervalJoin( join.getCluster(), providedTraitSet, @@ -141,7 +152,53 @@ public FlinkRelNode transform( join.getJoinType(), join.getCondition(), tuple2.f1.getOrElse(() -> join.getCluster().getRexBuilder().makeLiteral(true)), - tuple2.f0.get()); + tuple2.f0.get(), + earlyFire.delay, + earlyFire.timeMode); + } + + private static EarlyFire extractEarlyFire(List hints, boolean isEventTime) { + RelHint earlyFireHint = null; + for (RelHint hint : hints) { + if (JoinStrategy.isEarlyFireHint(hint.hintName)) { + earlyFireHint = hint; + break; + } + } + if (earlyFireHint == null) { + return new EarlyFire(null, null); + } + + Configuration conf = Configuration.fromMap(earlyFireHint.kvOptions); + Duration delay = conf.get(EarlyFireJoinHintOptions.DELAY); + TimeMode timeMode = conf.get(EarlyFireJoinHintOptions.TIME_MODE); + if (timeMode == null) { + timeMode = isEventTime ? TimeMode.ROWTIME : TimeMode.PROCTIME; + } + + if (!isEventTime && timeMode == TimeMode.ROWTIME) { + throw new ValidationException( + "EARLY_FIRE hint requested row-time triggering on a processing-time interval" + + " join. Row-time triggering requires a row-time interval join."); + } + if (isEventTime && timeMode == TimeMode.PROCTIME) { + // Processing-time triggering on an event-time interval join is not supported. + throw new TableException( + "EARLY_FIRE hint requested processing-time triggering on a row-time interval" + + " join, which is not yet supported."); + } + + return new EarlyFire(delay == null ? null : delay.toMillis(), timeMode); + } + + private static final class EarlyFire { + @Nullable private final Long delay; + @Nullable private final TimeMode timeMode; + + EarlyFire(@Nullable Long delay, @Nullable TimeMode timeMode) { + this.delay = delay; + this.timeMode = timeMode; + } } /** Configuration for {@link StreamPhysicalIntervalJoinRule}. */ diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java index 95bca1a1384ec6..aa2f8a22cc9f1d 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java @@ -22,8 +22,12 @@ import org.apache.flink.table.functions.BuiltInFunctionDefinition; import org.apache.flink.table.functions.BuiltInFunctionDefinitions; import org.apache.flink.table.functions.FunctionDefinition; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.functions.bridging.BridgingSqlFunction; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.TimestampType; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.type.RelDataType; @@ -81,12 +85,21 @@ public static RelDataType deriveRowType( RelDataType rightType, JoinRelType joinType, List systemFieldList) { + final FlinkTypeFactory flinkTypeFactory = (FlinkTypeFactory) typeFactory; final RelDataTypeFactory.Builder materializedRight = typeFactory.builder(); for (RelDataTypeField field : rightType.getFieldList()) { - final RelDataType fieldType = - field.getType() instanceof TimeIndicatorRelDataType - ? ((TimeIndicatorRelDataType) field.getType()).getOriginalType() - : field.getType(); + final RelDataType fieldType; + if (field.getType() instanceof TimeIndicatorRelDataType) { + // Materialize the build-side time attribute to a regular timestamp, following the + // same convention as RelTimeIndicatorConverter + final LogicalType materialized = + FlinkTypeFactory.isTimestampLtzIndicatorType(field.getType()) + ? new LocalZonedTimestampType(field.getType().isNullable(), 3) + : new TimestampType(field.getType().isNullable(), 3); + fieldType = flinkTypeFactory.createFieldTypeFromLogicalType(materialized); + } else { + fieldType = field.getType(); + } materializedRight.add(field.getName(), fieldType); } return SqlValidatorUtil.deriveJoinRowType( diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalIntervalJoin.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalIntervalJoin.scala index 4916d653b2e236..d3401783989d9f 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalIntervalJoin.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalIntervalJoin.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.planner.plan.nodes.physical.stream import org.apache.flink.table.api.TableException +import org.apache.flink.table.api.config.EarlyFireJoinHintOptions.TimeMode import org.apache.flink.table.planner.calcite.FlinkTypeFactory import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty} import org.apache.flink.table.planner.plan.nodes.exec.spec.IntervalJoinSpec @@ -45,7 +46,9 @@ class StreamPhysicalIntervalJoin( val originalCondition: RexNode, // remaining join condition contains all of join condition except window bounds remainingCondition: RexNode, - windowBounds: WindowBounds) + windowBounds: WindowBounds, + earlyFireDelay: java.lang.Long, + earlyFireTimeMode: TimeMode) extends CommonPhysicalJoin(cluster, traitSet, leftRel, rightRel, remainingCondition, joinType) with StreamPhysicalRel { @@ -76,7 +79,9 @@ class StreamPhysicalIntervalJoin( joinType, originalCondition, conditionExpr, - windowBounds) + windowBounds, + earlyFireDelay, + earlyFireTimeMode) } override def explainTerms(pw: RelWriter): RelWriter = { @@ -98,12 +103,16 @@ class StreamPhysicalIntervalJoin( preferExpressionFormat(pw), pw.getDetailLevel)) .item("select", getRowType.getFieldNames.mkString(", ")) + .itemIf("earlyFireDelay", earlyFireDelay, earlyFireDelay != null) + .itemIf("earlyFireTimeMode", earlyFireTimeMode, earlyFireTimeMode != null) } override def translateToExecNode(): ExecNode[_] = { new StreamExecIntervalJoin( unwrapTableConfig(this), new IntervalJoinSpec(joinSpec, windowBounds), + earlyFireDelay, + earlyFireTimeMode, InputProperty.DEFAULT, InputProperty.DEFAULT, FlinkTypeFactory.toLogicalRowType(getRowType), diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala index 3fb057bc08d3eb..26553f24ddcd04 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala @@ -18,7 +18,6 @@ package org.apache.flink.table.planner.plan.rules import org.apache.flink.table.planner.plan.nodes.logical._ -import org.apache.flink.table.planner.plan.rules.FlinkStreamRuleSets.SIMPLIFY_COALESCE_RULES import org.apache.flink.table.planner.plan.rules.logical._ import org.apache.flink.table.planner.plan.rules.physical.FlinkExpandConversionRule import org.apache.flink.table.planner.plan.rules.physical.batch._ @@ -419,7 +418,13 @@ object FlinkBatchRuleSets { // Avoid having async calls in multiple projections in a single calc. AsyncCalcSplitRule.ONE_PER_CALC_SPLIT, // remove output of rank number when it is not used by successor calc - RedundantRankNumberColumnRemoveRule.INSTANCE + RedundantRankNumberColumnRemoveRule.INSTANCE, + // Rewrites a join over a LATERAL SNAPSHOT table function call into a dedicated + // FlinkLogicalLateralSnapshotJoin + LogicalJoinToLateralSnapshotJoinRule.INSTANCE, + // Rejects SNAPSHOT scans that survived the rewrite above, i.e. SNAPSHOT calls used outside a + // LATERAL context. Must run after LogicalJoinToLateralSnapshotJoinRule. + ForbidSnapshotOutsideLateralRule.INSTANCE ) /** RuleSet to do physical optimize for batch */ @@ -460,6 +465,9 @@ object FlinkBatchRuleSets { BatchPhysicalPythonWindowAggregateRule.INSTANCE, // window tvf BatchPhysicalWindowTableFunctionRule.INSTANCE, + // Converts a LATERAL SNAPSHOT join into a (shuffle hash) batch join with the SNAPSHOT input as + // build side + BatchPhysicalLateralSnapshotJoinRule.INSTANCE, // join BatchPhysicalHashJoinRule.INSTANCE, BatchPhysicalSortMergeJoinRule.INSTANCE, diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/catalog/BrokenCurrentCatalogTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/catalog/BrokenCurrentCatalogTest.java new file mode 100644 index 00000000000000..8641f761b2dede --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/catalog/BrokenCurrentCatalogTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.catalog; + +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.catalog.GenericInMemoryCatalog; +import org.apache.flink.table.catalog.exceptions.CatalogException; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests that a fully-qualified query against a healthy catalog can still be resolved even when the + * current catalog is unreachable. + */ +class BrokenCurrentCatalogTest { + + @Test + void testFullyQualifiedQueryWhileCurrentCatalogIsBroken() throws Exception { + TableEnvironment tEnv = + TableEnvironment.create( + EnvironmentSettings.newInstance().inStreamingMode().build()); + + tEnv.registerCatalog("healthy", new GenericInMemoryCatalog("healthy", "default")); + tEnv.useCatalog("healthy"); + tEnv.executeSql( + "CREATE VIEW `healthy`.`default`.`v` AS SELECT * FROM (VALUES (1), (2), (3)) AS t(id)"); + + tEnv.registerCatalog("broken", new UnreachableCatalog("broken")); + tEnv.useCatalog("broken"); + + Table table = tEnv.sqlQuery("SELECT * FROM `healthy`.`default`.`v`"); + + assertThat(table.getResolvedSchema().getColumnNames()).containsExactly("id"); + } + + private static class UnreachableCatalog extends GenericInMemoryCatalog { + UnreachableCatalog(String name) { + super(name, "default"); + } + + @Override + public boolean databaseExists(String databaseName) { + throw new CatalogException( + "Failed to connect to database '" + + databaseName + + "' of catalog '" + + getName() + + "'."); + } + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java index df439bd2c9e7f3..650d55740acc65 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java @@ -124,6 +124,65 @@ public final class TestValuesRuntimeFunctions { private static final Map>>> localRawResultsObservers = new HashMap<>(); + // [table_name, cumulative number of rows emitted by all subtasks of the source] + private static final Map sourceEmittedCounts = new HashMap<>(); + // [table_name, [pending emission barriers]] + private static final Map> sourceEmissionBarriers = + new HashMap<>(); + + /** A pending {@link #awaitSourceEmitted} request, completed once its target is reached. */ + private static final class SourceEmissionBarrier { + private final int target; + private final CompletableFuture future; + + private SourceEmissionBarrier(int target, CompletableFuture future) { + this.target = target; + this.future = future; + } + } + + /** + * Records that a source emitted a row and completes any barrier whose target row count has been + * reached. Called by the source runtime after every emitted element. + */ + static void notifySourceEmitted(String tableName) { + final List> toComplete = new ArrayList<>(); + synchronized (LOCK) { + final int count = sourceEmittedCounts.merge(tableName, 1, Integer::sum); + final List barriers = sourceEmissionBarriers.get(tableName); + if (barriers != null) { + barriers.removeIf( + barrier -> { + if (count >= barrier.target) { + toComplete.add(barrier.future); + return true; + } + return false; + }); + } + } + // Complete outside the lock; the awaiting thread may re-enter this class. + toComplete.forEach(future -> future.complete(null)); + } + + /** + * Returns a future that completes once source {@code tableName} has emitted at least {@code + * targetCount} rows (cumulative across all subtasks). + */ + static CompletableFuture awaitSourceEmitted(String tableName, int targetCount) { + final CompletableFuture future = new CompletableFuture<>(); + synchronized (LOCK) { + if (sourceEmittedCounts.getOrDefault(tableName, 0) >= targetCount) { + future.complete(null); + } else { + sourceEmissionBarriers + .computeIfAbsent(tableName, n -> new ArrayList<>()) + .add(new SourceEmissionBarrier(targetCount, future)); + } + } + return future; + } + static List getRawResultsAsStrings(String tableName) { return getRawResults(tableName).stream() .map(TestValuesRuntimeFunctions::rowToString) @@ -205,6 +264,8 @@ static void clearResults() { globalRetractResult.clear(); watermarkHistory.clear(); localRawResultsObservers.clear(); + sourceEmittedCounts.clear(); + sourceEmissionBarriers.clear(); } } @@ -257,15 +318,24 @@ public static class FromElementSourceFunctionWithWatermark private final TerminatingLogic terminating; + /** Sleep for {@link #sleepTimeMillis} after emitting every {@code sleepAfterElements}. */ + private final int sleepAfterElements; + + private final long sleepTimeMillis; + public FromElementSourceFunctionWithWatermark( String tableName, TypeSerializer serializer, Iterable elements, WatermarkStrategy watermarkStrategy, - TerminatingLogic terminating) + TerminatingLogic terminating, + int sleepAfterElements, + long sleepTimeMillis) throws IOException { this.tableName = tableName; this.terminating = terminating; + this.sleepAfterElements = sleepAfterElements; + this.sleepTimeMillis = sleepTimeMillis; ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos); @@ -325,6 +395,14 @@ public RelativeClock getInputActivityClock() { generator.onEvent(next, Long.MIN_VALUE, output); generator.onPeriodicEmit(output); } + notifySourceEmitted(tableName); + + // If enabled, throttle emission of values + if (sleepAfterElements > 0 + && sleepTimeMillis > 0 + && numElementsEmitted % sleepAfterElements == 0) { + Thread.sleep(sleepTimeMillis); + } } if (terminating == TerminatingLogic.INFINITE) { diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java index 3387f522232e44..7049fc1c8c17bc 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java @@ -158,6 +158,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Function; @@ -309,6 +310,16 @@ public static void registerLocalRawResultsObserver( TestValuesRuntimeFunctions.registerLocalRawResultsObserver(tableName, observer); } + /** + * Returns a future that completes once source {@code tableName} has emitted at least {@code + * targetCount} rows (cumulative across all subtasks). Useful for triggering a savepoint at a + * controlled point when the operator under test produces no output yet. For now only wired for + * the watermark-push-down {@code NewSource} runtime used by restore tests. + */ + public static CompletableFuture awaitSourceEmitted(String tableName, int targetCount) { + return TestValuesRuntimeFunctions.awaitSourceEmitted(tableName, targetCount); + } + public static List getWatermarkOutput(String tableName) { return TestValuesRuntimeFunctions.getWatermarks(tableName); } @@ -716,7 +727,9 @@ public DynamicTableSource createDynamicTableSource(Context context) { partitions, readableMetadata, null, - enableAggregatePushDown); + enableAggregatePushDown, + sleepAfterElements, + sleepTimeMillis); source.setEnableMetadataFilterPushDown(enableMetadataFilterPushDown); return source; } else { @@ -1750,6 +1763,8 @@ private static class TestValuesScanTableSourceWithWatermarkPushDown extends TestValuesScanTableSource implements SupportsWatermarkPushDown, SupportsSourceWatermark { private final String tableName; + private final int sleepAfterElements; + private final long sleepTimeMillis; private WatermarkStrategy watermarkStrategy = WatermarkStrategy.noWatermarks(); @@ -1771,7 +1786,9 @@ private TestValuesScanTableSourceWithWatermarkPushDown( List> allPartitions, Map readableMetadata, @Nullable int[] projectedMetadataFields, - boolean enableAggregatePushDown) { + boolean enableAggregatePushDown, + int sleepAfterElements, + long sleepTimeMillis) { super( producedDataType, changelogMode, @@ -1792,6 +1809,8 @@ private TestValuesScanTableSourceWithWatermarkPushDown( projectedMetadataFields, enableAggregatePushDown); this.tableName = tableName; + this.sleepAfterElements = sleepAfterElements; + this.sleepTimeMillis = sleepTimeMillis; } @Override @@ -1817,7 +1836,13 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderCon try { return SourceFunctionProvider.of( new TestValuesRuntimeFunctions.FromElementSourceFunctionWithWatermark( - tableName, serializer, values, watermarkStrategy, terminating), + tableName, + serializer, + values, + watermarkStrategy, + terminating, + sleepAfterElements, + sleepTimeMillis), false); } catch (IOException e) { throw new TableException("Fail to init source function", e); @@ -1845,7 +1870,9 @@ public DynamicTableSource copy() { allPartitions, readableMetadata, projectedMetadataFields, - enableAggregatePushDown); + enableAggregatePushDown, + sleepAfterElements, + sleepTimeMillis); newSource.watermarkStrategy = watermarkStrategy; newSource.setEnableMetadataFilterPushDown(enableMetadataFilterPushDown); return newSource; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java index 40defdf485df63..fb18458fec6006 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java @@ -87,6 +87,7 @@ Stream getTestSetSpecs() { testCases.addAll(isJsonSpec()); testCases.addAll(jsonQuerySpec()); testCases.addAll(jsonStringSpec()); + testCases.addAll(parseJsonSpec()); testCases.addAll(jsonObjectSpec()); testCases.addAll(jsonSpec()); testCases.addAll(jsonArraySpec()); @@ -760,6 +761,45 @@ private static List jsonStringSpec() { STRING().notNull())); } + private static List parseJsonSpec() { + // The bulk of parsing behavior is covered by BinaryVariantInternalBuilderTest. + return List.of( + TestSetSpec.forFunction(BuiltInFunctionDefinitions.PARSE_JSON) + .onFieldsWithData("{\"a\":1,\"b\":[2,3]}", "1e400") + .andDataTypes(STRING().notNull(), STRING().notNull()) + .testResult( + jsonString(call("PARSE_JSON", $("f0"))), + "JSON_STRING(PARSE_JSON(f0))", + "{\"a\":1,\"b\":[2,3]}", + STRING().notNull()) + .testResult( + jsonString(call("PARSE_JSON", nullOf(STRING()))), + "JSON_STRING(PARSE_JSON(CAST(NULL AS STRING)))", + null, + STRING().nullable()) + .testSqlRuntimeError( + "PARSE_JSON(f1)", + TableRuntimeException.class, + "Failed to parse json string") + .testTableApiRuntimeError( + call("PARSE_JSON", $("f1")), + TableRuntimeException.class, + "Failed to parse json string"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.TRY_PARSE_JSON) + .onFieldsWithData("{\"a\":1}", "1e400") + .andDataTypes(STRING().notNull(), STRING().notNull()) + .testResult( + jsonString(call("TRY_PARSE_JSON", $("f0"))), + "JSON_STRING(TRY_PARSE_JSON(f0))", + "{\"a\":1}", + STRING()) + .testResult( + jsonString(call("TRY_PARSE_JSON", $("f1"))), + "JSON_STRING(TRY_PARSE_JSON(f1))", + null, + STRING())); + } + private static List jsonSpec() { return List.of( TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_OBJECT) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java index 8459a804163ead..aca973fa3ca6c7 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/casting/CastRulesTest.java @@ -1370,7 +1370,7 @@ Stream testCases() { new GenericArrayData(new Integer[] {1, 2, null}), new GenericArrayData(new Integer[] {3}) }), - NullPointerException.class) + TableRuntimeException.class) .fromCase( ARRAY(ARRAY(INT().nullable())), new GenericArrayData( @@ -1430,12 +1430,12 @@ Stream testCases() { .fail( MAP(INT().nullable(), DOUBLE().nullable()), mapData(entry(null, 1d)), - NullPointerException.class), + TableRuntimeException.class), CastTestSpecBuilder.testCastTo(MAP(STRING().notNull(), STRING().notNull())) .fail( MAP(INT().nullable(), DOUBLE().nullable()), mapData(entry(123, null)), - NullPointerException.class), + TableRuntimeException.class), CastTestSpecBuilder.testCastTo(MULTISET(DOUBLE().notNull())) .fromCase( MULTISET(INT().nullable()), @@ -1455,7 +1455,7 @@ Stream testCases() { .fail( MULTISET(INT().nullable()), mapData(entry(null, 1)), - NullPointerException.class), + TableRuntimeException.class), CastTestSpecBuilder.testCastTo( ROW(BIGINT().notNull(), BIGINT(), STRING(), ARRAY(STRING()))) .fromCase( diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlConnectionOperationConverterTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlConnectionOperationConverterTest.java new file mode 100644 index 00000000000000..a410c95ec0b659 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/operations/SqlConnectionOperationConverterTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.operations; + +import org.apache.flink.sql.parser.error.SqlValidateException; +import org.apache.flink.table.api.SqlParserException; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.SensitiveConnection; +import org.apache.flink.table.operations.Operation; +import org.apache.flink.table.operations.ddl.CreateConnectionOperation; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Test for converting connection statements to operations. */ +class SqlConnectionOperationConverterTest extends SqlNodeToOperationConversionTestBase { + + @Test + void testCreateConnection() { + Operation operation = parse("CREATE CONNECTION my_conn WITH ('k' = 'v')"); + assertThat(operation).isInstanceOf(CreateConnectionOperation.class); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + + assertThat(op.getConnectionIdentifier()) + .isEqualTo(ObjectIdentifier.of("builtin", "default", "my_conn")); + assertThat(op.getSensitiveConnection().getOptions()).isEqualTo(Map.of("k", "v")); + assertThat(op.getSensitiveConnection().getComment()).isNull(); + assertThat(op.isIgnoreIfExists()).isFalse(); + assertThat(op.isTemporary()).isFalse(); + } + + @Test + void testCreateConnectionIfNotExists() { + Operation operation = parse("CREATE CONNECTION IF NOT EXISTS my_conn WITH ('k' = 'v')"); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + assertThat(op.isIgnoreIfExists()).isTrue(); + assertThat(op.isTemporary()).isFalse(); + } + + @Test + void testCreateTemporaryConnection() { + Operation operation = parse("CREATE TEMPORARY CONNECTION my_conn WITH ('k' = 'v')"); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + assertThat(op.isTemporary()).isTrue(); + assertThat(op.isIgnoreIfExists()).isFalse(); + } + + @Test + void testCreateTemporarySystemConnection() { + Operation operation = parse("CREATE TEMPORARY SYSTEM CONNECTION my_conn WITH ('k' = 'v')"); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + assertThat(op.isTemporary()).isTrue(); + } + + @Test + void testCreateConnectionWithComment() { + Operation operation = + parse("CREATE CONNECTION my_conn COMMENT 'hi there' WITH ('k' = 'v')"); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + SensitiveConnection conn = op.getSensitiveConnection(); + assertThat(conn.getComment()).isEqualTo("hi there"); + } + + @Test + void testCreateConnectionWithFullyQualifiedName() { + Operation operation = parse("CREATE CONNECTION cat1.db1.my_conn WITH ('k' = 'v')"); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + assertThat(op.getConnectionIdentifier()) + .isEqualTo(ObjectIdentifier.of("cat1", "db1", "my_conn")); + } + + @Test + void testCreateConnectionOptions() { + Operation operation = + parse("CREATE CONNECTION my_conn WITH ('k1' = 'v1', 'k2' = 'v2', 'k3' = 'v3')"); + CreateConnectionOperation op = (CreateConnectionOperation) operation; + assertThat(op.getSensitiveConnection().getOptions()) + .isEqualTo(Map.of("k1", "v1", "k2", "v2", "k3", "v3")); + } + + @Test + void testAsSummaryStringMasksOptionValues() { + Operation operation = + parse( + "CREATE CONNECTION my_conn WITH ('user' = 'alice', 'password' = 'super-secret')"); + String summary = operation.asSummaryString(); + assertThat(summary).contains("user").contains("password").contains("****"); + assertThat(summary).doesNotContain("alice").doesNotContain("super-secret"); + } + + @Test + void testCreateSystemConnectionWithoutTemporaryRejected() { + assertThatThrownBy(() -> parse("CREATE SYSTEM CONNECTION my_conn WITH ('k' = 'v')")) + .isInstanceOf(SqlParserException.class) + .hasMessageContaining("CREATE SYSTEM CONNECTION is not supported"); + } + + @Test + void testCreateConnectionWithEmptyOptionsRejected() { + assertThatThrownBy(() -> parse("CREATE CONNECTION my_conn WITH ()")) + .isInstanceOf(SqlValidateException.class) + .hasMessageContaining("Connection property list can not be empty."); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/batch/sql/join/LateralSnapshotJoinTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/batch/sql/join/LateralSnapshotJoinTest.java new file mode 100644 index 00000000000000..50416c1560f3e3 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/batch/sql/join/LateralSnapshotJoinTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.batch.sql.join; + +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.planner.plan.rules.physical.batch.BatchPhysicalLateralSnapshotJoinRule; +import org.apache.flink.table.planner.utils.BatchTableTestUtil; +import org.apache.flink.table.planner.utils.TableTestBase; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Plan tests for {@code LATERAL SNAPSHOT} joins in batch mode. + * + *

In batch all input is bounded and append-only, so the processing-time {@code LATERAL SNAPSHOT} + * join degenerates to a regular join of the probe side against the (final) build side. {@link + * BatchPhysicalLateralSnapshotJoinRule} performs this translation and the SNAPSHOT-specific + * arguments are dropped. + */ +public class LateralSnapshotJoinTest extends TableTestBase { + + private BatchTableTestUtil util; + + @BeforeEach + void setup() { + util = batchTestUtil(TableConfig.getDefault()); + + util.tableEnv() + .executeSql( + "CREATE TABLE probe (" + + " pk STRING," + + " pv INT," + + " pts TIMESTAMP(3)," + + " WATERMARK FOR pts AS pts" + + ") WITH ('connector' = 'values', 'bounded' = 'true')"); + + util.tableEnv() + .executeSql( + "CREATE TABLE b (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " WATERMARK FOR bts AS bts" + + ") WITH ('connector' = 'values', 'bounded' = 'true')"); + } + + // ------------------------------------------------------------------------------------------ + // Translation to a regular join + // ------------------------------------------------------------------------------------------ + + @Test + void testInnerJoin() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testLeftJoin() { + util.verifyRelPlan( + "SELECT * FROM probe LEFT JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testInnerJoinWithCompositeKeys() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk AND probe.pv = s.bv"); + } + + @Test + void testInnerJoinWithNonEquiCondition() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk AND probe.pv > s.bv"); + } + + @Test + void testInnerJoinWithoutBuildTimeColumn() { + util.verifyRelPlan( + "SELECT probe.pk, probe.pv, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testBuildSideWithProctime() { + // A PROCTIME() build column is a time-attribute indicator in batch; the rule materializes + // it (into a PROCTIME_MATERIALIZE call) when degrading to a regular join. + util.tableEnv() + .executeSql( + "CREATE TABLE b_proctime (" + + " bk STRING," + + " bv INT," + + " pt AS PROCTIME()" + + ") WITH ('connector' = 'values', 'bounded' = 'true')"); + util.verifyRelPlan( + "SELECT probe.pk, s.bk, s.bv, s.pt FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b_proctime)) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testBuildSideWithoutWatermark() { + // A watermark on the build side is required in streaming (drives the LOAD phase) but not in + // batch, where the build side is already final. + util.tableEnv() + .executeSql( + "CREATE TABLE b_no_wm (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)" + + ") WITH ('connector' = 'values', 'bounded' = 'true')"); + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(input => TABLE b_no_wm)) AS s " + + "ON probe.pk = s.bk"); + } + + // ------------------------------------------------------------------------------------------ + // Validation: rejection paths + // ------------------------------------------------------------------------------------------ + + @Test + void testRejectMissingEqualityPredicate() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pv > s.bv"; + assertThatThrownBy(() -> util.verifyExecPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "LATERAL SNAPSHOT join requires at least one equality predicate."); + } + + @Test + void testRejectSnapshotOutsideLateral() { + // SNAPSHOT used outside a LATERAL context is not rewritten into a join; + // ForbidSnapshotOutsideLateralRule rejects the surviving SNAPSHOT scan with a clear + // message. + assertThatThrownBy(() -> util.verifyExecPlan("SELECT * FROM SNAPSHOT(input => TABLE b)")) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "The SNAPSHOT function can only be used as the build side " + + "(right-hand side) of a LATERAL join"); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/batch/JoinHintTestBase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/batch/JoinHintTestBase.java index 24ae0e4025d08d..217e3c2b8af7e1 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/batch/JoinHintTestBase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/batch/JoinHintTestBase.java @@ -61,8 +61,9 @@ public abstract class JoinHintTestBase extends TableTestBase { private final List allJoinHintNames = Lists.newArrayList(JoinStrategy.values()).stream() - // LOOKUP hint has different kv-options against other join hints - .filter(hint -> hint != JoinStrategy.LOOKUP) + // LOOKUP and EARLY_FIRE hints only support key-value options, unlike the + // list-option join hints exercised here + .filter(hint -> hint != JoinStrategy.LOOKUP && hint != JoinStrategy.EARLY_FIRE) .map(JoinStrategy::getJoinHintName) .collect(Collectors.toList()); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest.java new file mode 100644 index 00000000000000..88aac40078f8e2 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.hints.stream; + +import org.apache.flink.table.api.ExplainDetail; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.planner.utils.PlanKind; +import org.apache.flink.table.planner.utils.StreamTableTestUtil; +import org.apache.flink.table.planner.utils.TableTestBase; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import scala.Enumeration; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Test for the EARLY_FIRE join hint surface and option validation. */ +class EarlyFireJoinHintTest extends TableTestBase { + + protected StreamTableTestUtil util; + + @BeforeEach + void before() { + util = streamTestUtil(TableConfig.getDefault()); + util.tableEnv() + .executeSql( + "CREATE TABLE MyTable (\n" + + " a INT,\n" + + " b VARCHAR,\n" + + " c BIGINT,\n" + + " proctime AS PROCTIME(),\n" + + " rowtime TIMESTAMP(3),\n" + + " WATERMARK FOR rowtime AS rowtime\n" + + ") WITH (\n" + + " 'connector' = 'values',\n" + + " 'bounded' = 'false'\n" + + ")"); + util.tableEnv() + .executeSql( + "CREATE TABLE MyTable2 (\n" + + " a INT,\n" + + " b VARCHAR,\n" + + " c BIGINT,\n" + + " proctime AS PROCTIME(),\n" + + " rowtime TIMESTAMP(3),\n" + + " WATERMARK FOR rowtime AS rowtime\n" + + ") WITH (\n" + + " 'connector' = 'values',\n" + + " 'bounded' = 'false'\n" + + ")"); + util.tableEnv() + .executeSql( + "CREATE TABLE MySink (\n" + + " a INT,\n" + + " b VARCHAR\n" + + ") WITH (\n" + + " 'connector' = 'values'\n" + + ")"); + } + + @Test + void testEarlyFireMissingDelay() { + String sql = + "SELECT /*+ EARLY_FIRE('time-mode'='rowtime') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)).hasMessageContaining("incomplete required option(s)"); + } + + @Test + void testEarlyFireNonPositiveDelay() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='0s') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)) + .hasMessageContaining("value should be at least 1 millisecond"); + } + + @Test + void testEarlyFireSubMillisecondDelay() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='1ns') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)) + .hasMessageContaining("value should be at least 1 millisecond"); + } + + @Test + void testEarlyFireInvalidTimeMode() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='5s', 'time-mode'='unknown') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)) + .hasMessageContaining("Invalid EARLY_FIRE hint options"); + } + + @Test + void testEarlyFireUnknownOption() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='5s', 'timemode'='proctime') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)) + .hasMessageContaining("Unsupported EARLY_FIRE hint option(s) [timemode]"); + } + + @Test + void testEarlyFireListOptionsRejected() { + String sql = + "SELECT /*+ EARLY_FIRE('5s') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)) + .hasMessageContaining("only support key-value options"); + } + + @Test + void testEarlyFireLowerCaseHintNamePreservesOptions() { + String sql = + "SELECT /*+ early_fire('delay'='5s', 'time-mode'='rowtime') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + verify(sql); + } + + @Test + void testEarlyFireOnRowTimeLeftOuterJoin() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='5s') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + verify(sql); + } + + @Test + void testEarlyFireRowTimeOnProcTimeJoin() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='5s', 'time-mode'='rowtime') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.proctime BETWEEN t2.proctime - INTERVAL '1' HOUR AND t2.proctime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)) + .hasStackTraceContaining("requires a row-time interval join"); + } + + @Test + void testEarlyFireProcTimeOnRowTimeJoin() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='5s', 'time-mode'='proctime') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + assertThatThrownBy(() -> verify(sql)).hasStackTraceContaining("not yet supported"); + } + + @Test + void testEarlyFireOnProcTimeLeftOuterJoin() { + String sql = + "SELECT /*+ EARLY_FIRE('delay'='5s') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.proctime BETWEEN t2.proctime - INTERVAL '1' HOUR AND t2.proctime + INTERVAL '1' HOUR"; + verify(sql); + } + + @Test + void testEarlyFireJsonPlanRoundTrip() { + String insert = + "INSERT INTO MySink\n" + + "SELECT /*+ EARLY_FIRE('delay'='5s') */ t1.a, t2.b\n" + + "FROM MyTable t1 LEFT OUTER JOIN MyTable2 t2 ON\n" + + " t1.a = t2.a AND\n" + + " t1.rowtime BETWEEN t2.rowtime - INTERVAL '10' SECOND AND t2.rowtime + INTERVAL '1' HOUR"; + util.verifyJsonPlan(insert); + } + + private void verify(String sql) { + util.doVerifyPlan( + sql, + new ExplainDetail[] {}, + false, + new Enumeration.Value[] {PlanKind.AST(), PlanKind.OPT_EXEC()}, + false); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/LateralSnapshotJoinBatchSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/LateralSnapshotJoinBatchSemanticTests.java new file mode 100644 index 00000000000000..87725a60a26ca9 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/LateralSnapshotJoinBatchSemanticTests.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.batch; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.BatchSemanticTestBase; +import org.apache.flink.table.planner.plan.rules.physical.batch.BatchPhysicalLateralSnapshotJoinRule; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** + * Semantic tests for {@code LATERAL SNAPSHOT} joins in batch mode. {@link + * BatchPhysicalLateralSnapshotJoinRule} degrades the join to a regular join of the probe side + * against the (final) build side. + */ +public class LateralSnapshotJoinBatchSemanticTests extends BatchSemanticTestBase { + + @Override + public List programs() { + return List.of( + LateralSnapshotJoinTestPrograms.INNER_JOIN, + LateralSnapshotJoinTestPrograms.LEFT_JOIN, + LateralSnapshotJoinTestPrograms.INNER_JOIN_WITH_NON_EQUI_CONDITION, + LateralSnapshotJoinTestPrograms.SNAPSHOT_ARGUMENTS_IGNORED); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/LateralSnapshotJoinTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/LateralSnapshotJoinTestPrograms.java new file mode 100644 index 00000000000000..8b8be2119520f2 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/batch/LateralSnapshotJoinTestPrograms.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.batch; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +/** + * {@link TableTestProgram} definitions for semantically testing {@code LATERAL SNAPSHOT} joins in + * batch mode, where the join degenerates to a regular join of the probe side against the (final) + * build side. + */ +public class LateralSnapshotJoinTestPrograms { + + private static SourceTestStep probe() { + return SourceTestStep.newBuilder("probe") + .addSchema("pk STRING", "pv INT") + .producedValues(Row.of("a", 1), Row.of("b", 2), Row.of("c", 3)) + .build(); + } + + private static SourceTestStep build() { + // The key "a" appears twice: the batch join runs against the final (complete) build side, + // so both "a" rows participate. + return SourceTestStep.newBuilder("b") + .addSchema("bk STRING", "bv INT") + .producedValues(Row.of("a", 10), Row.of("a", 11), Row.of("b", 20)) + .build(); + } + + public static final TableTestProgram INNER_JOIN = + TableTestProgram.of( + "lateral-snapshot-join-inner", + "batch LATERAL SNAPSHOT inner join degrades to a regular join") + .setupTableSource(probe()) + .setupTableSource(build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "pv INT", "bk STRING", "bv INT") + .consumedValues( + Row.of("a", 1, "a", 10), + Row.of("a", 1, "a", 11), + Row.of("b", 2, "b", 20)) + .build()) + .runSql( + "INSERT INTO sink SELECT pk, pv, bk, bv FROM probe JOIN LATERAL " + + "TABLE(SNAPSHOT(input => TABLE b)) AS s ON probe.pk = s.bk") + .build(); + + public static final TableTestProgram LEFT_JOIN = + TableTestProgram.of( + "lateral-snapshot-join-left", + "batch LATERAL SNAPSHOT left join null-pads unmatched probe rows") + .setupTableSource(probe()) + .setupTableSource(build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "pv INT", "bk STRING", "bv INT") + .consumedValues( + Row.of("a", 1, "a", 10), + Row.of("a", 1, "a", 11), + Row.of("b", 2, "b", 20), + Row.of("c", 3, null, null)) + .build()) + .runSql( + "INSERT INTO sink SELECT pk, pv, bk, bv FROM probe LEFT JOIN LATERAL " + + "TABLE(SNAPSHOT(input => TABLE b)) AS s ON probe.pk = s.bk") + .build(); + + public static final TableTestProgram INNER_JOIN_WITH_NON_EQUI_CONDITION = + TableTestProgram.of( + "lateral-snapshot-join-non-equi", + "batch LATERAL SNAPSHOT join with an additional non-equi predicate") + .setupTableSource(probe()) + .setupTableSource(build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "pv INT", "bk STRING", "bv INT") + .consumedValues( + Row.of("a", 1, "a", 11), Row.of("b", 2, "b", 20)) + .build()) + .runSql( + "INSERT INTO sink SELECT pk, pv, bk, bv FROM probe JOIN LATERAL " + + "TABLE(SNAPSHOT(input => TABLE b)) AS s " + + "ON probe.pk = s.bk AND s.bv > 10") + .build(); + + public static final TableTestProgram SNAPSHOT_ARGUMENTS_IGNORED = + TableTestProgram.of( + "lateral-snapshot-join-arguments-ignored", + "streaming-only SNAPSHOT arguments do not affect the batch result") + .setupTableSource(probe()) + .setupTableSource(build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "pv INT", "bk STRING", "bv INT") + .consumedValues( + Row.of("a", 1, "a", 10), + Row.of("a", 1, "a", 11), + Row.of("b", 2, "b", 20)) + .build()) + .runSql( + "INSERT INTO sink SELECT pk, pv, bk, bv FROM probe JOIN LATERAL " + + "TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'compile_time', " + + "load_completed_idle_timeout => INTERVAL '10' SECOND, " + + "state_ttl => INTERVAL '1' DAY" + + ")) AS s ON probe.pk = s.bk") + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java new file mode 100644 index 00000000000000..806ecf72ef3b10 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.factories.TestValuesTableFactory; +import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** Restore tests for {@link StreamExecLateralSnapshotJoin}. */ +public class LateralSnapshotJoinRestoreTest extends RestoreTestBase { + + // Bounds the wait for the LOAD-phase savepoint trigger so a stuck source fails fast. + private static final long SAVEPOINT_READY_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); + + public LateralSnapshotJoinRestoreTest() { + super(StreamExecLateralSnapshotJoin.class); + } + + @Override + public List programs() { + return Arrays.asList( + LateralSnapshotJoinTestPrograms.LATERAL_SNAPSHOT_JOIN_PHASE_LOAD, + LateralSnapshotJoinTestPrograms.LATERAL_SNAPSHOT_JOIN_PHASE_JOIN); + } + + @Override + protected void awaitSavepointReady(TableTestProgram program, List> futures) + throws Exception { + if (program != LateralSnapshotJoinTestPrograms.LATERAL_SNAPSHOT_JOIN_PHASE_LOAD) { + super.awaitSavepointReady(program, futures); + return; + } + // The operator emits nothing during its LOAD phase, so we can't use the default sink-based + // trigger. Instead, we gate on the number of records emitted by the sources. Once every + // source has emitted its before-restore rows, stop-with-savepoint drains them into keyed + // state before snapshotting. + for (SourceTestStep source : program.getSetupSourceTestSteps()) { + final int count = source.dataBeforeRestore.size(); + if (count > 0) { + TestValuesTableFactory.awaitSourceEmitted(source.name, count) + .get(SAVEPOINT_READY_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } + } + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinSemanticTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinSemanticTestPrograms.java new file mode 100644 index 00000000000000..ca2f3696938cfc --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinSemanticTestPrograms.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Deterministic result {@link TableTestProgram} definitions for the {@code LATERAL SNAPSHOT} + * processing-time temporal join ({@link StreamExecLateralSnapshotJoin}). + * + *

Processing-time semantics make the join non-deterministic in general. To get stable results, + * each build source appends a non-matching "flip-trigger" row at the flip timestamp ({@link + * #FLIP_TRIGGER_TS}) while all real build rows are earlier. With per-record ({@code on-event}) + * watermarks the operator flips to the JOIN phase. Some programs throttle source emission ({@code + * source.sleep-*}) to place probes deterministically around the flip. + */ +public class LateralSnapshotJoinSemanticTestPrograms { + + /** The {@code 'user_time'} condition reached mid-stream by the build-side flip-trigger row. */ + private static final String MID_FLIP = + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2020-01-01 00:00:10' AS TIMESTAMP_LTZ(3))"; + + /** A far-future flip condition: the flip happens only at end of all input. */ + private static final String END_FLIP = + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2100-01-01 00:00:00' AS TIMESTAMP_LTZ(3))"; + + /** Event time of the flip-trigger row; equal to the {@link #MID_FLIP} timestamp. */ + private static final String FLIP_TRIGGER_TS = "00:00:10"; + + /** A build-side key that never matches any probe row. */ + private static final String FLIP_TRIGGER_KEY = "__flip_trigger__"; + + private static final String[] PROBE_SCHEMA = { + "pk STRING", "pv INT", "pts TIMESTAMP(3)", "WATERMARK FOR pts AS pts" + }; + + private static final String[] BUILD_SCHEMA = { + "bk STRING", "bv INT", "bts TIMESTAMP(3)", "WATERMARK FOR bts AS bts" + }; + + // ------------------------------------------------------------------------------------------ + // Core join semantics + // ------------------------------------------------------------------------------------------ + + public static final TableTestProgram INNER_JOIN = + TableTestProgram.of("lateral-snapshot-inner-join", "LATERAL SNAPSHOT inner join") + // Throttle the probe so the fast build flips first; the probes are then joined + // live in the JOIN phase (the build-side is finalized, so buffered-vs-live does + // not change the result). + .setupTableSource(throttledProbe(defaultProbe(), 40L)) + .setupTableSource(appendBuild(withFlipTrigger(defaultBuild()))) + .setupTableSink( + keyValueSink() + .consumedValues( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]") + .build()) + .runSql( + innerJoin( + "probe.pk, probe.pv, s.bk, s.bv", MID_FLIP, "probe.pk = s.bk")) + .build(); + + public static final TableTestProgram LEFT_JOIN = + TableTestProgram.of("lateral-snapshot-left-join", "LATERAL SNAPSHOT left join") + .setupTableSource(throttledProbe(defaultProbe(), 40L)) + .setupTableSource(appendBuild(withFlipTrigger(defaultBuild()))) + .setupTableSink( + keyValueSink() + .consumedValues( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]", + "+I[c, 300, null, null]") + .build()) + .runSql(leftJoin("probe.pk, probe.pv, s.bk, s.bv", MID_FLIP, "probe.pk = s.bk")) + .build(); + + public static final TableTestProgram SELECT_STAR = + TableTestProgram.of( + "lateral-snapshot-select-star", + "SELECT * materializes the build-side rowtime as a regular TIMESTAMP") + .setupTableSource(probe(List.of(Row.of("a", 100, ts("00:01:00"))))) + .setupTableSource( + appendBuild(withFlipTrigger(List.of(Row.of("a", 10, ts("00:00:01")))))) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "pk STRING", + "pv INT", + "pts TIMESTAMP(3)", + "bk STRING", + "bv INT", + "bts TIMESTAMP(3)") + .testMaterializedData() + .consumedValues( + "+I[a, 100, 2020-01-01T00:01, a, 10, 2020-01-01T00:00:01]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_FLIP + + ")) AS s ON probe.pk = s.bk") + .build(); + + public static final TableTestProgram COMPOSITE_KEYS = + TableTestProgram.of("lateral-snapshot-composite-keys", "join on composite keys") + .setupTableSource( + probe( + Arrays.asList( + Row.of("a", 10, ts("00:01:00")), + Row.of("a", 11, ts("00:01:01")), + Row.of("z", 10, ts("00:01:01")), + Row.of("b", 20, ts("00:01:02"))))) + .setupTableSource( + appendBuild( + withFlipTrigger( + Arrays.asList( + Row.of("a", 10, ts("00:00:01")), + Row.of("a", 99, ts("00:00:02")), + Row.of("b", 20, ts("00:00:03")))))) + .setupTableSink( + keyValueSink() + .consumedValues("+I[a, 10, a, 10]", "+I[b, 20, b, 20]") + .build()) + .runSql( + innerJoin( + "probe.pk, probe.pv, s.bk, s.bv", + MID_FLIP, + "probe.pk = s.bk AND probe.pv = s.bv")) + .build(); + + public static final TableTestProgram NON_EQUI = + TableTestProgram.of("lateral-snapshot-non-equi", "join with a non-equi condition") + .setupTableSource( + probe( + Arrays.asList( + Row.of("a", 15, ts("00:01:00")), + Row.of("a", 5, ts("00:01:01"))))) + .setupTableSource( + appendBuild( + withFlipTrigger( + Arrays.asList( + Row.of("a", 10, ts("00:00:01")), + Row.of("a", 20, ts("00:00:02")))))) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "pv INT", "bv INT") + .testMaterializedData() + .consumedValues("+I[a, 15, 10]") + .build()) + .runSql( + innerJoin( + "probe.pk, probe.pv, s.bv", + MID_FLIP, + "probe.pk = s.bk AND probe.pv > s.bv")) + .build(); + + public static final TableTestProgram EMPTY_BUILD_INNER = + TableTestProgram.of( + "lateral-snapshot-empty-build-inner", + "inner join over an empty build side yields no rows") + .setupTableSource(probe(defaultProbe())) + .setupTableSource(appendBuild(withFlipTrigger(List.of()))) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "bk STRING") + .testMaterializedData() + .consumedValues(new String[0]) + .build()) + .runSql(innerJoin("probe.pk, s.bk", MID_FLIP, "probe.pk = s.bk")) + .build(); + + public static final TableTestProgram EMPTY_BUILD_LEFT = + TableTestProgram.of( + "lateral-snapshot-empty-build-left", + "left join over an empty build side preserves the probe rows") + .setupTableSource(probe(defaultProbe())) + .setupTableSource(appendBuild(withFlipTrigger(List.of()))) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "bk STRING") + .testMaterializedData() + .consumedValues("+I[a, null]", "+I[b, null]", "+I[c, null]") + .build()) + .runSql(leftJoin("probe.pk, s.bk", MID_FLIP, "probe.pk = s.bk")) + .build(); + + // ------------------------------------------------------------------------------------------ + // Flip timing + // ------------------------------------------------------------------------------------------ + + public static final TableTestProgram FLIP_AT_END = + TableTestProgram.of( + "lateral-snapshot-flip-at-end", + "far-future flip condition: the operator flips only at end-of-input") + .setupTableSource(probe(defaultProbe())) + .setupTableSource(appendBuild(withFlipTrigger(defaultBuild()))) + .setupTableSink( + keyValueSink() + .consumedValues( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]") + .build()) + .runSql( + innerJoin( + "probe.pk, probe.pv, s.bk, s.bv", END_FLIP, "probe.pk = s.bk")) + .build(); + + public static final TableTestProgram DEFAULT_COMPILE_TIME = + TableTestProgram.of( + "lateral-snapshot-default-compile-time", + "default 'compile_time' condition flips at end-of-input for 2020 data") + .setupTableSource(probe(defaultProbe())) + .setupTableSource(appendBuild(withFlipTrigger(defaultBuild()))) + .setupTableSink( + keyValueSink() + .consumedValues( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]") + .build()) + // No options: 'load_completed_condition' defaults to 'compile_time'. + .runSql( + "INSERT INTO sink SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b" + + ")) AS s ON probe.pk = s.bk") + .build(); + + public static final TableTestProgram LIVE_JOIN = + TableTestProgram.of( + "lateral-snapshot-live-join", + "probes are joined live per-record in the JOIN phase") + // Fast build flips almost immediately; the throttled probe stream is consumed + // entirely in the JOIN phase (including non-matching keys c). The snapshot is + // static after the flip, so the result is deterministic. + .setupTableSource( + throttledProbe( + Arrays.asList( + Row.of("a", 1, ts("00:00:01")), + Row.of("b", 2, ts("00:00:02")), + Row.of("c", 3, ts("00:00:03")), + Row.of("a", 4, ts("00:00:04")), + Row.of("b", 5, ts("00:00:05")), + Row.of("c", 6, ts("00:00:06"))), + 40L)) + .setupTableSource( + appendBuild( + withFlipTrigger( + Arrays.asList( + Row.of("a", 10, ts("00:00:01")), + Row.of("b", 20, ts("00:00:02")))))) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pv INT", "pk STRING", "bv INT") + .testMaterializedData() + .consumedValues( + "+I[1, a, 10]", + "+I[4, a, 10]", + "+I[2, b, 20]", + "+I[5, b, 20]") + .build()) + .runSql(innerJoin("probe.pv, probe.pk, s.bv", MID_FLIP, "probe.pk = s.bk")) + .build(); + + public static final TableTestProgram BUFFERED_THEN_DRAINED = + TableTestProgram.of( + "lateral-snapshot-buffered-then-drained", + "probes buffered during LOAD are drained against the final version") + // Un-throttled: buffered within a few ms, long before the flip. + .setupTableSource(probe(manyProbes(20))) + // Throttled build flips at ~240 ms, by which time the probe stream is fully + // buffered in LOAD; the flip drains all buffered probes against version 3. + .setupTableSource( + throttledUpsertBuild( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "k", 1, ts("00:00:01")), + Row.ofKind( + RowKind.UPDATE_AFTER, "k", 2, ts("00:00:02")), + Row.ofKind( + RowKind.UPDATE_AFTER, "k", 3, ts("00:00:05")), + Row.ofKind( + RowKind.INSERT, + FLIP_TRIGGER_KEY, + 0, + ts(FLIP_TRIGGER_TS))), + 80L)) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema("pv INT", "bv INT") + .testMaterializedData() + .consumedValues( + IntStream.rangeClosed(1, 20) + .mapToObj(i -> String.format("+I[%d, 3]", i)) + .toArray(String[]::new)) + .build()) + .runSql(innerJoin("probe.pv, s.bv", MID_FLIP, "probe.pk = s.bk")) + .build(); + + // ------------------------------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------------------------------ + + private static String innerJoin(String projection, String flip, String condition) { + return join("JOIN", projection, flip, condition); + } + + private static String leftJoin(String projection, String flip, String condition) { + return join("LEFT JOIN", projection, flip, condition); + } + + private static String join(String joinType, String projection, String flip, String condition) { + return "INSERT INTO sink SELECT " + + projection + + " FROM probe " + + joinType + + " LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + flip + + ")) AS s ON " + + condition; + } + + private static List defaultProbe() { + return Arrays.asList( + Row.of("a", 100, ts("00:01:00")), + Row.of("b", 200, ts("00:01:01")), + Row.of("c", 300, ts("00:01:02"))); + } + + private static List defaultBuild() { + return Arrays.asList( + Row.of("a", 10, ts("00:00:01")), + Row.of("b", 20, ts("00:00:02")), + Row.of("a", 11, ts("00:00:03"))); + } + + private static List manyProbes(int count) { + return IntStream.rangeClosed(1, count) + .mapToObj(i -> Row.of("k", i, ts(String.format("00:00:%02d", i)))) + .collect(Collectors.toList()); + } + + /** + * Appends a non-matching build row at the {@link #FLIP_TRIGGER_TS} timestamp; its watermark + * flips the operator to the JOIN phase mid-stream (all real build rows are earlier). + */ + private static List withFlipTrigger(List data) { + final List withTrigger = new ArrayList<>(data); + withTrigger.add(Row.of(FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS))); + return withTrigger; + } + + private static SourceTestStep probe(List data) { + return SourceTestStep.newBuilder("probe") + .addSchema(PROBE_SCHEMA) + .producedValues(data.toArray(new Row[0])) + .build(); + } + + private static SourceTestStep throttledProbe(List data, long sleepMillis) { + return SourceTestStep.newBuilder("probe") + .addSchema(PROBE_SCHEMA) + .addOptions(throttleOptions(sleepMillis)) + .producedValues(data.toArray(new Row[0])) + .build(); + } + + private static SourceTestStep appendBuild(List data) { + return SourceTestStep.newBuilder("b") + .addSchema(BUILD_SCHEMA) + .producedValues(data.toArray(new Row[0])) + .build(); + } + + private static SourceTestStep throttledUpsertBuild(List data, long sleepMillis) { + return SourceTestStep.newBuilder("b") + .addSchema( + "bk STRING", + "bv INT", + "bts TIMESTAMP(3)", + "WATERMARK FOR bts AS bts", + "PRIMARY KEY (bk) NOT ENFORCED") + .addOption("changelog-mode", "I,UA,D") + .addOptions(throttleOptions(sleepMillis)) + .producedValues(data.toArray(new Row[0])) + .build(); + } + + private static SinkTestStep.Builder keyValueSink() { + return SinkTestStep.newBuilder("sink") + .addSchema("pk STRING", "pv INT", "bk STRING", "bv INT") + .testMaterializedData(); + } + + private static Map throttleOptions(long sleepMillis) { + final Map options = new HashMap<>(); + options.put("source.sleep-after-elements", "1"); + options.put("source.sleep-time", sleepMillis + "ms"); + return options; + } + + private static LocalDateTime ts(String time) { + return LocalDateTime.parse("2020-01-01T" + time); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinSemanticTests.java new file mode 100644 index 00000000000000..6536f34a591492 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinSemanticTests.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.SemanticTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.List; + +/** Semantic tests for {@link StreamExecLateralSnapshotJoin}. */ +public class LateralSnapshotJoinSemanticTests extends SemanticTestBase { + @Override + public List programs() { + return List.of( + LateralSnapshotJoinSemanticTestPrograms.INNER_JOIN, + LateralSnapshotJoinSemanticTestPrograms.LEFT_JOIN, + LateralSnapshotJoinSemanticTestPrograms.SELECT_STAR, + LateralSnapshotJoinSemanticTestPrograms.COMPOSITE_KEYS, + LateralSnapshotJoinSemanticTestPrograms.NON_EQUI, + LateralSnapshotJoinSemanticTestPrograms.EMPTY_BUILD_INNER, + LateralSnapshotJoinSemanticTestPrograms.EMPTY_BUILD_LEFT, + LateralSnapshotJoinSemanticTestPrograms.FLIP_AT_END, + LateralSnapshotJoinSemanticTestPrograms.DEFAULT_COMPILE_TIME, + LateralSnapshotJoinSemanticTestPrograms.LIVE_JOIN, + LateralSnapshotJoinSemanticTestPrograms.BUFFERED_THEN_DRAINED); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java new file mode 100644 index 00000000000000..b1c58be353bb04 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +/** + * {@link TableTestProgram} definitions for testing {@link StreamExecLateralSnapshotJoin}. + * + *

The programs cover a savepoint taken in each of the operator's two phases; the {@code + * 'user_time'} gate is at {@code 00:00:03} in both. + * + *

    + *
  • {@link #LATERAL_SNAPSHOT_JOIN_PHASE_LOAD}: the transition to JOIN is not triggered before + * the savepoint, so the operator is still in LOAD; the savepoint captures the partial build + * multi-set and the buffered probe row. The flip to JOIN phase happens only after restore. + *
  • {@link #LATERAL_SNAPSHOT_JOIN_PHASE_JOIN}: the whole build side is loaded before the + * savepoint (the last build-side row triggers the flip), so the operator has flipped to JOIN + * and its build-side snapshot is materialized and frozen when the stop-with-savepoint fires. + *
+ * + *

Both verify that the build state and the LOAD/JOIN phase (union operator state) survive the + * savepoint: after restore the "after restore" probe rows join the restored snapshot. + */ +public class LateralSnapshotJoinTestPrograms { + + static final String[] PROBE_SCHEMA = { + "pk STRING", + "pv INT", + "pts_str STRING", + "pts AS TO_TIMESTAMP(pts_str)", + "WATERMARK FOR pts AS pts" + }; + + static final String[] BUILD_SCHEMA = { + "bk STRING", + "bv INT", + "bts_str STRING", + "bts AS TO_TIMESTAMP(bts_str)", + "WATERMARK FOR bts AS bts" + }; + + static final String[] SINK_SCHEMA = {"pk STRING", "pv INT", "bk STRING", "bv INT"}; + + // Two rows for key 'a' exercise the per-key multi-set; the last row's watermark (00:00:03) + // reaches the gate and flips the operator to JOIN. + static final Row[] BUILD_BEFORE_DATA = { + Row.of("a", 10, "2020-01-01 00:00:01"), + Row.of("b", 20, "2020-01-01 00:00:02"), + Row.of("a", 11, "2020-01-01 00:00:03") + }; + + static final Row[] PROBE_BEFORE_DATA = { + Row.of("a", 100, "2020-01-01 00:00:06"), Row.of("b", 200, "2020-01-01 00:00:07") + }; + + // 'a' matches the restored snapshot; 'c' has no match. + static final Row[] PROBE_AFTER_DATA = { + Row.of("a", 101, "2020-01-01 00:00:10"), Row.of("c", 300, "2020-01-01 00:00:11") + }; + + // LOAD-phase restore scenario: none of these build rows reaches the 00:00:03 gate, so the + // operator is still in LOAD when the savepoint is taken. The number of rows is the count the + // savepoint trigger waits for (see LateralSnapshotJoinRestoreTest#awaitSavepointReady). + static final Row[] LOAD_BUILD_BEFORE_DATA = { + Row.of("a", 10, "2020-01-01 00:00:01"), Row.of("a", 11, "2020-01-01 00:00:02") + }; + + // The gate row (00:00:03) arrives only after restore and flips the operator to JOIN, draining + // the probe rows buffered before the savepoint. + static final Row[] LOAD_BUILD_AFTER_DATA = {Row.of("a", 12, "2020-01-01 00:00:03")}; + + // Buffered during LOAD (LOAD emits nothing), so it survives the savepoint in probe state. + static final Row[] LOAD_PROBE_BEFORE_DATA = {Row.of("a", 100, "2020-01-01 00:00:06")}; + + static final Row[] LOAD_PROBE_AFTER_DATA = {Row.of("a", 101, "2020-01-01 00:00:10")}; + + private static final String SNAPSHOT_BUILD = + "LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2020-01-01 00:00:03' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk"; + + // Restore taken while the operator is in LOAD phase: the savepoint captures the partial build + // multi-set and the buffered probe row, the LOAD phase is recorded in union operator state, and + // nothing has been emitted yet. After restore the 00:00:03 build row completes the load, flips + // to JOIN, and the buffered + after-restore probe rows join the full snapshot {10, 11, 12} for + // key 'a'. + public static final TableTestProgram LATERAL_SNAPSHOT_JOIN_PHASE_LOAD = + TableTestProgram.of( + "lateral-snapshot-join-load-phase", + "validates a LATERAL SNAPSHOT inner join restored from a LOAD-phase savepoint") + .setupTableSource( + SourceTestStep.newBuilder("probe") + .addSchema(PROBE_SCHEMA) + .producedBeforeRestore(LOAD_PROBE_BEFORE_DATA) + .producedAfterRestore(LOAD_PROBE_AFTER_DATA) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("b") + .addSchema(BUILD_SCHEMA) + .producedBeforeRestore(LOAD_BUILD_BEFORE_DATA) + .producedAfterRestore(LOAD_BUILD_AFTER_DATA) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(SINK_SCHEMA) + // Empty before restore: LOAD phase emits nothing before the + // savepoint. The explicit String[] picks the String overload + // and marks this a SINK_WITH_RESTORE_DATA step. + .consumedBeforeRestore(new String[0]) + .consumedAfterRestore( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[a, 100, a, 12]", + "+I[a, 101, a, 10]", + "+I[a, 101, a, 11]", + "+I[a, 101, a, 12]") + .build()) + .runSql( + "INSERT INTO sink SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN " + + SNAPSHOT_BUILD) + .build(); + + // Restore taken while the operator is in JOIN phase + public static final TableTestProgram LATERAL_SNAPSHOT_JOIN_PHASE_JOIN = + TableTestProgram.of( + "lateral-snapshot-join-inner", + "validates a LATERAL SNAPSHOT inner join across a restore") + .setupTableSource( + SourceTestStep.newBuilder("probe") + .addSchema(PROBE_SCHEMA) + .producedBeforeRestore(PROBE_BEFORE_DATA) + .producedAfterRestore(PROBE_AFTER_DATA) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("b") + .addSchema(BUILD_SCHEMA) + .producedBeforeRestore(BUILD_BEFORE_DATA) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(SINK_SCHEMA) + .consumedBeforeRestore( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]") + .consumedAfterRestore("+I[a, 101, a, 10]", "+I[a, 101, a, 11]") + .build()) + .runSql( + "INSERT INTO sink SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN " + + SNAPSHOT_BUILD) + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionSemanticTests.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionSemanticTests.java index 39131ff4ed3c97..2357235b11864b 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionSemanticTests.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionSemanticTests.java @@ -42,6 +42,8 @@ public List programs() { ProcessTableFunctionTestPrograms.PROCESS_SET_SEMANTIC_TABLE_TABLE_API_INLINE, ProcessTableFunctionTestPrograms.PROCESS_SET_SEMANTIC_TABLE_TABLE_API_INLINE_NAMED, ProcessTableFunctionTestPrograms.PROCESS_TYPED_SET_SEMANTIC_TABLE, + ProcessTableFunctionTestPrograms.PROCESS_MULTI_PARTITION_BY, + ProcessTableFunctionTestPrograms.PROCESS_MULTI_PARTITION_BY_AND_ORDER_BY, ProcessTableFunctionTestPrograms.PROCESS_TYPED_SET_SEMANTIC_TABLE_TABLE_API, ProcessTableFunctionTestPrograms.PROCESS_POJO_ARGS, ProcessTableFunctionTestPrograms.PROCESS_INTERVAL_DAY_ARGS, diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionTestPrograms.java index 7ea4950c23fe25..327c3167fe53dc 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionTestPrograms.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ProcessTableFunctionTestPrograms.java @@ -1961,4 +1961,86 @@ public class ProcessTableFunctionTestPrograms { // Also in constructed types: ROW (table input) vs. STRUCTURED (expected). .runSql("INSERT INTO sink SELECT * FROM f(p => TABLE v, b => 42)") .build(); + + public static final TableTestProgram PROCESS_MULTI_PARTITION_BY = + TableTestProgram.of( + "process-set-from-session-view-with-multi-partition-by", + "set semantic table partitioned by multiple columns, sourced from a view wrapping a SESSION window aggregate") + .setupTemporarySystemFunction("f", SetSemanticTableFunction.class) + .setupTableSource( + SourceTestStep.newBuilder("t") + .addSchema( + "suite_name STRING", + "test_name STRING", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '0.001' SECOND") + .producedValues( + Row.of("suiteA", "test1", Instant.ofEpochMilli(0)), + Row.of("suiteB", "test2", Instant.ofEpochMilli(1)), + Row.of("suiteA", "test1", Instant.ofEpochMilli(2)), + Row.of("suiteA", "test1", Instant.ofEpochMilli(3)), + Row.of("suiteA", "test1", Instant.ofEpochMilli(4)), + Row.of("suiteA", "test1", Instant.ofEpochMilli(5)), + Row.of("suiteA", "test1", Instant.ofEpochMilli(6))) + .build()) + .setupSql( + "CREATE VIEW v AS " + + "SELECT suite_name, test_name, COUNT(*) AS c " + + "FROM SESSION(TABLE t PARTITION BY (suite_name, test_name), DESCRIPTOR(ts), INTERVAL '0.002' SECOND) " + + "GROUP BY suite_name, test_name, window_start, window_end") + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "`suite_name` STRING", + "`test_name` STRING", + "`out` STRING") + .consumedValues( + "+I[suiteB, test2, {+I[suiteB, test2, 1], 1}]", + "+I[suiteA, test1, {+I[suiteA, test1, 6], 1}]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM f(r => TABLE v PARTITION BY (suite_name, test_name), i => 1)") + .build(); + + public static final TableTestProgram PROCESS_MULTI_PARTITION_BY_AND_ORDER_BY = + TableTestProgram.of( + "process-order-by-multi-partition-key-and-order-by", + "set semantic table partitioned and ordered by multiple columns, sourced from a view wrapping a SESSION window aggregate") + .setupTemporarySystemFunction("f", SetSemanticTableFunction.class) + .setupTableSource( + SourceTestStep.newBuilder("t") + .addSchema( + "suite_name STRING", + "test_name STRING", + "`group` STRING", + "ts TIMESTAMP_LTZ(3)", + "WATERMARK FOR ts AS ts - INTERVAL '0.001' SECOND") + .producedValues( + // group is only used to force two independent SESSION + // windows for suiteA/test1 whose window_time ties. + Row.of("suiteA", "test1", "x", Instant.ofEpochMilli(0)), + Row.of("suiteB", "test2", "x", Instant.ofEpochMilli(1)), + Row.of("suiteA", "test1", "x", Instant.ofEpochMilli(2)), + Row.of("suiteA", "test1", "y", Instant.ofEpochMilli(2))) + .build()) + .setupSql( + "CREATE VIEW v AS " + + "SELECT suite_name, test_name, window_time, COUNT(*) AS c " + + "FROM SESSION(TABLE t PARTITION BY (suite_name, test_name, `group`), DESCRIPTOR(ts), INTERVAL '0.002' SECOND) " + + "GROUP BY suite_name, test_name, `group`, window_start, window_end, window_time") + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema( + "`suite_name` STRING", + "`test_name` STRING", + "`out` STRING") + .consumedValues( + "+I[suiteB, test2, {+I[suiteB, test2, 1970-01-01T00:00:00.002Z, 1], 1}]", + "+I[suiteA, test1, {+I[suiteA, test1, 1970-01-01T00:00:00.003Z, 2], 1}]", + "+I[suiteA, test1, {+I[suiteA, test1, 1970-01-01T00:00:00.003Z, 1], 1}]") + .build()) + .runSql( + "INSERT INTO sink SELECT * FROM f(" + + "r => TABLE v PARTITION BY (suite_name, test_name) ORDER BY (window_time ASC, c DESC), i => 1)") + .build(); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/CommonSemanticTestBase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/CommonSemanticTestBase.java index f406cd972e394a..2a4d9e67d36d7e 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/CommonSemanticTestBase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/CommonSemanticTestBase.java @@ -22,6 +22,7 @@ import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.config.OptimizerConfigOptions; +import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.table.planner.factories.TestValuesModelFactory; import org.apache.flink.table.planner.factories.TestValuesTableFactory; import org.apache.flink.table.test.program.ConfigOptionTestStep; @@ -192,6 +193,7 @@ protected void applyDefaultEnvironmentOptions(TableConfig config) { config.set( OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY, OptimizerConfigOptions.NonDeterministicUpdateStrategy.TRY_RESOLVE); + config.set(TableConfigOptions.LOCAL_TIME_ZONE, "UTC"); } private Map createSourceOptions(SourceTestStep sourceTestStep, String id) { diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java index f1cdc2ca0bcc23..dc61bbe030ba97 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestBase.java @@ -76,6 +76,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Stream; @@ -282,6 +283,21 @@ private void registerSinkObserver( }); } + /** + * Awaits the point during {@link #generateTestSetupFiles} at which the stop-with-savepoint + * should be triggered. The default waits until every sink has produced its "before restore" + * expected rows. + * + *

Override for operators that produce no output at the point of interest or when the test + * logic requires more customized savepoint triggering (e.g., based on processed input). See + * {@link TestValuesTableFactory#awaitSourceEmitted(String, int)}. + */ + protected void awaitSavepointReady( + final TableTestProgram program, final List> futures) + throws Exception { + awaitExpectedResults(program, futures, true); + } + private void awaitExpectedResults( final TableTestProgram program, final List> futures, @@ -336,6 +352,7 @@ public void generateTestSetupFiles(TableTestProgram program) throws Exception { .set( TableConfigOptions.PLAN_COMPILE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanCompilation.SCHEMA); + tEnv.getConfig().set(TableConfigOptions.LOCAL_TIME_ZONE, "UTC"); for (SourceTestStep sourceTestStep : program.getSetupSourceTestSteps()) { final String id = TestValuesTableFactory.registerData(sourceTestStep.dataBeforeRestore); @@ -379,12 +396,9 @@ public void generateTestSetupFiles(TableTestProgram program) throws Exception { compiledPlan.writeToFile(getPlanPath(program, getLatestMetadata())); final TableResult tableResult = compiledPlan.execute(); - awaitExpectedResults(program, futures, true); + awaitSavepointReady(program, futures); final JobClient jobClient = tableResult.getJobClient().get(); - final String savepoint = - jobClient - .stopWithSavepoint(false, tmpDir.toString(), SavepointFormatType.DEFAULT) - .get(); + final String savepoint = stopWithSavepointWhenRunning(jobClient); CommonTestUtils.waitForJobStatus(jobClient, Collections.singletonList(JobStatus.FINISHED)); final Path savepointPath = Paths.get(new URI(savepoint)); final Path savepointDirPath = @@ -414,6 +428,7 @@ void testRestore(TableTestProgram program, Path planPath, String savepointPath) .set( TableConfigOptions.PLAN_RESTORE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanRestore.IDENTIFIER); + tEnv.getConfig().set(TableConfigOptions.LOCAL_TIME_ZONE, "UTC"); program.getSetupConfigOptionTestSteps().forEach(s -> s.apply(tEnv)); @@ -491,6 +506,31 @@ void testRestore(TableTestProgram program, Path planPath, String savepointPath) } } + /** + * Triggers a stop-with-savepoint, retrying while the job is not yet fully running. A savepoint + * gated on source emission (see {@link #awaitSavepointReady}) can be requested before all + * downstream tasks have reached RUNNING, which aborts with "Not all required tasks are + * currently running". Sink-gated triggers never hit this (sink output implies a running + * pipeline), so they succeed on the first attempt. + */ + private String stopWithSavepointWhenRunning(JobClient jobClient) throws Exception { + final long deadline = System.currentTimeMillis() + RESULT_AWAIT_TIMEOUT_MILLIS; + while (true) { + try { + return jobClient + .stopWithSavepoint(false, tmpDir.toString(), SavepointFormatType.DEFAULT) + .get(); + } catch (ExecutionException e) { + final boolean notAllRunning = + e.getMessage() != null && e.getMessage().contains("Not all required tasks"); + if (!notAllRunning || System.currentTimeMillis() >= deadline) { + throw e; + } + Thread.sleep(200); + } + } + } + private Path getPlanPath(TableTestProgram program, ExecNodeMetadata metadata) { return Paths.get( getTestResourceDirectory(program, metadata) + "/plan/" + program.id + ".json"); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java index 29d9f0db3b9c0e..ea183bf3dbad39 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java @@ -19,7 +19,6 @@ package org.apache.flink.table.planner.plan.nodes.exec.testutils; import org.apache.flink.table.planner.plan.nodes.exec.ExecNode; -import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLateralSnapshotJoin; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonAsyncCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCorrelate; @@ -50,11 +49,6 @@ public class RestoreTestCompleteness { private static final Set>> SKIP_EXEC_NODES = new HashSet>>() { { - // TODO: FLINK-39781 - the LATERAL SNAPSHOT runtime operator is still a stub, - // so a restore test cannot generate a savepoint yet. Remove this entry and - // add LateralSnapshotJoinRestoreTest once the operator is implemented. - add(StreamExecLateralSnapshotJoin.class); - /** Ignoring python based exec nodes temporarily. */ add(StreamExecPythonCalc.class); add(StreamExecPythonCorrelate.class); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.java index b409e02cf9db97..93a9c5b937d4e1 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.java @@ -30,6 +30,7 @@ import java.util.Collections; +import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for model table value function in stream mode. */ @@ -76,8 +77,7 @@ public void setup() { @Test public void testInputTableIsInsertOnlyStream() { String sql = - "SELECT *\n" - + "FROM TABLE(ML_PREDICT(TABLE MyTable, MODEL MyModel, DESCRIPTOR(a, b)))"; + "SELECT *\n" + "FROM ML_PREDICT(TABLE MyTable, MODEL MyModel, DESCRIPTOR(a, b))"; util.verifyRelPlan( sql, JavaScalaConversionUtil.toScala( @@ -103,4 +103,30 @@ public void testInputTableIsCdcStream() { .hasMessageContaining( "StreamPhysicalMLPredictTableFunction doesn't support consuming update and delete changes which is produced by node TableSourceScan(table=[[default_catalog, default_database, CdcTable]], fields=[a, b])"); } + + @Test + void testOnTimeArgumentNotAllowed() { + // ML_PREDICT disables the implicit system arguments. Supplying `on_time` must be rejected + // at the SQL level even though ML_PREDICT is handled by a dedicated optimizer rule. + String sql = + "SELECT * FROM ML_PREDICT(INPUT => TABLE MyTable, MODEL => MODEL MyModel, " + + "ARGS => DESCRIPTOR(a, b), on_time => DESCRIPTOR(rowtime))"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .satisfies( + anyCauseMatches( + "The 'on_time' argument is not supported because function " + + "'ML_PREDICT' does not use system arguments.")); + } + + @Test + void testUidArgumentNotAllowed() { + String sql = + "SELECT * FROM ML_PREDICT(INPUT => TABLE MyTable, MODEL => MODEL MyModel, " + + "ARGS => DESCRIPTOR(a, b), uid => 'my-uid')"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .satisfies( + anyCauseMatches( + "The 'uid' argument is not supported because function " + + "'ML_PREDICT' does not use system arguments.")); + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ProcessTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ProcessTableFunctionTest.java index d34b3f3bc8d992..7500bef9c98ba6 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ProcessTableFunctionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ProcessTableFunctionTest.java @@ -76,6 +76,7 @@ import static org.apache.flink.table.annotation.ArgumentTrait.SET_SEMANTIC_TABLE; import static org.apache.flink.table.annotation.ArgumentTrait.SUPPORT_UPDATES; import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.lit; import static org.apache.flink.table.api.Expressions.row; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -283,6 +284,63 @@ void testNoSystemArgsAllowedForTablePtf() { "Disabling system arguments is not supported for user-defined PTF.")); } + @Test + void testOnTimeArgRejectedForDisabledPtf() { + util.addTemporarySystemFunction("f", NoSystemArgsTableFunction.class); + assertThatThrownBy( + () -> + util.verifyRelPlan( + "SELECT * FROM f(r => TABLE t_watermarked, i => 1, " + + "on_time => DESCRIPTOR(ts));")) + .satisfies( + anyCauseMatches( + "The 'on_time' argument is not supported because function " + + "'f' does not use system arguments.")); + } + + @Test + void testUidArgRejectedForDisabledPtf() { + util.addTemporarySystemFunction("f", NoSystemArgsScalarFunction.class); + assertThatThrownBy(() -> util.verifyRelPlan("SELECT * FROM f(i => 1, uid => 'my-uid');")) + .satisfies( + anyCauseMatches( + "The 'uid' argument is not supported because function " + + "'f' does not use system arguments.")); + } + + @Test + void testSystemArgRejectedByNameBeforeTypeCheck() { + // System arguments are rejected by name, rather than a type mismatch. + util.addTemporarySystemFunction("f", NoSystemArgsTableFunction.class); + assertThatThrownBy( + () -> + util.verifyRelPlan( + "SELECT * FROM f(r => TABLE t, i => 1, on_time => 1);")) + .satisfies( + anyCauseMatches( + "The 'on_time' argument is not supported because function " + + "'f' does not use system arguments.")); + } + + @Test + void testSystemArgRejectedForDisabledPtfViaTableApi() { + // The same enforcement applies to the Table API path, which resolves calls via + // ResolveCallByArgumentsRule instead of the SQL validator. + util.addTemporarySystemFunction("f", NoSystemArgsTableFunction.class); + assertThatThrownBy( + () -> + util.tableEnv() + .fromCall( + "f", + util.tableEnv().from("t").asArgument("r"), + lit(1).asArgument("i"), + lit("my-uid").asArgument("uid"))) + .satisfies( + anyCauseMatches( + "The 'uid' argument is not supported because function " + + "'f' does not use system arguments.")); + } + @Test void testUidPipelineSplitIntoTwoFunctions() { util.addTemporarySystemFunction("f", SetSemanticTableFunction.class); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java index 6d254b220b86f8..eef09609aebd90 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java @@ -24,7 +24,6 @@ import org.apache.flink.table.planner.utils.TableTestUtil; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches; @@ -121,9 +120,6 @@ void testSnapshotWithViewArgument() { } @Test - @Disabled( - "SNAPSHOT sets disableSystemArguments(true), but that flag is currently not enforced. " - + "Re-enable once FLINK-40079 is fixed.") void testSystemArgumentsNotAllowed() { // SNAPSHOT disables the implicit system arguments (e.g. `on_time`). Passing one in a // LATERAL context must be rejected because the argument is not part of the function @@ -136,7 +132,10 @@ void testSystemArgumentsNotAllowed() { + "input => TABLE Rates, " + "on_time => DESCRIPTOR(rate_time))) AS r " + "WHERE o.currency = r.currency")) - .satisfies(anyCauseMatches("on_time")); + .satisfies( + anyCauseMatches( + "The 'on_time' argument is not supported because function " + + "'SNAPSHOT' does not use system arguments.")); } @Test diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtilTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtilTest.java new file mode 100644 index 00000000000000..fb1dd9649180bc --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtilTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.utils; + +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; +import org.apache.flink.table.planner.calcite.FlinkTypeSystem; +import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; + +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.BasicSqlType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link LateralSnapshotJoinUtil}. */ +class LateralSnapshotJoinUtilTest { + + private FlinkTypeFactory typeFactory; + + @BeforeEach + void setup() { + typeFactory = + new FlinkTypeFactory( + Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + } + + @ParameterizedTest(name = "nullableCols={0}, leftOuter={1}") + @CsvSource({"true, true", "true, false", "false, true", "false, false"}) + void testDeriveRowType(boolean nullableCols, boolean leftOuter) { + // Probe (left) side: a scalar column and a rowtime attribute (both forwarded unchanged -- + // left time attributes are not materialized). + final TimeIndicatorRelDataType leftRowtime = rowtime(nullableCols); + final RelDataType leftType = + typeFactory + .builder() + .add("pk", varchar(nullableCols)) + .add("pts", leftRowtime) + .build(); + + // Build (right) side: a scalar column and a rowtime attribute whose originalType has the + // same nullability as the indicator. + final TimeIndicatorRelDataType buildRowtime = rowtime(nullableCols); + final RelDataType rightType = + typeFactory + .builder() + .add("bk", varchar(nullableCols)) + .add("bts", buildRowtime) + .build(); + + final RelDataType rowType = + LateralSnapshotJoinUtil.deriveRowType( + typeFactory, + leftType, + rightType, + leftOuter ? JoinRelType.LEFT : JoinRelType.INNER, + Collections.emptyList()); + + final List fields = rowType.getFieldList(); + assertThat(fields) + .extracting(RelDataTypeField::getName) + .containsExactly("pk", "pts", "bk", "bts"); + + // Left fields are forwarded unchanged (left is never the null-padded side of INNER/LEFT). + assertThat(fields.get(0).getType()) + .as("left scalar forwarded unchanged") + .isEqualTo(leftType.getFieldList().get(0).getType()); + assertThat(fields.get(1).getType()) + .as("left time attribute forwarded unchanged (still a time indicator)") + .isInstanceOf(TimeIndicatorRelDataType.class) + .isEqualTo(leftRowtime); + + // Build-side scalar: base type preserved; nullable iff the column is nullable or it is + // null-padded by a LEFT join. + assertThat(fields.get(2).getType().getSqlTypeName()).isEqualTo(SqlTypeName.VARCHAR); + assertThat(fields.get(2).getType().isNullable()) + .as("build-side scalar nullability") + .isEqualTo(nullableCols || leftOuter); + + // Build-side time attribute: materialized to a regular timestamp (no longer an indicator); + // nullable iff the column is nullable or it is null-padded by a LEFT join. + assertThat(fields.get(3).getType()) + .as("build-side time attribute is materialized") + .isNotInstanceOf(TimeIndicatorRelDataType.class); + assertThat(fields.get(3).getType().getSqlTypeName()).isEqualTo(SqlTypeName.TIMESTAMP); + assertThat(fields.get(3).getType().isNullable()) + .as("materialized build-side rowtime nullability") + .isEqualTo(nullableCols || leftOuter); + } + + /** + * Regression guard for the case where a build-side time attribute's {@code originalType} + * nullability does NOT match the indicator's. {@code + * FlinkTypeFactory#createTypeWithNullability} can widen a NOT NULL rowtime to nullable by + * flipping only the indicator, leaving {@code originalType} inconsistent. {@link + * LateralSnapshotJoinUtil#deriveRowType} must materialize using the indicator's nullability, + * not {@code originalType}'s. + */ + @ParameterizedTest(name = "nullableTimeCol={0}") + @ValueSource(booleans = {true, false}) + void testDeriveRowTypeIgnoresStaleOriginalTypeNullability(boolean nullableTimeCol) { + // originalType is deliberately the opposite nullability of the indicator. + final TimeIndicatorRelDataType buildRowtime = rowtime(nullableTimeCol, !nullableTimeCol); + assertThat(buildRowtime.isNullable()).isEqualTo(nullableTimeCol); + assertThat(buildRowtime.getOriginalType().isNullable()).isEqualTo(!nullableTimeCol); + + final RelDataType leftType = + typeFactory + .builder() + .add("pk", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .build(); + final RelDataType rightType = + typeFactory + .builder() + .add("bk", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("bts", buildRowtime) + .build(); + + final RelDataType rowType = + LateralSnapshotJoinUtil.deriveRowType( + typeFactory, + leftType, + rightType, + JoinRelType.INNER, + Collections.emptyList()); + + final RelDataTypeField bts = rowType.getField("bts", true, false); + assertThat(bts).isNotNull(); + assertThat(bts.getType()) + .as("build-side time attribute is materialized") + .isNotInstanceOf(TimeIndicatorRelDataType.class); + // Nullability must follow the indicator, not the (opposite) originalType. + assertThat(bts.getType().isNullable()) + .as("materialized rowtime uses indicator nullability, not originalType") + .isEqualTo(nullableTimeCol); + } + + private RelDataType varchar(boolean nullable) { + return typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARCHAR), nullable); + } + + private TimeIndicatorRelDataType rowtime(boolean nullable) { + return rowtime(nullable, nullable); + } + + /** + * Builds an event-time {@link TimeIndicatorRelDataType} whose indicator and underlying {@code + * originalType} nullabilities can be set independently. + */ + private TimeIndicatorRelDataType rowtime( + boolean indicatorNullable, boolean originalTypeNullable) { + final BasicSqlType originalType = + (BasicSqlType) + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.TIMESTAMP, 3), + originalTypeNullable); + return new TimeIndicatorRelDataType( + typeFactory.getTypeSystem(), + originalType, + indicatorNullable, + /* isEventTime= */ true); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/CreateConnectionITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/CreateConnectionITCase.java new file mode 100644 index 00000000000000..a9d99cbfe5d328 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/CreateConnectionITCase.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.batch.sql; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.internal.TableEnvironmentInternal; +import org.apache.flink.table.catalog.CatalogManager; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.planner.runtime.utils.BatchTestBase; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.entry; + +/** IT case for CREATE CONNECTION statement. */ +class CreateConnectionITCase extends BatchTestBase { + + @Test + void testCreateTemporaryConnection() { + tEnv().executeSql( + "CREATE TEMPORARY CONNECTION my_conn COMMENT 'hi there' " + + "WITH ('k' = 'v')"); + + assertThat(catalogManager().getConnection(connectionIdentifier("my_conn"))) + .hasValueSatisfying( + connection -> { + assertThat(connection.getOptions()).containsOnly(entry("k", "v")); + assertThat(connection.getComment()).isEqualTo("hi there"); + }); + } + + @Test + void testCreateTemporaryConnectionRejectsDuplicate() { + tEnv().executeSql("CREATE TEMPORARY CONNECTION my_conn WITH ('k' = 'v1')"); + + assertThatThrownBy( + () -> + tEnv().executeSql( + "CREATE TEMPORARY CONNECTION my_conn WITH ('k' = 'v2')")) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("Temporary connection"); + + tEnv().executeSql("CREATE TEMPORARY CONNECTION IF NOT EXISTS my_conn WITH ('k' = 'v2')"); + + assertThat(catalogManager().getConnection(connectionIdentifier("my_conn"))) + .hasValueSatisfying( + connection -> + assertThat(connection.getOptions()).containsOnly(entry("k", "v1"))); + } + + @Test + void testCreatePermanentConnectionRejectedWithoutSecretStore() { + assertThatThrownBy(() -> tEnv().executeSql("CREATE CONNECTION my_conn WITH ('k' = 'v')")) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("WritableSecretStore must be configured"); + } + + private CatalogManager catalogManager() { + return ((TableEnvironmentInternal) tEnv()).getCatalogManager(); + } + + private ObjectIdentifier connectionIdentifier(String connectionName) { + CatalogManager catalogManager = catalogManager(); + return ObjectIdentifier.of( + catalogManager.getCurrentCatalog(), + catalogManager.getCurrentDatabase(), + connectionName); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java new file mode 100644 index 00000000000000..58632e44326543 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.stream.sql.join; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.TableDescriptor; +import org.apache.flink.table.planner.factories.TestValuesTableFactory; +import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase; +import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameters; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; +import org.apache.flink.util.CollectionUtil; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThatList; + +/** + * Result tests for the {@code LATERAL SNAPSHOT} processing-time temporal join that are + * non-deterministic or benefit from dual-backend (HEAP/ROCKSDB) coverage. Deterministic, + * backend-agnostic cases live in {@code LateralSnapshotJoinSemanticTests}; time-driven behavior + * (idle-timeout flip, state-TTL eviction) in {@code LateralSnapshotJoinOperatorTest}. + * + *

To stabilize results, each build source appends a non-matching "flip-trigger" row at {@link + * #MID_FLIP} (after all real rows); its per-record watermark flips the operator to the JOIN phase. + */ +@ExtendWith(ParameterizedTestExtension.class) +public class LateralSnapshotJoinITCase extends StreamingWithStateTestBase { + + /** The {@code 'user_time'} condition reached mid-stream by the build-side flip-trigger row. */ + private static final String MID_FLIP = + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2020-01-01 00:00:10' AS TIMESTAMP_LTZ(3))"; + + /** Event time of the flip-trigger row; equal to the {@link #MID_FLIP} timestamp. */ + private static final String FLIP_TRIGGER_TS = "00:00:10"; + + /** A build-side key that never matches any probe row. */ + private static final String FLIP_TRIGGER_KEY = "__flip_trigger__"; + + public LateralSnapshotJoinITCase(StateBackendMode state) { + super(state); + } + + @BeforeEach + @Override + public void before() { + super.before(); + env().setParallelism(1); + tEnv().getConfig().setLocalTimeZone(ZoneId.of("UTC")); + } + + @Parameters(name = "StateBackend={0}") + public static Collection parameters() { + return Arrays.asList( + new Object[][] { + {StreamingWithStateTestBase.HEAP_BACKEND()}, + {StreamingWithStateTestBase.ROCKSDB_BACKEND()} + }); + } + + // ------------------------------------------------------------------------------------------ + // Build-side changelog consolidation (all changes have rowtime < the flip ts) + // ------------------------------------------------------------------------------------------ + + @TestTemplate + void testRetractingBuildChangelog() { + createProbe( + Arrays.asList(Row.of("a", 100, ts("00:01:00")), Row.of("b", 200, ts("00:01:01")))); + // Key a is updated 10 -> 11; key b is inserted then deleted. + createChangelogBuild( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "a", 10, ts("00:00:01")), + Row.ofKind(RowKind.INSERT, "b", 20, ts("00:00:03")), + Row.ofKind(RowKind.UPDATE_BEFORE, "a", 10, ts("00:00:01")), + Row.ofKind(RowKind.UPDATE_AFTER, "a", 11, ts("00:00:02")), + Row.ofKind(RowKind.DELETE, "b", 20, ts("00:00:03")))); + + final List actual = + collect( + "SELECT probe.pk, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_FLIP + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual).containsExactlyInAnyOrder(Row.of("a", 11)); + } + + @TestTemplate + void testUpsertBuildSource() { + createProbe( + Arrays.asList(Row.of("a", 100, ts("00:01:00")), Row.of("b", 200, ts("00:01:01")))); + // Upsert source (I,UA,D with PK): a is upserted 10 -> 11, b is deleted. + createUpsertBuild( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "a", 10, ts("00:00:01")), + Row.ofKind(RowKind.UPDATE_AFTER, "a", 11, ts("00:00:02")), + Row.ofKind(RowKind.INSERT, "b", 20, ts("00:00:03")), + Row.ofKind(RowKind.DELETE, "b", 20, ts("00:00:04")))); + + final List actual = + collect( + "SELECT probe.pk, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_FLIP + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual).containsExactlyInAnyOrder(Row.of("a", 11)); + } + + // ------------------------------------------------------------------------------------------ + // Non-deterministic behavior (asserted with tolerant checks) + // ------------------------------------------------------------------------------------------ + + @TestTemplate + void testProbesObserveProgressiveBuildVersions() { + // Throttled build applies v2, v3, v4 progressively after the flip while a throttled probe + // stream spans the progression. Observed versions are non-decreasing; the first probe sees + // v1 and the last sees v4. Exact per-probe versions are not asserted (source timing and + // post-flip visibility latency are not precise enough across environments). + final int probeCount = 8; + final List probes = + IntStream.range(0, probeCount) + .mapToObj(i -> Row.of("k", i, ts(String.format("00:00:%02d", i)))) + .collect(Collectors.toList()); + createProbe(probes, 200L); // last probe at ~1400 ms, well past the last post-flip update + createUpsertBuild( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "k", 1, ts("00:00:01")), + Row.ofKind(RowKind.INSERT, FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS)), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 2, ts("00:00:20")), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 3, ts("00:00:30")), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 4, ts("00:00:40"))), + 50L); + + final List byProbeId = + sortedByProbeId( + collect( + "SELECT probe.pv, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_FLIP + + ")) AS s ON probe.pk = s.bk")); + + assertThat(byProbeId).hasSize(probeCount); + assertMonotonicVersions(byProbeId); + assertThat((Integer) byProbeId.get(0).getField(1)).isEqualTo(1); + assertThat((Integer) byProbeId.get(probeCount - 1).getField(1)).isEqualTo(4); + } + + // ------------------------------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------------------------------ + + private List collect(String query) { + return CollectionUtil.iteratorToList(tEnv().executeSql(query).collect()); + } + + /** + * Appends a non-matching build row at the {@link #MID_FLIP} timestamp; its watermark flips the + * operator to the JOIN phase mid-stream (all real build rows are earlier). + */ + private List withFlipTrigger(List data) { + final List withTrigger = new ArrayList<>(data); + withTrigger.add(Row.of(FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS))); + return withTrigger; + } + + private void createProbe(List data) { + createProbe(data, 0L); + } + + private void createProbe(List data, long sleepMillis) { + final String id = TestValuesTableFactory.registerData(data); + final Schema schema = + Schema.newBuilder() + .column("pk", DataTypes.STRING()) + .column("pv", DataTypes.INT()) + .column("pts", DataTypes.TIMESTAMP(3)) + .watermark("pts", "pts") + .build(); + tEnv().createTable("probe", valuesDescriptor(schema, id, sleepMillis).build()); + } + + private void createChangelogBuild(List data) { + final String id = TestValuesTableFactory.registerData(withFlipTrigger(data)); + final Schema schema = + Schema.newBuilder() + .column("bk", DataTypes.STRING()) + .column("bv", DataTypes.INT()) + .column("bts", DataTypes.TIMESTAMP(3)) + .watermark("bts", "bts") + .build(); + tEnv().createTable( + "b", + valuesDescriptor(schema, id, 0L) + .option("changelog-mode", "I,UB,UA,D") + .build()); + } + + private void createUpsertBuild(List data) { + createUpsertBuild(withFlipTrigger(data), 0L); + } + + private void createUpsertBuild(List data, long sleepMillis) { + final String id = TestValuesTableFactory.registerData(data); + final Schema schema = + Schema.newBuilder() + .column("bk", DataTypes.STRING().notNull()) + .column("bv", DataTypes.INT()) + .column("bts", DataTypes.TIMESTAMP(3)) + .watermark("bts", "bts") + .primaryKey("bk") + .build(); + tEnv().createTable( + "b", + valuesDescriptor(schema, id, sleepMillis) + .option("changelog-mode", "I,UA,D") + .build()); + } + + private static TableDescriptor.Builder valuesDescriptor( + Schema schema, String id, long sleepMillis) { + final TableDescriptor.Builder descriptor = + TableDescriptor.forConnector("values") + .schema(schema) + .option("bounded", "false") + .option("disable-lookup", "true") + .option("enable-watermark-push-down", "true") + .option("scan.watermark.emit.strategy", "on-event") + .option("data-id", id); + if (sleepMillis > 0) { + descriptor + .option("source.sleep-after-elements", "1") + .option("source.sleep-time", sleepMillis + "ms"); + } + return descriptor; + } + + /** Sorts join results (pv, bv) by the probe id pv, i.e. into probe processing order. */ + private static List sortedByProbeId(List results) { + return results.stream() + .sorted(Comparator.comparingInt(r -> r.getFieldAs(0))) + .collect(Collectors.toList()); + } + + /** Asserts the build version (field 1) is non-decreasing across the probe-ordered results. */ + private static void assertMonotonicVersions(List byProbeId) { + int previous = Integer.MIN_VALUE; + for (Row r : byProbeId) { + final int version = r.getFieldAs(1); + assertThat(version).isGreaterThanOrEqualTo(previous); + previous = version; + } + } + + private static LocalDateTime ts(String time) { + return LocalDateTime.parse("2020-01-01T" + time); + } +} diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/LateralSnapshotJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/LateralSnapshotJoinTest.xml new file mode 100644 index 00000000000000..5375c4f7eb30ba --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/LateralSnapshotJoinTest.xml @@ -0,0 +1,191 @@ + + + + + + TABLE b_proctime)) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b_no_wm)) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk AND probe.pv = s.bv]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk AND probe.pv > s.bv]]> + + + ($1, $4))], joinType=[inner]) + :- LogicalTableScan(table=[[default_catalog, default_database, probe]]) + +- LogicalTableFunctionScan(invocation=[SNAPSHOT(TABLE(#0), _UTF-16LE'user_time', CAST(2026-07-01 00:00:00):TIMESTAMP_WITH_LOCAL_TIME_ZONE(3) NOT NULL, DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) bk, INTEGER bv, TIMESTAMP(3) bts)]) + +- LogicalProject(bk=[$0], bv=[$1], bts=[$2]) + +- LogicalTableScan(table=[[default_catalog, default_database, b]]) +]]> + + + (pv, bv))], select=[pk, pv, pts, bk, bv, bts], build=[right]) +:- Exchange(distribution=[hash[pk]]) +: +- TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts]) ++- Exchange(distribution=[hash[bk]]) + +- TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts]) +]]> + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest.xml new file mode 100644 index 00000000000000..3df60f3cb8d756 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest.xml @@ -0,0 +1,119 @@ + + + + + + + + + =($4, -($9, 10000:INTERVAL SECOND)), <=($4, +($9, 3600000:INTERVAL HOUR)))], joinType=[left], joinHints=[[[EARLY_FIRE inheritPath:[0] options:{delay=5s, time-mode=rowtime}]]]) + :- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[$4]) + : +- LogicalProject(a=[$0], b=[$1], c=[$2], proctime=[PROCTIME()], rowtime=[$3]) + : +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[$4]) + +- LogicalProject(a=[$0], b=[$1], c=[$2], proctime=[PROCTIME()], rowtime=[$3]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]]) +]]> + + + = (rowtime0 - 10000:INTERVAL SECOND)) AND (rowtime <= (rowtime0 + 3600000:INTERVAL HOUR)))], select=[a, rowtime, a0, b, rowtime0], earlyFireDelay=[5000], earlyFireTimeMode=[ROWTIME]) + :- Exchange(distribution=[hash[a]]) + : +- WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime]) + : +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime]) + +- Exchange(distribution=[hash[a]]) + +- WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable2, project=[a, b, rowtime], metadata=[]]], fields=[a, b, rowtime]) +]]> + + + + + + + + =($3, -($8, 3600000:INTERVAL HOUR)), <=($3, +($8, 3600000:INTERVAL HOUR)))], joinType=[left], joinHints=[[[EARLY_FIRE inheritPath:[0] options:{delay=5s}]]]) + :- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[$4]) + : +- LogicalProject(a=[$0], b=[$1], c=[$2], proctime=[PROCTIME()], rowtime=[$3]) + : +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[$4]) + +- LogicalProject(a=[$0], b=[$1], c=[$2], proctime=[PROCTIME()], rowtime=[$3]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]]) +]]> + + + = (proctime0 - 3600000:INTERVAL HOUR)) AND (proctime <= (proctime0 + 3600000:INTERVAL HOUR)))], select=[a, proctime, a0, b, proctime0], earlyFireDelay=[5000], earlyFireTimeMode=[PROCTIME]) + :- Exchange(distribution=[hash[a]]) + : +- Calc(select=[a, proctime]) + : +- WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime]) + : +- Calc(select=[a, PROCTIME() AS proctime, rowtime]) + : +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime]) + +- Exchange(distribution=[hash[a]]) + +- Calc(select=[a, b, proctime]) + +- WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime]) + +- Calc(select=[a, b, PROCTIME() AS proctime, rowtime]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable2, project=[a, b, rowtime], metadata=[]]], fields=[a, b, rowtime]) +]]> + + + + + + + + =($4, -($9, 10000:INTERVAL SECOND)), <=($4, +($9, 3600000:INTERVAL HOUR)))], joinType=[left], joinHints=[[[EARLY_FIRE inheritPath:[0] options:{delay=5s}]]]) + :- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[$4]) + : +- LogicalProject(a=[$0], b=[$1], c=[$2], proctime=[PROCTIME()], rowtime=[$3]) + : +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[$4]) + +- LogicalProject(a=[$0], b=[$1], c=[$2], proctime=[PROCTIME()], rowtime=[$3]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable2]]) +]]> + + + = (rowtime0 - 10000:INTERVAL SECOND)) AND (rowtime <= (rowtime0 + 3600000:INTERVAL HOUR)))], select=[a, rowtime, a0, b, rowtime0], earlyFireDelay=[5000], earlyFireTimeMode=[ROWTIME]) + :- Exchange(distribution=[hash[a]]) + : +- WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime]) + : +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime]) + +- Exchange(distribution=[hash[a]]) + +- WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime]) + +- TableSourceScan(table=[[default_catalog, default_database, MyTable2, project=[a, b, rowtime], metadata=[]]], fields=[a, b, rowtime]) +]]> + + + diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest_jsonplan/testEarlyFireJsonPlanRoundTrip.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest_jsonplan/testEarlyFireJsonPlanRoundTrip.out new file mode 100644 index 00000000000000..2b98b28751eb25 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/hints/stream/EarlyFireJoinHintTest_jsonplan/testEarlyFireJsonPlanRoundTrip.out @@ -0,0 +1,446 @@ +{ + "flinkVersion" : "", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`MyTable`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "a", + "dataType" : "INT" + }, { + "name" : "b", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "c", + "dataType" : "BIGINT" + }, { + "name" : "proctime", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$PROCTIME$1", + "type" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "nullable" : false, + "precision" : 3, + "kind" : "PROCTIME" + } + }, + "serializableString" : "PROCTIME()" + } + }, { + "name" : "rowtime", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "rowtime", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 4, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`rowtime`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + }, + "abilities" : [ { + "type" : "ProjectPushDown", + "projectedFields" : [ [ 0 ], [ 3 ] ], + "producedType" : "ROW<`a` INT, `rowtime` TIMESTAMP(3)> NOT NULL" + }, { + "type" : "ReadingMetadata", + "metadataKeys" : [ ], + "producedType" : "ROW<`a` INT, `rowtime` TIMESTAMP(3)> NOT NULL" + } ] + }, + "outputType" : "ROW<`a` INT, `rowtime` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime])" + }, { + "id" : 2, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 1, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "a", + "fieldType" : "INT" + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "a", + "fieldType" : "INT" + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[a]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`MyTable2`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "a", + "dataType" : "INT" + }, { + "name" : "b", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "c", + "dataType" : "BIGINT" + }, { + "name" : "proctime", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$PROCTIME$1", + "type" : { + "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE", + "nullable" : false, + "precision" : 3, + "kind" : "PROCTIME" + } + }, + "serializableString" : "PROCTIME()" + } + }, { + "name" : "rowtime", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "rowtime", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 4, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`rowtime`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + }, + "abilities" : [ { + "type" : "ProjectPushDown", + "projectedFields" : [ [ 0 ], [ 1 ], [ 3 ] ], + "producedType" : "ROW<`a` INT, `b` VARCHAR(2147483647), `rowtime` TIMESTAMP(3)> NOT NULL" + }, { + "type" : "ReadingMetadata", + "metadataKeys" : [ ], + "producedType" : "ROW<`a` INT, `b` VARCHAR(2147483647), `rowtime` TIMESTAMP(3)> NOT NULL" + } ] + }, + "outputType" : "ROW<`a` INT, `b` VARCHAR(2147483647), `rowtime` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, MyTable2, project=[a, b, rowtime], metadata=[]]], fields=[a, b, rowtime])" + }, { + "id" : 5, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "a", + "fieldType" : "INT" + }, { + "name" : "b", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[rowtime], watermark=[rowtime])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "a", + "fieldType" : "INT" + }, { + "name" : "b", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[a]])" + }, { + "id" : 7, + "type" : "stream-exec-interval-join_1", + "intervalJoinSpec" : { + "joinSpec" : { + "joinType" : "LEFT", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "windowBounds" : { + "isEventTime" : true, + "leftLowerBound" : -10000, + "leftUpperBound" : 3600000, + "leftTimeIndex" : 1, + "rightTimeIndex" : 2 + } + }, + "earlyFireDelay" : 5000, + "earlyFireTimeMode" : "ROWTIME", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "a", + "fieldType" : "INT" + }, { + "name" : "rowtime", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "a0", + "fieldType" : "INT" + }, { + "name" : "b", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "rowtime0", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "IntervalJoin(joinType=[LeftOuterJoin], windowBounds=[isRowTime=true, leftLowerBound=-10000, leftUpperBound=3600000, leftTimeIndex=1, rightTimeIndex=2], where=[((a = a0) AND (rowtime >= (rowtime0 - 10000:INTERVAL SECOND)) AND (rowtime <= (rowtime0 + 3600000:INTERVAL HOUR)))], select=[a, rowtime, a0, b, rowtime0], earlyFireDelay=[5000], earlyFireTimeMode=[ROWTIME])" + }, { + "id" : 8, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "INT" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "VARCHAR(2147483647)" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`a` INT, `b` VARCHAR(2147483647)>", + "description" : "Calc(select=[a, b])" + }, { + "id" : 9, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`MySink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "a", + "dataType" : "INT" + }, { + "name" : "b", + "dataType" : "VARCHAR(2147483647)" + } ] + }, + "options" : { + "connector" : "values" + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`a` INT, `b` VARCHAR(2147483647)>", + "description" : "Sink(table=[default_catalog.default_database.MySink], fields=[a, b])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 8, + "target" : 9, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.xml index b1d51479529028..778451f067e6b4 100644 --- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.xml +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/MLPredictTableFunctionTest.xml @@ -869,7 +869,7 @@ Calc(select=[a, b, c, d, rowtime, PROCTIME_MATERIALIZE(proctime) AS proctime, c0 +FROM ML_PREDICT(TABLE MyTable, MODEL MyModel, DESCRIPTOR(a, b))]]> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe, watermark=[TO_TIMESTAMP(pts_str)], watermarkEmitStrategy=[on-event]]], fields=[pk, pv, pts_str])" + }, { + "id" : 2, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Calc(select=[pk, pv])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`bts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b, watermark=[TO_TIMESTAMP(bts_str)], watermarkEmitStrategy=[on-event]]], fields=[bk, bv, bts_str])" + }, { + "id" : 5, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Calc(select=[bk, bv, Reinterpret(TO_TIMESTAMP(bts_str)) AS bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1577836803000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[(pk = bk)], select=[pk, pv, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1577836803000])" + }, { + "id" : 8, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Calc(select=[pk, pv, bk, bv])" + }, { + "id" : 9, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "upsertMaterializeStrategy" : "VALUE", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, bk, bv])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 8, + "target" : 9, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/savepoint/_metadata new file mode 100644 index 00000000000000..ea9e93db4c1e8b Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/savepoint/_metadata differ diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-load-phase/plan/lateral-snapshot-join-load-phase.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-load-phase/plan/lateral-snapshot-join-load-phase.json new file mode 100644 index 00000000000000..d6799f7ab90571 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-load-phase/plan/lateral-snapshot-join-load-phase.json @@ -0,0 +1,468 @@ +{ + "flinkVersion" : "2.4", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`pts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe, watermark=[TO_TIMESTAMP(pts_str)], watermarkEmitStrategy=[on-event]]], fields=[pk, pv, pts_str])" + }, { + "id" : 2, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Calc(select=[pk, pv])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`bts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b, watermark=[TO_TIMESTAMP(bts_str)], watermarkEmitStrategy=[on-event]]], fields=[bk, bv, bts_str])" + }, { + "id" : 5, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Calc(select=[bk, bv, Reinterpret(TO_TIMESTAMP(bts_str)) AS bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1577836803000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[(pk = bk)], select=[pk, pv, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1577836803000])" + }, { + "id" : 8, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Calc(select=[pk, pv, bk, bv])" + }, { + "id" : 9, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, bk, bv])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 8, + "target" : 9, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-load-phase/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-load-phase/savepoint/_metadata new file mode 100644 index 00000000000000..cf486d2151fa84 Binary files /dev/null and b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-load-phase/savepoint/_metadata differ diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/interval/RowTimeIntervalJoinTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/interval/RowTimeIntervalJoinTest.java index 19a104a652f7ed..d0e6530c190a21 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/interval/RowTimeIntervalJoinTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/interval/RowTimeIntervalJoinTest.java @@ -500,6 +500,9 @@ public void testInterruptibleTimers() throws Exception { final List expectedOutput = new ArrayList<>(); expectedOutput.add(insertRecord(5L, "k1", null, null)); + // Intermediate watermark surfacing progress after firing the first timer, before the + // drain is interrupted by the test's injected mail. + expectedOutput.add(new Watermark(5)); expectedOutput.add(insertRecord(null, null, 6L, "k2")); expectedOutput.add(insertRecord(7L, "k3", null, null)); expectedOutput.add(insertRecord(null, null, 8L, "k4")); diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/temporal/BaseTwoInputStreamOperatorWithStateRetentionTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/temporal/BaseTwoInputStreamOperatorWithStateRetentionTest.java index c0e5f705dc9bdc..0aba8733f1ec05 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/temporal/BaseTwoInputStreamOperatorWithStateRetentionTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/temporal/BaseTwoInputStreamOperatorWithStateRetentionTest.java @@ -94,6 +94,9 @@ void testInterruptibleTimersWithWatermarks() throws Exception { assertThat(output) .containsExactly( firedDesc(0L), + // Intermediate watermark surfacing progress after firing the first of + // the 2 timers due at firstWindowEnd, before the drain is interrupted. + "Watermark@" + (firstWindowEnd.toEpochMilli() - 1), mailDesc(0L), firedDesc(1L), mailDesc(1L), @@ -101,6 +104,7 @@ void testInterruptibleTimersWithWatermarks() throws Exception { firedDesc(0L), mailDesc(0L), firedDesc(1L), + "Watermark@" + (secondWindowEnd.toEpochMilli() - 1), mailDesc(1L), watermarkDesc(secondWindowEnd)); } diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java index 15258ff24cb5d6..9e8e3d43e2b992 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/ProcessSetTableOperatorInterruptibleTimersTest.java @@ -149,15 +149,18 @@ void testTimersThroughProcessTableRunner(boolean interruptibleTimers) throws Exc recordLabel(3000L, null), firedLabel(null, 1000L, 5000L), mailLabel(null, 1000L), + firedLabel(NAMED_TIMER, 1000L, 5000L), + watermarkLabel(999L), + mailLabel(NAMED_TIMER, 1000L), firedLabel(null, 2000L, 5000L), mailLabel(null, 2000L), firedLabel(null, 3000L, 5000L), + watermarkLabel(2999L), mailLabel(null, 3000L), - firedLabel(NAMED_TIMER, 1000L, 5000L), - mailLabel(NAMED_TIMER, 1000L), watermarkLabel(5000L), recordLabel(6000L, 5000L), firedLabel(null, 6000L, 7000L), + watermarkLabel(5999L), mailLabel(null, 6000L), watermarkLabel(7000L)); } else { diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/sort/BaseTemporalSortOperatorTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/sort/BaseTemporalSortOperatorTest.java index 65eaa89001e2a9..05d29a4d6340f5 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/sort/BaseTemporalSortOperatorTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/sort/BaseTemporalSortOperatorTest.java @@ -83,12 +83,18 @@ void testInterruptibleTimersWithWatermarks() throws Exception { assertThat(output) .containsExactly( firedDesc(1000L), + // Intermediate watermarks surfacing progress after each timer fires, + // before the drain is interrupted by the next mail. + watermarkDesc(999L), mailDesc(1000L), firedDesc(2000L), + watermarkDesc(1999L), mailDesc(2000L), firedDesc(3000L), + watermarkDesc(2999L), mailDesc(3000L), firedDesc(4000L), + watermarkDesc(3999L), mailDesc(4000L), watermarkDesc(5000L)); } diff --git a/flink-table/flink-table-runtime/src/test/resources/linked-list-serializer-2.3/serializer-snapshot b/flink-table/flink-table-runtime/src/test/resources/linked-list-serializer-2.3/serializer-snapshot new file mode 100644 index 00000000000000..90bb9910edd9d5 Binary files /dev/null and b/flink-table/flink-table-runtime/src/test/resources/linked-list-serializer-2.3/serializer-snapshot differ diff --git a/flink-table/flink-table-runtime/src/test/resources/linked-list-serializer-2.3/test-data b/flink-table/flink-table-runtime/src/test/resources/linked-list-serializer-2.3/test-data new file mode 100644 index 00000000000000..120473d48e9ca5 Binary files /dev/null and b/flink-table/flink-table-runtime/src/test/resources/linked-list-serializer-2.3/test-data differ diff --git a/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-HEAP-V1-snapshot b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-HEAP-V1-snapshot new file mode 100644 index 00000000000000..17ae15e15afe58 Binary files /dev/null and b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-HEAP-V1-snapshot differ diff --git a/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-HEAP-V2-snapshot b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-HEAP-V2-snapshot new file mode 100644 index 00000000000000..bf92e3b9f5f955 Binary files /dev/null and b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-HEAP-V2-snapshot differ diff --git a/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-ROCKSDB-V1-snapshot b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-ROCKSDB-V1-snapshot new file mode 100644 index 00000000000000..091baae41c88d1 Binary files /dev/null and b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-ROCKSDB-V1-snapshot differ diff --git a/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-ROCKSDB-V2-snapshot b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-ROCKSDB-V2-snapshot new file mode 100644 index 00000000000000..932754ff7f1c82 Binary files /dev/null and b/flink-table/flink-table-runtime/src/test/resources/sink-upsert-materializer/migration-flink-2.3-ROCKSDB-V2-snapshot differ diff --git a/flink-test-utils-parent/flink-migration-test-utils/src/main/resources/most_recently_published_version b/flink-test-utils-parent/flink-migration-test-utils/src/main/resources/most_recently_published_version index c6db680226304b..91eac8d5b56f96 100644 --- a/flink-test-utils-parent/flink-migration-test-utils/src/main/resources/most_recently_published_version +++ b/flink-test-utils-parent/flink-migration-test-utils/src/main/resources/most_recently_published_version @@ -1 +1 @@ -v2_2 +v2_3 diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/extension/eventtime/EventTimeWatermarkHandlerTest.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/extension/eventtime/EventTimeWatermarkHandlerTest.java index 36c987701cd3d7..8e866c2d7e513a 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/extension/eventtime/EventTimeWatermarkHandlerTest.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/datastream/extension/eventtime/EventTimeWatermarkHandlerTest.java @@ -276,6 +276,11 @@ public boolean tryAdvanceWatermark( throw new UnsupportedOperationException(); } + @Override + public long getReachedWatermark() { + throw new UnsupportedOperationException(); + } + @Override public void snapshotToRawKeyedState( KeyedStateCheckpointOutputStream stateCheckpointOutputStream, String operatorName) diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/072d2681-63b7-4c7b-80f6-40bef19f27e3 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/072d2681-63b7-4c7b-80f6-40bef19f27e3 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/072d2681-63b7-4c7b-80f6-40bef19f27e3 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/1b0148d4-1999-4687-969d-22e0e623000f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/1b0148d4-1999-4687-969d-22e0e623000f new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/1b0148d4-1999-4687-969d-22e0e623000f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/1dff9aa3-919c-445c-9020-1ffcf84f8c96 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/1dff9aa3-919c-445c-9020-1ffcf84f8c96 new file mode 100644 index 00000000000000..8093dbf0aa072d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/1dff9aa3-919c-445c-9020-1ffcf84f8c96 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/227f3b81-d934-4ba8-aa2d-b3dcd6bed2d7 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/227f3b81-d934-4ba8-aa2d-b3dcd6bed2d7 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/227f3b81-d934-4ba8-aa2d-b3dcd6bed2d7 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/2f96c9f3-1ba8-440f-9d60-259c817db230 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/2f96c9f3-1ba8-440f-9d60-259c817db230 new file mode 100644 index 00000000000000..ea4ba41338cdf0 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/2f96c9f3-1ba8-440f-9d60-259c817db230 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/35e533a3-7385-47d1-86a4-642f1e33ecaa b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/35e533a3-7385-47d1-86a4-642f1e33ecaa new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/35e533a3-7385-47d1-86a4-642f1e33ecaa differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/3cae0bc8-b3d8-4cb5-84fc-e3e9770af29a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/3cae0bc8-b3d8-4cb5-84fc-e3e9770af29a new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/3cae0bc8-b3d8-4cb5-84fc-e3e9770af29a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/3e30c60b-69d0-429d-b872-e0e65dbdd94b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/3e30c60b-69d0-429d-b872-e0e65dbdd94b new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/3e30c60b-69d0-429d-b872-e0e65dbdd94b differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/432717c3-5612-4571-87db-71017780a90a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/432717c3-5612-4571-87db-71017780a90a new file mode 100644 index 00000000000000..ea4ba41338cdf0 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/432717c3-5612-4571-87db-71017780a90a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/4853f6b6-ad8a-4677-823f-4e2eb2d56828 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/4853f6b6-ad8a-4677-823f-4e2eb2d56828 new file mode 100644 index 00000000000000..8093dbf0aa072d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/4853f6b6-ad8a-4677-823f-4e2eb2d56828 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/685d3dc8-d1fa-4f34-a085-83d7d339a821 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/685d3dc8-d1fa-4f34-a085-83d7d339a821 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/685d3dc8-d1fa-4f34-a085-83d7d339a821 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/6c6b8709-117a-4b3d-bf10-56135a412d7d b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/6c6b8709-117a-4b3d-bf10-56135a412d7d new file mode 100644 index 00000000000000..11ef5c3a9cf3c7 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/6c6b8709-117a-4b3d-bf10-56135a412d7d differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/876da3c2-29b2-4c19-bea0-c6172e3ca685 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/876da3c2-29b2-4c19-bea0-c6172e3ca685 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/876da3c2-29b2-4c19-bea0-c6172e3ca685 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/_metadata new file mode 100644 index 00000000000000..6bbd1920add0b1 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/a8ff8410-6e60-48b1-a959-50a575973f57 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/a8ff8410-6e60-48b1-a959-50a575973f57 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/a8ff8410-6e60-48b1-a959-50a575973f57 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/b02aaf8e-c39c-41b5-868f-4a96919d4b7a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/b02aaf8e-c39c-41b5-868f-4a96919d4b7a new file mode 100644 index 00000000000000..dd330a8e917283 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/b02aaf8e-c39c-41b5-868f-4a96919d4b7a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/b07cb79e-92dd-452a-9d6a-e0c3110f7b3d b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/b07cb79e-92dd-452a-9d6a-e0c3110f7b3d new file mode 100644 index 00000000000000..dd330a8e917283 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/b07cb79e-92dd-452a-9d6a-e0c3110f7b3d differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/e27c68e5-70fa-4c5a-845f-a67d1555c18b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/e27c68e5-70fa-4c5a-845f-a67d1555c18b new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/e27c68e5-70fa-4c5a-845f-a67d1555c18b differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/e91fc44d-2321-4318-bcab-f4791f6e09bd b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/e91fc44d-2321-4318-bcab-f4791f6e09bd new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/e91fc44d-2321-4318-bcab-f4791f6e09bd differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/f17fda57-d832-4d2d-a2f7-e93d3cc5e3bf b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/f17fda57-d832-4d2d-a2f7-e93d3cc5e3bf new file mode 100644 index 00000000000000..11ef5c3a9cf3c7 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/f17fda57-d832-4d2d-a2f7-e93d3cc5e3bf differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/fbeea499-c8a6-4724-abbe-d4e1a0bcddb2 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/fbeea499-c8a6-4724-abbe-d4e1a0bcddb2 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-checkpoint/fbeea499-c8a6-4724-abbe-d4e1a0bcddb2 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/0254150e-e19b-42a5-8b5e-0cd267700a90 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/0254150e-e19b-42a5-8b5e-0cd267700a90 new file mode 100644 index 00000000000000..dd330a8e917283 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/0254150e-e19b-42a5-8b5e-0cd267700a90 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/175bd4b8-0f44-4bd9-a646-bf0232aac778 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/175bd4b8-0f44-4bd9-a646-bf0232aac778 new file mode 100644 index 00000000000000..8093dbf0aa072d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/175bd4b8-0f44-4bd9-a646-bf0232aac778 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/241c0d48-7e88-4f94-8fce-80fbf17c0383 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/241c0d48-7e88-4f94-8fce-80fbf17c0383 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/241c0d48-7e88-4f94-8fce-80fbf17c0383 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/29c5745d-8ad6-4f59-9464-ddd0468077f0 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/29c5745d-8ad6-4f59-9464-ddd0468077f0 new file mode 100644 index 00000000000000..ea4ba41338cdf0 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/29c5745d-8ad6-4f59-9464-ddd0468077f0 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/3cdd3a80-f974-4799-a62c-0d5584b81eca b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/3cdd3a80-f974-4799-a62c-0d5584b81eca new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/3cdd3a80-f974-4799-a62c-0d5584b81eca differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/3f411ee6-daf7-43bf-a75d-470b827c9903 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/3f411ee6-daf7-43bf-a75d-470b827c9903 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/3f411ee6-daf7-43bf-a75d-470b827c9903 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4b1978f4-0aa4-44a7-8ec8-2baf7c18b840 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4b1978f4-0aa4-44a7-8ec8-2baf7c18b840 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4b1978f4-0aa4-44a7-8ec8-2baf7c18b840 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4c52b484-4a4d-406c-8152-cad35d667728 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4c52b484-4a4d-406c-8152-cad35d667728 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4c52b484-4a4d-406c-8152-cad35d667728 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4fb46874-d9f8-4bbe-afd0-ec8d92552380 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4fb46874-d9f8-4bbe-afd0-ec8d92552380 new file mode 100644 index 00000000000000..dd330a8e917283 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/4fb46874-d9f8-4bbe-afd0-ec8d92552380 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/51d94027-781f-4928-b92a-e3f62217ffee b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/51d94027-781f-4928-b92a-e3f62217ffee new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/51d94027-781f-4928-b92a-e3f62217ffee differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/691d0b5e-b9fa-4d8e-930d-889389c6cd4f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/691d0b5e-b9fa-4d8e-930d-889389c6cd4f new file mode 100644 index 00000000000000..8093dbf0aa072d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/691d0b5e-b9fa-4d8e-930d-889389c6cd4f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/_metadata new file mode 100644 index 00000000000000..66053b3b22a618 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/a0d9fe75-f916-46e9-8bcd-542fa1e94823 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/a0d9fe75-f916-46e9-8bcd-542fa1e94823 new file mode 100644 index 00000000000000..11ef5c3a9cf3c7 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/a0d9fe75-f916-46e9-8bcd-542fa1e94823 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/a5ddb891-c417-4496-a106-f98dda39c291 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/a5ddb891-c417-4496-a106-f98dda39c291 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/a5ddb891-c417-4496-a106-f98dda39c291 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c11b2683-674c-4b65-9475-2f00c68344a9 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c11b2683-674c-4b65-9475-2f00c68344a9 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c11b2683-674c-4b65-9475-2f00c68344a9 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c3b57c60-2989-4dec-b8c9-7683e1324de7 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c3b57c60-2989-4dec-b8c9-7683e1324de7 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c3b57c60-2989-4dec-b8c9-7683e1324de7 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c546110f-c69b-4df5-acbe-ba86ef4c0ace b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c546110f-c69b-4df5-acbe-ba86ef4c0ace new file mode 100644 index 00000000000000..ea4ba41338cdf0 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/c546110f-c69b-4df5-acbe-ba86ef4c0ace differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/cfb0a9a3-a903-4e01-b25f-dfcc9b94e88e b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/cfb0a9a3-a903-4e01-b25f-dfcc9b94e88e new file mode 100644 index 00000000000000..11ef5c3a9cf3c7 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/cfb0a9a3-a903-4e01-b25f-dfcc9b94e88e differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ded470f0-de4f-456f-8a40-d2931c643cab b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ded470f0-de4f-456f-8a40-d2931c643cab new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ded470f0-de4f-456f-8a40-d2931c643cab differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/e9264ddd-9dc6-4199-9257-9488c9777950 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/e9264ddd-9dc6-4199-9257-9488c9777950 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/e9264ddd-9dc6-4199-9257-9488c9777950 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/eb28972e-cca9-468c-b6e1-c5546f1abe21 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/eb28972e-cca9-468c-b6e1-c5546f1abe21 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint-native/eb28972e-cca9-468c-b6e1-c5546f1abe21 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/0c598f38-4b73-4bb2-b7ee-26c1ff1900f0 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/0c598f38-4b73-4bb2-b7ee-26c1ff1900f0 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/0c598f38-4b73-4bb2-b7ee-26c1ff1900f0 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/20d3afb2-b900-4e31-b056-5ca08c9ad6b7 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/20d3afb2-b900-4e31-b056-5ca08c9ad6b7 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/20d3afb2-b900-4e31-b056-5ca08c9ad6b7 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/3740676a-9d6a-4a1c-9b16-d44028a588f0 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/3740676a-9d6a-4a1c-9b16-d44028a588f0 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/3740676a-9d6a-4a1c-9b16-d44028a588f0 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/3f69c997-de8e-4284-9db9-6e717b85f25f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/3f69c997-de8e-4284-9db9-6e717b85f25f new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/3f69c997-de8e-4284-9db9-6e717b85f25f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/401a8148-e43c-4ae1-9c77-0f26bd125bc4 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/401a8148-e43c-4ae1-9c77-0f26bd125bc4 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/401a8148-e43c-4ae1-9c77-0f26bd125bc4 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/46cae258-01e6-4861-8b5e-787c84208067 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/46cae258-01e6-4861-8b5e-787c84208067 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/46cae258-01e6-4861-8b5e-787c84208067 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/5fea057a-54ed-4b1d-afa1-210bc98b310a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/5fea057a-54ed-4b1d-afa1-210bc98b310a new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/5fea057a-54ed-4b1d-afa1-210bc98b310a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/6502c41b-d28d-4fdf-8b19-981412484cfd b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/6502c41b-d28d-4fdf-8b19-981412484cfd new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/6502c41b-d28d-4fdf-8b19-981412484cfd differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/65618368-85c8-4fe2-8a32-e2a0f715e53f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/65618368-85c8-4fe2-8a32-e2a0f715e53f new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/65618368-85c8-4fe2-8a32-e2a0f715e53f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/76860612-4931-4996-a221-55654dba3172 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/76860612-4931-4996-a221-55654dba3172 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/76860612-4931-4996-a221-55654dba3172 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/863a5a3c-ff1c-48f1-a342-62f431b626ee b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/863a5a3c-ff1c-48f1-a342-62f431b626ee new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/863a5a3c-ff1c-48f1-a342-62f431b626ee differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/8a747645-d445-470c-8eb3-d8cae8629ca6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/8a747645-d445-470c-8eb3-d8cae8629ca6 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/8a747645-d445-470c-8eb3-d8cae8629ca6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/8ecbdad6-d14a-464c-aaf3-5db55e52c2f6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/8ecbdad6-d14a-464c-aaf3-5db55e52c2f6 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/8ecbdad6-d14a-464c-aaf3-5db55e52c2f6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/_metadata new file mode 100644 index 00000000000000..7273b44f644ed3 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/bdc79a1e-5432-49ec-bb1c-6bc96eee23c6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/bdc79a1e-5432-49ec-bb1c-6bc96eee23c6 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/bdc79a1e-5432-49ec-bb1c-6bc96eee23c6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e148bc0a-2707-4366-9ce6-23a665679a37 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e148bc0a-2707-4366-9ce6-23a665679a37 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e148bc0a-2707-4366-9ce6-23a665679a37 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e23d0934-4fdf-4278-a5a0-1e6ffe4265c0 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e23d0934-4fdf-4278-a5a0-1e6ffe4265c0 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e23d0934-4fdf-4278-a5a0-1e6ffe4265c0 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e3ba9568-1e04-4468-b7f2-3647c91d736e b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e3ba9568-1e04-4468-b7f2-3647c91d736e new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/e3ba9568-1e04-4468-b7f2-3647c91d736e differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/ebdcc597-63e2-48fb-852f-f20a52cf4f2e b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/ebdcc597-63e2-48fb-852f-f20a52cf4f2e new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/ebdcc597-63e2-48fb-852f-f20a52cf4f2e differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/ec8df3dd-6c2a-47f8-b678-d5adc0bafff7 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/ec8df3dd-6c2a-47f8-b678-d5adc0bafff7 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/ec8df3dd-6c2a-47f8-b678-d5adc0bafff7 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/f1530039-b1f6-4625-be1d-ee3e0e5fc315 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/f1530039-b1f6-4625-be1d-ee3e0e5fc315 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-hashmap-savepoint/f1530039-b1f6-4625-be1d-ee3e0e5fc315 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/01197b86-dabe-4c37-8fcf-90fe677c0fcc b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/01197b86-dabe-4c37-8fcf-90fe677c0fcc new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/01197b86-dabe-4c37-8fcf-90fe677c0fcc @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0d3d34c4-343c-42b3-b05f-c0c60c13dde3 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0d3d34c4-343c-42b3-b05f-c0c60c13dde3 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0d3d34c4-343c-42b3-b05f-c0c60c13dde3 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1b3a9340-8eb2-44df-8e1a-d6e83df16a02 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1b3a9340-8eb2-44df-8e1a-d6e83df16a02 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1b3a9340-8eb2-44df-8e1a-d6e83df16a02 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1b8040a8-ae25-4296-91fd-6e4de18f1501 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1b8040a8-ae25-4296-91fd-6e4de18f1501 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1b8040a8-ae25-4296-91fd-6e4de18f1501 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1d60e78f-1b06-4c8c-8e44-058f8bf49f7c b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1d60e78f-1b06-4c8c-8e44-058f8bf49f7c new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1d60e78f-1b06-4c8c-8e44-058f8bf49f7c differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/219b149c-ac5c-4b3c-9f8f-415028d63b82 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/219b149c-ac5c-4b3c-9f8f-415028d63b82 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/219b149c-ac5c-4b3c-9f8f-415028d63b82 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2c78b5e2-da30-41c6-bfbc-b203d506365c b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2c78b5e2-da30-41c6-bfbc-b203d506365c new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2c78b5e2-da30-41c6-bfbc-b203d506365c differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2dcc0143-a438-4104-ab36-ddad184fda25 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2dcc0143-a438-4104-ab36-ddad184fda25 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2dcc0143-a438-4104-ab36-ddad184fda25 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/36bfc42a-6d2b-4a1c-bcc7-016cd97900ad b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/36bfc42a-6d2b-4a1c-bcc7-016cd97900ad new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/36bfc42a-6d2b-4a1c-bcc7-016cd97900ad @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/401683f2-6977-4cde-9a24-2aa6cc363d4a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/401683f2-6977-4cde-9a24-2aa6cc363d4a new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/401683f2-6977-4cde-9a24-2aa6cc363d4a @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/432949de-9ad7-4ee6-a83b-961c162df0fb b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/432949de-9ad7-4ee6-a83b-961c162df0fb new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/432949de-9ad7-4ee6-a83b-961c162df0fb differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4a6a884b-1027-4697-a39b-56cc7a1a948e b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4a6a884b-1027-4697-a39b-56cc7a1a948e new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4a6a884b-1027-4697-a39b-56cc7a1a948e differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/532e33e2-cba6-41f5-ab2f-2c539c17eb5d b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/532e33e2-cba6-41f5-ab2f-2c539c17eb5d new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/532e33e2-cba6-41f5-ab2f-2c539c17eb5d differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/58138011-d05c-491a-8f7d-72e233a73ab1 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/58138011-d05c-491a-8f7d-72e233a73ab1 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/58138011-d05c-491a-8f7d-72e233a73ab1 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5b450560-72ea-4daf-8056-faae820d3245 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5b450560-72ea-4daf-8056-faae820d3245 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5b450560-72ea-4daf-8056-faae820d3245 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/643281f6-6893-4b63-8d07-01493527f163 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/643281f6-6893-4b63-8d07-01493527f163 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/643281f6-6893-4b63-8d07-01493527f163 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65449348-b3e4-4b73-b59c-2e5e3b930ba1 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65449348-b3e4-4b73-b59c-2e5e3b930ba1 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65449348-b3e4-4b73-b59c-2e5e3b930ba1 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6bf4774f-25b1-4059-b557-07b2f6dfb820 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6bf4774f-25b1-4059-b557-07b2f6dfb820 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6bf4774f-25b1-4059-b557-07b2f6dfb820 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6d64adea-4d82-481f-a74b-2387d20fe413 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6d64adea-4d82-481f-a74b-2387d20fe413 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6d64adea-4d82-481f-a74b-2387d20fe413 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6da281da-a137-4bb9-a970-865d55de12bb b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6da281da-a137-4bb9-a970-865d55de12bb new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6da281da-a137-4bb9-a970-865d55de12bb @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/769640d8-9e47-445e-bfab-6d9584ed4a5c b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/769640d8-9e47-445e-bfab-6d9584ed4a5c new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/769640d8-9e47-445e-bfab-6d9584ed4a5c differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/769c6c3f-47ff-4428-96a0-087ddf701f40 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/769c6c3f-47ff-4428-96a0-087ddf701f40 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/769c6c3f-47ff-4428-96a0-087ddf701f40 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/78d26e5c-c70a-432b-b78b-2c14a5f4cb7a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/78d26e5c-c70a-432b-b78b-2c14a5f4cb7a new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/78d26e5c-c70a-432b-b78b-2c14a5f4cb7a @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/7ea2ec98-d255-448b-a70b-ca3c4c8115fc b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/7ea2ec98-d255-448b-a70b-ca3c4c8115fc new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/7ea2ec98-d255-448b-a70b-ca3c4c8115fc differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/91be4aef-69d0-4a1b-a855-edc760636719 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/91be4aef-69d0-4a1b-a855-edc760636719 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/91be4aef-69d0-4a1b-a855-edc760636719 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/951de9c3-ba23-4036-a5c8-fbd0dc7bceb4 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/951de9c3-ba23-4036-a5c8-fbd0dc7bceb4 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/951de9c3-ba23-4036-a5c8-fbd0dc7bceb4 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/_metadata new file mode 100644 index 00000000000000..d6d97c3394c85e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a9b395c2-ddbc-4501-bcd8-3fc8bb87e217 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a9b395c2-ddbc-4501-bcd8-3fc8bb87e217 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a9b395c2-ddbc-4501-bcd8-3fc8bb87e217 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b5c838c7-d458-40c6-9fe0-683a6f4fc58b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b5c838c7-d458-40c6-9fe0-683a6f4fc58b new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b5c838c7-d458-40c6-9fe0-683a6f4fc58b @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c0297b23-1374-471d-9cc7-120a1e825b74 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c0297b23-1374-471d-9cc7-120a1e825b74 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c0297b23-1374-471d-9cc7-120a1e825b74 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c09ec99f-89fd-4786-a535-701f6b6794b6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c09ec99f-89fd-4786-a535-701f6b6794b6 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c09ec99f-89fd-4786-a535-701f6b6794b6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c1c6fa62-9f6b-4fce-bc05-6c0888dc5681 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c1c6fa62-9f6b-4fce-bc05-6c0888dc5681 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c1c6fa62-9f6b-4fce-bc05-6c0888dc5681 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d0ebb7b8-ff96-40d7-a8e1-162949ed36ea b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d0ebb7b8-ff96-40d7-a8e1-162949ed36ea new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d0ebb7b8-ff96-40d7-a8e1-162949ed36ea differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d89254ac-1992-41cb-9905-017e78adcab3 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d89254ac-1992-41cb-9905-017e78adcab3 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d89254ac-1992-41cb-9905-017e78adcab3 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/db593648-50aa-49b3-ad55-9e497de5e369 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/db593648-50aa-49b3-ad55-9e497de5e369 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/db593648-50aa-49b3-ad55-9e497de5e369 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e2d0994d-fe90-4051-8dc9-6f5f418c140d b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e2d0994d-fe90-4051-8dc9-6f5f418c140d new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e2d0994d-fe90-4051-8dc9-6f5f418c140d @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e5a904c3-56b5-4965-8b26-99373feaebb0 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e5a904c3-56b5-4965-8b26-99373feaebb0 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e5a904c3-56b5-4965-8b26-99373feaebb0 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e681adcf-84e7-40f5-9a04-0292f75db142 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e681adcf-84e7-40f5-9a04-0292f75db142 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e681adcf-84e7-40f5-9a04-0292f75db142 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e7822fdd-7038-4066-b5f6-cf06b8d09500 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e7822fdd-7038-4066-b5f6-cf06b8d09500 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e7822fdd-7038-4066-b5f6-cf06b8d09500 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e8322799-4490-42b9-a653-6d2425bcfb42 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e8322799-4490-42b9-a653-6d2425bcfb42 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e8322799-4490-42b9-a653-6d2425bcfb42 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ea779b8b-8d8e-4ab6-bac6-395eb429b749 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ea779b8b-8d8e-4ab6-bac6-395eb429b749 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ea779b8b-8d8e-4ab6-bac6-395eb429b749 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f5014293-9647-47ab-8685-fec899807da2 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f5014293-9647-47ab-8685-fec899807da2 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f5014293-9647-47ab-8685-fec899807da2 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f6a6fbb1-d2c0-459f-9079-e783131cd2f4 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f6a6fbb1-d2c0-459f-9079-e783131cd2f4 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f6a6fbb1-d2c0-459f-9079-e783131cd2f4 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fa00dc04-f764-442f-ab24-00312bc33788 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fa00dc04-f764-442f-ab24-00312bc33788 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fa00dc04-f764-442f-ab24-00312bc33788 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fa27bf38-eeff-46a6-90ef-abc2dafc2a48 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fa27bf38-eeff-46a6-90ef-abc2dafc2a48 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fa27bf38-eeff-46a6-90ef-abc2dafc2a48 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/01fb91fc-8e84-4e12-8a44-bb2b546b4787 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/01fb91fc-8e84-4e12-8a44-bb2b546b4787 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/01fb91fc-8e84-4e12-8a44-bb2b546b4787 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0732b46b-af1e-49e8-b7c4-c71c265c3755 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0732b46b-af1e-49e8-b7c4-c71c265c3755 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0732b46b-af1e-49e8-b7c4-c71c265c3755 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0820f20c-f077-4d74-97a4-65f3999758b1 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0820f20c-f077-4d74-97a4-65f3999758b1 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0820f20c-f077-4d74-97a4-65f3999758b1 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0d3ad900-46cd-42a6-8bb8-a11f885fc90f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0d3ad900-46cd-42a6-8bb8-a11f885fc90f new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0d3ad900-46cd-42a6-8bb8-a11f885fc90f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1038cde8-e97f-4a38-8cdf-03af8145b6d7 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1038cde8-e97f-4a38-8cdf-03af8145b6d7 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1038cde8-e97f-4a38-8cdf-03af8145b6d7 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/135da05c-1a8f-490b-b1ca-500a5811666a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/135da05c-1a8f-490b-b1ca-500a5811666a new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/135da05c-1a8f-490b-b1ca-500a5811666a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/23602bb1-4778-4d05-a53d-e5285b6c4844 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/23602bb1-4778-4d05-a53d-e5285b6c4844 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/23602bb1-4778-4d05-a53d-e5285b6c4844 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/239c5d4f-5a1d-4d82-94cc-8ebee5dd77b1 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/239c5d4f-5a1d-4d82-94cc-8ebee5dd77b1 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/239c5d4f-5a1d-4d82-94cc-8ebee5dd77b1 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/28ade820-f6c1-456c-982d-9292118d8948 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/28ade820-f6c1-456c-982d-9292118d8948 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/28ade820-f6c1-456c-982d-9292118d8948 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3409cc82-59da-4248-8e82-d5b4481e3e9a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3409cc82-59da-4248-8e82-d5b4481e3e9a new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3409cc82-59da-4248-8e82-d5b4481e3e9a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/39f83302-8ef4-4e4e-add2-59c2c34fd59e b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/39f83302-8ef4-4e4e-add2-59c2c34fd59e new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/39f83302-8ef4-4e4e-add2-59c2c34fd59e @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d66b41a-c24a-4b95-8547-51dea66dce54 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d66b41a-c24a-4b95-8547-51dea66dce54 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d66b41a-c24a-4b95-8547-51dea66dce54 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4baf8c14-d5d1-46c6-a22b-6cf01b015fd5 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4baf8c14-d5d1-46c6-a22b-6cf01b015fd5 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4baf8c14-d5d1-46c6-a22b-6cf01b015fd5 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/520f29a7-9ddd-4bac-ba41-550bb701a0ef b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/520f29a7-9ddd-4bac-ba41-550bb701a0ef new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/520f29a7-9ddd-4bac-ba41-550bb701a0ef @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6f7d7bf7-df0d-4488-9c77-a2ffddace92f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6f7d7bf7-df0d-4488-9c77-a2ffddace92f new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6f7d7bf7-df0d-4488-9c77-a2ffddace92f @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/776a81d6-b722-4468-acde-8d1bd79a7e90 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/776a81d6-b722-4468-acde-8d1bd79a7e90 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/776a81d6-b722-4468-acde-8d1bd79a7e90 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/79b9437d-ee76-411a-9759-6e220e5fb58b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/79b9437d-ee76-411a-9759-6e220e5fb58b new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/79b9437d-ee76-411a-9759-6e220e5fb58b @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7db9c167-b8ca-455c-b0ab-aa3612ad1e7d b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7db9c167-b8ca-455c-b0ab-aa3612ad1e7d new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7db9c167-b8ca-455c-b0ab-aa3612ad1e7d differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/833084b5-0e6a-47f7-9c15-fe99efa35dba b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/833084b5-0e6a-47f7-9c15-fe99efa35dba new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/833084b5-0e6a-47f7-9c15-fe99efa35dba @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/83ef3b5d-8be9-47ab-95b4-a498934c73df b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/83ef3b5d-8be9-47ab-95b4-a498934c73df new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/83ef3b5d-8be9-47ab-95b4-a498934c73df differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/87fc8e49-ed6e-4abb-b28f-a64106810954 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/87fc8e49-ed6e-4abb-b28f-a64106810954 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/87fc8e49-ed6e-4abb-b28f-a64106810954 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/980e34c7-19d2-460b-97ca-5ef0935af84f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/980e34c7-19d2-460b-97ca-5ef0935af84f new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/980e34c7-19d2-460b-97ca-5ef0935af84f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/98486669-d06d-4542-b4e7-75a645019924 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/98486669-d06d-4542-b4e7-75a645019924 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/98486669-d06d-4542-b4e7-75a645019924 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9b7ef958-6f0b-4f71-a94d-f46dacbab5b1 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9b7ef958-6f0b-4f71-a94d-f46dacbab5b1 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9b7ef958-6f0b-4f71-a94d-f46dacbab5b1 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9c9b0059-0d9c-4b13-b39d-3b264253f867 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9c9b0059-0d9c-4b13-b39d-3b264253f867 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9c9b0059-0d9c-4b13-b39d-3b264253f867 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9ccbe2cb-48b2-4832-b382-4aa3b977bc0f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9ccbe2cb-48b2-4832-b382-4aa3b977bc0f new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9ccbe2cb-48b2-4832-b382-4aa3b977bc0f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/_metadata new file mode 100644 index 00000000000000..836b309007d4b4 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a2a9d08a-37f0-4e18-ad91-d4daea99360b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a2a9d08a-37f0-4e18-ad91-d4daea99360b new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a2a9d08a-37f0-4e18-ad91-d4daea99360b differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a7886ab2-3f30-42c7-997d-a135d957ef70 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a7886ab2-3f30-42c7-997d-a135d957ef70 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a7886ab2-3f30-42c7-997d-a135d957ef70 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a7a60edd-501b-4239-b25f-6998471bfff4 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a7a60edd-501b-4239-b25f-6998471bfff4 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a7a60edd-501b-4239-b25f-6998471bfff4 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ad98e1eb-4239-4773-b56f-d90baa191c63 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ad98e1eb-4239-4773-b56f-d90baa191c63 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ad98e1eb-4239-4773-b56f-d90baa191c63 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b4e6e54c-7d83-4755-921e-c2f7a4f11374 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b4e6e54c-7d83-4755-921e-c2f7a4f11374 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b4e6e54c-7d83-4755-921e-c2f7a4f11374 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b516b1f8-f3a1-48af-a399-a06dc6925d4c b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b516b1f8-f3a1-48af-a399-a06dc6925d4c new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b516b1f8-f3a1-48af-a399-a06dc6925d4c differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c5904d8e-f8c3-48e1-ba83-d4ddbd34f298 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c5904d8e-f8c3-48e1-ba83-d4ddbd34f298 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c5904d8e-f8c3-48e1-ba83-d4ddbd34f298 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d35d6576-d4d7-47ff-a3ba-47f471a77a7f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d35d6576-d4d7-47ff-a3ba-47f471a77a7f new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d35d6576-d4d7-47ff-a3ba-47f471a77a7f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d7fa4f76-34ac-4d74-b4d1-05f849e50f05 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d7fa4f76-34ac-4d74-b4d1-05f849e50f05 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d7fa4f76-34ac-4d74-b4d1-05f849e50f05 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d92d5d95-5d6e-4b9b-a81d-d13f7d29145a b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d92d5d95-5d6e-4b9b-a81d-d13f7d29145a new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d92d5d95-5d6e-4b9b-a81d-d13f7d29145a differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/db0a028e-b85b-4095-8266-c86722a9f3d4 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/db0a028e-b85b-4095-8266-c86722a9f3d4 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/db0a028e-b85b-4095-8266-c86722a9f3d4 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dc537dc6-012e-49ec-af31-2d19442b4350 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dc537dc6-012e-49ec-af31-2d19442b4350 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dc537dc6-012e-49ec-af31-2d19442b4350 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e0cc4b62-857d-4339-81c2-a286f6bdc860 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e0cc4b62-857d-4339-81c2-a286f6bdc860 new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e0cc4b62-857d-4339-81c2-a286f6bdc860 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ef2429f6-5c70-4717-bd71-225d54589059 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ef2429f6-5c70-4717-bd71-225d54589059 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ef2429f6-5c70-4717-bd71-225d54589059 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f267cf3d-c032-4014-ba7d-1645d4a09bcb b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f267cf3d-c032-4014-ba7d-1645d4a09bcb new file mode 100644 index 00000000000000..ad0dfcf758edc5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f267cf3d-c032-4014-ba7d-1645d4a09bcb @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_user-timers"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_user-timers"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f42dbd75-7256-4fb8-b02e-689381c24aa6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f42dbd75-7256-4fb8-b02e-689381c24aa6 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f42dbd75-7256-4fb8-b02e-689381c24aa6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f544f6ee-24e5-442d-a48b-6e8b0c4e6257 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f544f6ee-24e5-442d-a48b-6e8b0c4e6257 new file mode 100644 index 00000000000000..0785704387c0dd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f544f6ee-24e5-442d-a48b-6e8b0c4e6257 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/fa75483b-2b31-4800-b1a5-e8da5365470c b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/fa75483b-2b31-4800-b1a5-e8da5365470c new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/fa75483b-2b31-4800-b1a5-e8da5365470c differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/0d494c52-b6ba-4a4c-8945-9d005e9076d7 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/0d494c52-b6ba-4a4c-8945-9d005e9076d7 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/0d494c52-b6ba-4a4c-8945-9d005e9076d7 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/1c04996a-c2b0-461e-ba2f-6b9421422eaa b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/1c04996a-c2b0-461e-ba2f-6b9421422eaa new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/1c04996a-c2b0-461e-ba2f-6b9421422eaa differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/23226810-7944-4f58-810b-b67f0e9e4714 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/23226810-7944-4f58-810b-b67f0e9e4714 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/23226810-7944-4f58-810b-b67f0e9e4714 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/2c1e43a4-a684-4c08-93e5-6703d5afc981 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/2c1e43a4-a684-4c08-93e5-6703d5afc981 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/2c1e43a4-a684-4c08-93e5-6703d5afc981 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/4e6ffb17-b32d-42d2-beeb-96b8f8089e20 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/4e6ffb17-b32d-42d2-beeb-96b8f8089e20 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/4e6ffb17-b32d-42d2-beeb-96b8f8089e20 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/51d51ace-9822-4bb6-b545-5826e83d85eb b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/51d51ace-9822-4bb6-b545-5826e83d85eb new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/51d51ace-9822-4bb6-b545-5826e83d85eb differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/53ddf49d-a390-4cb1-96ee-93ce3ee5383b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/53ddf49d-a390-4cb1-96ee-93ce3ee5383b new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/53ddf49d-a390-4cb1-96ee-93ce3ee5383b differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/54532690-d6cf-4018-a474-e1d7381db82b b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/54532690-d6cf-4018-a474-e1d7381db82b new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/54532690-d6cf-4018-a474-e1d7381db82b differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/548cfd37-4e78-4044-90a9-0d25025b9676 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/548cfd37-4e78-4044-90a9-0d25025b9676 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/548cfd37-4e78-4044-90a9-0d25025b9676 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/6130062c-fc07-4fdc-a83d-dc12f1ca43e8 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/6130062c-fc07-4fdc-a83d-dc12f1ca43e8 new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/6130062c-fc07-4fdc-a83d-dc12f1ca43e8 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/74b89435-4357-491b-964e-71ad08801e67 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/74b89435-4357-491b-964e-71ad08801e67 new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/74b89435-4357-491b-964e-71ad08801e67 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/765e995a-2c1b-4d8c-a748-807a7f009d8f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/765e995a-2c1b-4d8c-a748-807a7f009d8f new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/765e995a-2c1b-4d8c-a748-807a7f009d8f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/7b921a60-0480-4cb2-b932-7be5cb9751dc b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/7b921a60-0480-4cb2-b932-7be5cb9751dc new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/7b921a60-0480-4cb2-b932-7be5cb9751dc differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/_metadata new file mode 100644 index 00000000000000..08dd8b7d7e2eac Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/b0596240-bb80-4ed2-982a-70d50266490e b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/b0596240-bb80-4ed2-982a-70d50266490e new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/b0596240-bb80-4ed2-982a-70d50266490e differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/bccf06b8-9cae-4ab7-b1e8-c6934c20fc9f b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/bccf06b8-9cae-4ab7-b1e8-c6934c20fc9f new file mode 100644 index 00000000000000..1aded13f6e4b64 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/bccf06b8-9cae-4ab7-b1e8-c6934c20fc9f differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/c0719ae1-222b-41b5-b6d7-46c9b45c164c b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/c0719ae1-222b-41b5-b6d7-46c9b45c164c new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/c0719ae1-222b-41b5-b6d7-46c9b45c164c differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f4303062-bd50-4a4b-9120-8ecf367c4dea b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f4303062-bd50-4a4b-9120-8ecf367c4dea new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f4303062-bd50-4a4b-9120-8ecf367c4dea differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f68fc0e0-58dd-4dd1-bfbb-c084d30912f6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f68fc0e0-58dd-4dd1-bfbb-c084d30912f6 new file mode 100644 index 00000000000000..89e40fd41805fd Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f68fc0e0-58dd-4dd1-bfbb-c084d30912f6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f8db5709-13f7-4160-ae7e-f30730fd66b6 b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f8db5709-13f7-4160-ae7e-f30730fd66b6 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/f8db5709-13f7-4160-ae7e-f30730fd66b6 differ diff --git a/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/fe68e282-fca1-4546-af59-520965d3f07d b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/fe68e282-fca1-4546-af59-520965d3f07d new file mode 100644 index 00000000000000..a7cffe07e97ea8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-broadcast-udf-migration-itcase-flink2.3-rocksdb-savepoint/fe68e282-fca1-4546-af59-520965d3f07d differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/0fafc3b1-2e14-4e96-a743-cb60274183e8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/0fafc3b1-2e14-4e96-a743-cb60274183e8 new file mode 100644 index 00000000000000..f4a3b3f6a28aaf Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/0fafc3b1-2e14-4e96-a743-cb60274183e8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/1b797e8a-b107-474d-87d7-cabcead69566 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/1b797e8a-b107-474d-87d7-cabcead69566 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/1b797e8a-b107-474d-87d7-cabcead69566 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/1f490290-e01a-49ab-88e7-bcf8ab5013a9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/1f490290-e01a-49ab-88e7-bcf8ab5013a9 new file mode 100644 index 00000000000000..8fc941d2b0310b Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/1f490290-e01a-49ab-88e7-bcf8ab5013a9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/4dbe42e6-3d46-46c7-814b-2920346542c8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/4dbe42e6-3d46-46c7-814b-2920346542c8 new file mode 100644 index 00000000000000..7ca6fef2a8e966 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/4dbe42e6-3d46-46c7-814b-2920346542c8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/52f3cba8-a6cf-4504-bf99-7589c5220091 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/52f3cba8-a6cf-4504-bf99-7589c5220091 new file mode 100644 index 00000000000000..2ee66e2f957a06 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/52f3cba8-a6cf-4504-bf99-7589c5220091 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/7a5ca5f3-c8c0-47b2-a032-be17db22401f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/7a5ca5f3-c8c0-47b2-a032-be17db22401f new file mode 100644 index 00000000000000..6bf035568ab390 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/7a5ca5f3-c8c0-47b2-a032-be17db22401f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/898c9b7d-7420-462e-95f2-3e510a2f8320 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/898c9b7d-7420-462e-95f2-3e510a2f8320 new file mode 100644 index 00000000000000..dde2299078c1fe Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/898c9b7d-7420-462e-95f2-3e510a2f8320 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/9c594fd9-33e6-4e29-a6bf-ead5ebbc9eb3 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/9c594fd9-33e6-4e29-a6bf-ead5ebbc9eb3 new file mode 100644 index 00000000000000..8fc941d2b0310b Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/9c594fd9-33e6-4e29-a6bf-ead5ebbc9eb3 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/9ecc3371-f3e0-45bd-b7b7-edc593bd3663 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/9ecc3371-f3e0-45bd-b7b7-edc593bd3663 new file mode 100644 index 00000000000000..f4a3b3f6a28aaf Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/9ecc3371-f3e0-45bd-b7b7-edc593bd3663 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/_metadata new file mode 100644 index 00000000000000..c3e73fa38778af Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/bdf02913-f03e-4a7e-abb2-5e96cc6e9cba b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/bdf02913-f03e-4a7e-abb2-5e96cc6e9cba new file mode 100644 index 00000000000000..dde2299078c1fe Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/bdf02913-f03e-4a7e-abb2-5e96cc6e9cba differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/c617bc43-e885-4b1c-a3cc-5453afc0b9f6 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/c617bc43-e885-4b1c-a3cc-5453afc0b9f6 new file mode 100644 index 00000000000000..0bc166a72ed948 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/c617bc43-e885-4b1c-a3cc-5453afc0b9f6 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/c750b486-f90f-4df5-9346-5e3677dfac75 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/c750b486-f90f-4df5-9346-5e3677dfac75 new file mode 100644 index 00000000000000..7ca6fef2a8e966 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/c750b486-f90f-4df5-9346-5e3677dfac75 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/d109679b-99ed-45eb-a118-0b031e0ea0b2 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/d109679b-99ed-45eb-a118-0b031e0ea0b2 new file mode 100644 index 00000000000000..2ee66e2f957a06 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/d109679b-99ed-45eb-a118-0b031e0ea0b2 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/dc9d41ba-33e3-4faa-81a8-2d05014eb1c4 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/dc9d41ba-33e3-4faa-81a8-2d05014eb1c4 new file mode 100644 index 00000000000000..6bf035568ab390 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/dc9d41ba-33e3-4faa-81a8-2d05014eb1c4 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/eeed3950-5111-4cb6-9707-51a2eed5207a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/eeed3950-5111-4cb6-9707-51a2eed5207a new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/eeed3950-5111-4cb6-9707-51a2eed5207a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/fbdfafe2-ad27-4f56-8107-7c68ceb0b78e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/fbdfafe2-ad27-4f56-8107-7c68ceb0b78e new file mode 100644 index 00000000000000..0bc166a72ed948 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-checkpoint/fbdfafe2-ad27-4f56-8107-7c68ceb0b78e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/0d56b254-d4d2-4226-8e67-53343fb829da b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/0d56b254-d4d2-4226-8e67-53343fb829da new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/0d56b254-d4d2-4226-8e67-53343fb829da differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/198994c5-87c9-45c3-a0db-17f4b52a2068 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/198994c5-87c9-45c3-a0db-17f4b52a2068 new file mode 100644 index 00000000000000..432645f554474c Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/198994c5-87c9-45c3-a0db-17f4b52a2068 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/47ac03eb-4cdd-4995-a09b-ec167dd569ef b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/47ac03eb-4cdd-4995-a09b-ec167dd569ef new file mode 100644 index 00000000000000..6bf035568ab390 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/47ac03eb-4cdd-4995-a09b-ec167dd569ef differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/54881c11-6731-49fc-95ff-b7380b267b3c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/54881c11-6731-49fc-95ff-b7380b267b3c new file mode 100644 index 00000000000000..9bb072cc48cc33 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/54881c11-6731-49fc-95ff-b7380b267b3c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/5b3c1cbc-a071-40ce-93a1-6ea0248c3ed9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/5b3c1cbc-a071-40ce-93a1-6ea0248c3ed9 new file mode 100644 index 00000000000000..2fed7b0aa3b078 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/5b3c1cbc-a071-40ce-93a1-6ea0248c3ed9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/6144de29-daca-41b8-9091-d698397c6b1a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/6144de29-daca-41b8-9091-d698397c6b1a new file mode 100644 index 00000000000000..0bc166a72ed948 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/6144de29-daca-41b8-9091-d698397c6b1a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/738ada90-e74f-49ac-9d78-5c7f9e5bb599 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/738ada90-e74f-49ac-9d78-5c7f9e5bb599 new file mode 100644 index 00000000000000..432645f554474c Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/738ada90-e74f-49ac-9d78-5c7f9e5bb599 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/746408cf-cdb8-4341-90ed-0e8e537776d9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/746408cf-cdb8-4341-90ed-0e8e537776d9 new file mode 100644 index 00000000000000..b55d54feede067 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/746408cf-cdb8-4341-90ed-0e8e537776d9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/82c38e44-ac91-47ac-a3cc-fb76736ca71c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/82c38e44-ac91-47ac-a3cc-fb76736ca71c new file mode 100644 index 00000000000000..7ca6fef2a8e966 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/82c38e44-ac91-47ac-a3cc-fb76736ca71c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/936a7b39-75a0-4657-bf5f-f295ffb34c79 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/936a7b39-75a0-4657-bf5f-f295ffb34c79 new file mode 100644 index 00000000000000..7ca6fef2a8e966 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/936a7b39-75a0-4657-bf5f-f295ffb34c79 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/_metadata new file mode 100644 index 00000000000000..a2e7dcbba0d926 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ab25e2b9-0168-490c-9e69-2be833dcc353 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ab25e2b9-0168-490c-9e69-2be833dcc353 new file mode 100644 index 00000000000000..b55d54feede067 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ab25e2b9-0168-490c-9e69-2be833dcc353 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ab44c145-c837-4862-b0e5-a9f3e6d6a353 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ab44c145-c837-4862-b0e5-a9f3e6d6a353 new file mode 100644 index 00000000000000..6bf035568ab390 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ab44c145-c837-4862-b0e5-a9f3e6d6a353 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/d7585297-5b26-4ff3-9590-6f40b5a30c7c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/d7585297-5b26-4ff3-9590-6f40b5a30c7c new file mode 100644 index 00000000000000..f4a3b3f6a28aaf Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/d7585297-5b26-4ff3-9590-6f40b5a30c7c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ea039358-a684-455d-b8d8-098af219c0ea b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ea039358-a684-455d-b8d8-098af219c0ea new file mode 100644 index 00000000000000..0bc166a72ed948 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/ea039358-a684-455d-b8d8-098af219c0ea differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/f6c8c93d-551e-437e-9705-ac0dea275783 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/f6c8c93d-551e-437e-9705-ac0dea275783 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/f6c8c93d-551e-437e-9705-ac0dea275783 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/fa413e74-7411-4abb-89a1-90e492721580 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/fa413e74-7411-4abb-89a1-90e492721580 new file mode 100644 index 00000000000000..f4a3b3f6a28aaf Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint-native/fa413e74-7411-4abb-89a1-90e492721580 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/07c6ae20-da34-494f-b0df-5af6b753d676 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/07c6ae20-da34-494f-b0df-5af6b753d676 new file mode 100644 index 00000000000000..06a691dddf6c28 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/07c6ae20-da34-494f-b0df-5af6b753d676 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/0e072358-d5a9-4485-99df-815c3bf7c82b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/0e072358-d5a9-4485-99df-815c3bf7c82b new file mode 100644 index 00000000000000..7ff4929406c960 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/0e072358-d5a9-4485-99df-815c3bf7c82b differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/1244f5ff-6c1b-4387-898b-41b875dadd94 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/1244f5ff-6c1b-4387-898b-41b875dadd94 new file mode 100644 index 00000000000000..173ca714499919 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/1244f5ff-6c1b-4387-898b-41b875dadd94 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/271c51f9-ee0d-4fd9-8e93-19fd92f42a21 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/271c51f9-ee0d-4fd9-8e93-19fd92f42a21 new file mode 100644 index 00000000000000..5d6822766127c6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/271c51f9-ee0d-4fd9-8e93-19fd92f42a21 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/3308ecd2-84c1-4743-886c-4e211ce781e3 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/3308ecd2-84c1-4743-886c-4e211ce781e3 new file mode 100644 index 00000000000000..b1a06476d73bce Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/3308ecd2-84c1-4743-886c-4e211ce781e3 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/58c296ae-1728-44c1-8906-6c287ccdef10 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/58c296ae-1728-44c1-8906-6c287ccdef10 new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/58c296ae-1728-44c1-8906-6c287ccdef10 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/63be2c2f-3f94-432c-8e12-2111d32d8259 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/63be2c2f-3f94-432c-8e12-2111d32d8259 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/63be2c2f-3f94-432c-8e12-2111d32d8259 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/78d70a3a-33b3-47b7-a56b-98e36e1a599e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/78d70a3a-33b3-47b7-a56b-98e36e1a599e new file mode 100644 index 00000000000000..5d6822766127c6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/78d70a3a-33b3-47b7-a56b-98e36e1a599e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/7ccc1ae0-7621-4931-899d-a630a9caf807 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/7ccc1ae0-7621-4931-899d-a630a9caf807 new file mode 100644 index 00000000000000..3ca8fe8552489a Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/7ccc1ae0-7621-4931-899d-a630a9caf807 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/86a44de5-99ac-4ad2-aae2-1f01d7e8347f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/86a44de5-99ac-4ad2-aae2-1f01d7e8347f new file mode 100644 index 00000000000000..7ff4929406c960 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/86a44de5-99ac-4ad2-aae2-1f01d7e8347f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/88cb2bb0-e37f-4859-8266-6df52c3bddec b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/88cb2bb0-e37f-4859-8266-6df52c3bddec new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/88cb2bb0-e37f-4859-8266-6df52c3bddec differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/8c2d1a1b-debb-4750-b6f2-260fe94c5869 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/8c2d1a1b-debb-4750-b6f2-260fe94c5869 new file mode 100644 index 00000000000000..3ca8fe8552489a Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/8c2d1a1b-debb-4750-b6f2-260fe94c5869 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/_metadata new file mode 100644 index 00000000000000..0cee084b49bd85 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/a4c13094-c6e2-4d05-9958-6caca82b26f9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/a4c13094-c6e2-4d05-9958-6caca82b26f9 new file mode 100644 index 00000000000000..7bdcb98af9b491 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/a4c13094-c6e2-4d05-9958-6caca82b26f9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/e96e1b5d-a394-49c1-a2b2-62fb188ad953 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/e96e1b5d-a394-49c1-a2b2-62fb188ad953 new file mode 100644 index 00000000000000..7e6a6520d61607 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/e96e1b5d-a394-49c1-a2b2-62fb188ad953 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/f59d2379-9d30-4383-8608-35853ca9b1c8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/f59d2379-9d30-4383-8608-35853ca9b1c8 new file mode 100644 index 00000000000000..7bdcb98af9b491 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/f59d2379-9d30-4383-8608-35853ca9b1c8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/fc90f5f2-5b58-42ac-a127-6c3a9dbba40a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/fc90f5f2-5b58-42ac-a127-6c3a9dbba40a new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-hashmap-savepoint/fc90f5f2-5b58-42ac-a127-6c3a9dbba40a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/077484c0-5e19-436f-bfb0-84f42b476f81 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/077484c0-5e19-436f-bfb0-84f42b476f81 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/077484c0-5e19-436f-bfb0-84f42b476f81 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/07b4db4d-9266-4bfa-b04a-12ef0d1016dd b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/07b4db4d-9266-4bfa-b04a-12ef0d1016dd new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/07b4db4d-9266-4bfa-b04a-12ef0d1016dd @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/089196b8-e406-4e62-a65c-acef06aafa68 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/089196b8-e406-4e62-a65c-acef06aafa68 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/089196b8-e406-4e62-a65c-acef06aafa68 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0b46e588-c8f8-407e-9551-44c624b3576c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0b46e588-c8f8-407e-9551-44c624b3576c new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0b46e588-c8f8-407e-9551-44c624b3576c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0db591bd-5a43-4c20-beed-57d95e0335ca b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0db591bd-5a43-4c20-beed-57d95e0335ca new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0db591bd-5a43-4c20-beed-57d95e0335ca differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0e7f5ab5-f1ca-4ffc-8210-d1cd323daaec b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0e7f5ab5-f1ca-4ffc-8210-d1cd323daaec new file mode 100644 index 00000000000000..310ba6cd200ce4 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/0e7f5ab5-f1ca-4ffc-8210-d1cd323daaec differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/10a90885-081e-495b-a4bb-581bfc15f5b3 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/10a90885-081e-495b-a4bb-581bfc15f5b3 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/10a90885-081e-495b-a4bb-581bfc15f5b3 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/141372b9-6605-477b-abcb-38f68b761350 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/141372b9-6605-477b-abcb-38f68b761350 new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/141372b9-6605-477b-abcb-38f68b761350 @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1867edd7-fea1-47ca-aa9b-0efa10336f4d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1867edd7-fea1-47ca-aa9b-0efa10336f4d new file mode 100644 index 00000000000000..480e5ab0207e7d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/1867edd7-fea1-47ca-aa9b-0efa10336f4d differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/19ca0409-d61a-4bb2-bc4f-1dcf578f55b8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/19ca0409-d61a-4bb2-bc4f-1dcf578f55b8 new file mode 100644 index 00000000000000..781eaac5eff988 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/19ca0409-d61a-4bb2-bc4f-1dcf578f55b8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/21ec736f-8eb6-465d-8211-7493e1f68aae b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/21ec736f-8eb6-465d-8211-7493e1f68aae new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/21ec736f-8eb6-465d-8211-7493e1f68aae @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/29779921-ddc6-4961-92cd-76658b8cb36b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/29779921-ddc6-4961-92cd-76658b8cb36b new file mode 100644 index 00000000000000..078ddf0b33708e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/29779921-ddc6-4961-92cd-76658b8cb36b differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2f2bd39e-11c0-4e4e-9514-4637528ebbc5 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2f2bd39e-11c0-4e4e-9514-4637528ebbc5 new file mode 100644 index 00000000000000..d081033f84a43f Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/2f2bd39e-11c0-4e4e-9514-4637528ebbc5 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/361dfc6c-e533-4a2e-bd89-f2c0e78ac821 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/361dfc6c-e533-4a2e-bd89-f2c0e78ac821 new file mode 100644 index 00000000000000..63d731feafc904 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/361dfc6c-e533-4a2e-bd89-f2c0e78ac821 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/37eb0a14-acc4-40cc-a1b9-b93fef5c6c81 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/37eb0a14-acc4-40cc-a1b9-b93fef5c6c81 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/37eb0a14-acc4-40cc-a1b9-b93fef5c6c81 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/3cd5632d-9b87-4837-91bf-7144ff50475e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/3cd5632d-9b87-4837-91bf-7144ff50475e new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/3cd5632d-9b87-4837-91bf-7144ff50475e @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/40753da1-4b3c-487f-b7f9-913ccc5b5fa9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/40753da1-4b3c-487f-b7f9-913ccc5b5fa9 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/40753da1-4b3c-487f-b7f9-913ccc5b5fa9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/419ea493-eb76-4b4f-a824-44f26ec4abc7 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/419ea493-eb76-4b4f-a824-44f26ec4abc7 new file mode 100644 index 00000000000000..0971a3e3dc6f85 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/419ea493-eb76-4b4f-a824-44f26ec4abc7 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/44f696f8-a99c-4607-98c6-784497efc0fe b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/44f696f8-a99c-4607-98c6-784497efc0fe new file mode 100644 index 00000000000000..4a8c84bac290a1 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/44f696f8-a99c-4607-98c6-784497efc0fe differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/49817ff0-028b-4f91-9523-a176abd0fd63 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/49817ff0-028b-4f91-9523-a176abd0fd63 new file mode 100644 index 00000000000000..901a0cbbc60b0e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/49817ff0-028b-4f91-9523-a176abd0fd63 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4a44398c-bf64-4982-90f2-51c2e8b7db91 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4a44398c-bf64-4982-90f2-51c2e8b7db91 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4a44398c-bf64-4982-90f2-51c2e8b7db91 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4d6e907f-83e1-42d6-aca4-65c51814208f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4d6e907f-83e1-42d6-aca4-65c51814208f new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/4d6e907f-83e1-42d6-aca4-65c51814208f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/594f69f3-2448-400a-91e2-c1e8ec1df26a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/594f69f3-2448-400a-91e2-c1e8ec1df26a new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/594f69f3-2448-400a-91e2-c1e8ec1df26a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5cc02864-4379-4391-83d2-f66f341bce0a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5cc02864-4379-4391-83d2-f66f341bce0a new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5cc02864-4379-4391-83d2-f66f341bce0a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5f51a027-431a-4163-8052-4accc7924269 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5f51a027-431a-4163-8052-4accc7924269 new file mode 100644 index 00000000000000..0e36dbd4cdcd04 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5f51a027-431a-4163-8052-4accc7924269 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5f67e42e-3a37-48a8-a6af-7e9b8849f096 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5f67e42e-3a37-48a8-a6af-7e9b8849f096 new file mode 100644 index 00000000000000..5c308cbc7ebee3 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/5f67e42e-3a37-48a8-a6af-7e9b8849f096 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65afb798-744c-446a-ae93-79f6a0d39f44 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65afb798-744c-446a-ae93-79f6a0d39f44 new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65afb798-744c-446a-ae93-79f6a0d39f44 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65d7affe-e53d-4c89-ba95-6c29315a8a66 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65d7affe-e53d-4c89-ba95-6c29315a8a66 new file mode 100644 index 00000000000000..fb8221f4910e8d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/65d7affe-e53d-4c89-ba95-6c29315a8a66 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/679bef9a-7383-490c-9f71-e25fe0a8024c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/679bef9a-7383-490c-9f71-e25fe0a8024c new file mode 100644 index 00000000000000..841e3633209809 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/679bef9a-7383-490c-9f71-e25fe0a8024c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6a38f8a7-234a-4f4d-925c-ec009cbe30ef b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6a38f8a7-234a-4f4d-925c-ec009cbe30ef new file mode 100644 index 00000000000000..f3b193d1ccd1fe Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6a38f8a7-234a-4f4d-925c-ec009cbe30ef differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6db805a0-39bf-4dcd-8437-b8849feae0b7 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6db805a0-39bf-4dcd-8437-b8849feae0b7 new file mode 100644 index 00000000000000..a6df1e7cf376a2 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6db805a0-39bf-4dcd-8437-b8849feae0b7 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6f3394ca-a742-4217-b760-4d6d2a133076 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6f3394ca-a742-4217-b760-4d6d2a133076 new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6f3394ca-a742-4217-b760-4d6d2a133076 @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6fc258f2-7197-4f82-8a6a-7b3276b8388f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6fc258f2-7197-4f82-8a6a-7b3276b8388f new file mode 100644 index 00000000000000..dfb020f3e5b1de Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/6fc258f2-7197-4f82-8a6a-7b3276b8388f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/71f680a8-f58f-4147-ac88-71f8aa3cd52e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/71f680a8-f58f-4147-ac88-71f8aa3cd52e new file mode 100644 index 00000000000000..b40d0990518a6e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/71f680a8-f58f-4147-ac88-71f8aa3cd52e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/763f1f04-100a-4b26-9db2-40ad7a7008e1 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/763f1f04-100a-4b26-9db2-40ad7a7008e1 new file mode 100644 index 00000000000000..356cb8d0d91227 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/763f1f04-100a-4b26-9db2-40ad7a7008e1 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/78b9528d-4b5a-4adb-b9d2-91b71b224f1b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/78b9528d-4b5a-4adb-b9d2-91b71b224f1b new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/78b9528d-4b5a-4adb-b9d2-91b71b224f1b differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/797e42be-9ffb-41b7-b6c9-f7ff31723365 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/797e42be-9ffb-41b7-b6c9-f7ff31723365 new file mode 100644 index 00000000000000..aec654b2326db5 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/797e42be-9ffb-41b7-b6c9-f7ff31723365 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/7d704553-4d4d-468c-beb8-e477b7c7b431 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/7d704553-4d4d-468c-beb8-e477b7c7b431 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/7d704553-4d4d-468c-beb8-e477b7c7b431 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/80cf218e-9aed-4d83-8771-e810a2900fe9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/80cf218e-9aed-4d83-8771-e810a2900fe9 new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/80cf218e-9aed-4d83-8771-e810a2900fe9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/818df800-fd2c-4b30-96d4-367532290fc0 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/818df800-fd2c-4b30-96d4-367532290fc0 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/818df800-fd2c-4b30-96d4-367532290fc0 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/88203641-edd9-49f3-8ada-379556908c20 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/88203641-edd9-49f3-8ada-379556908c20 new file mode 100644 index 00000000000000..6e29c943a14c1f Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/88203641-edd9-49f3-8ada-379556908c20 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/89cb3686-20a7-49cc-829a-d2b613fa5d27 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/89cb3686-20a7-49cc-829a-d2b613fa5d27 new file mode 100644 index 00000000000000..edf748bb34469b Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/89cb3686-20a7-49cc-829a-d2b613fa5d27 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/8a3a5998-08e6-4b69-b8bd-a1d6bc6b4eb1 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/8a3a5998-08e6-4b69-b8bd-a1d6bc6b4eb1 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/8a3a5998-08e6-4b69-b8bd-a1d6bc6b4eb1 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/8f08d36a-7b6f-4779-a57f-2da89facf68d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/8f08d36a-7b6f-4779-a57f-2da89facf68d new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/8f08d36a-7b6f-4779-a57f-2da89facf68d @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9170c08c-8c89-4e11-863d-6916574b32ee b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9170c08c-8c89-4e11-863d-6916574b32ee new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9170c08c-8c89-4e11-863d-6916574b32ee @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/91e74daf-23bf-4e9e-b3d8-88c86e5cb7c0 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/91e74daf-23bf-4e9e-b3d8-88c86e5cb7c0 new file mode 100644 index 00000000000000..eb08b7d96f2f6a Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/91e74daf-23bf-4e9e-b3d8-88c86e5cb7c0 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/94e347a5-ffb7-4042-a984-7da49a8889da b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/94e347a5-ffb7-4042-a984-7da49a8889da new file mode 100644 index 00000000000000..413a56910cada5 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/94e347a5-ffb7-4042-a984-7da49a8889da differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/95d078d3-fc51-423c-96ec-e6773f8d8487 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/95d078d3-fc51-423c-96ec-e6773f8d8487 new file mode 100644 index 00000000000000..c0eb747ddecc37 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/95d078d3-fc51-423c-96ec-e6773f8d8487 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9962d291-cbeb-42f5-b3d3-e8d9cad92d91 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9962d291-cbeb-42f5-b3d3-e8d9cad92d91 new file mode 100644 index 00000000000000..138aa09ae97c42 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9962d291-cbeb-42f5-b3d3-e8d9cad92d91 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9cbdfdb9-732d-487c-860f-74ad0503e6fa b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9cbdfdb9-732d-487c-860f-74ad0503e6fa new file mode 100644 index 00000000000000..7578e57efccbcc Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/9cbdfdb9-732d-487c-860f-74ad0503e6fa differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/_metadata new file mode 100644 index 00000000000000..ca615ca174bfa6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a2cfdcbb-ebc7-4665-b3e9-6478d2d6157d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a2cfdcbb-ebc7-4665-b3e9-6478d2d6157d new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a2cfdcbb-ebc7-4665-b3e9-6478d2d6157d @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a2edf219-5899-4af0-9ecb-e7ccc022b829 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a2edf219-5899-4af0-9ecb-e7ccc022b829 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a2edf219-5899-4af0-9ecb-e7ccc022b829 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a85b94c3-b539-43ff-93a3-1a9c592f96ae b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a85b94c3-b539-43ff-93a3-1a9c592f96ae new file mode 100644 index 00000000000000..10a37a890d0a5f Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/a85b94c3-b539-43ff-93a3-1a9c592f96ae differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/abdf15c6-c997-4607-989c-7b793f81ac0f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/abdf15c6-c997-4607-989c-7b793f81ac0f new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/abdf15c6-c997-4607-989c-7b793f81ac0f @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ad91efab-f85b-4c40-93c8-01f9044236de b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ad91efab-f85b-4c40-93c8-01f9044236de new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ad91efab-f85b-4c40-93c8-01f9044236de @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ae233392-97f4-44a4-ac68-6a04e8b61272 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ae233392-97f4-44a4-ac68-6a04e8b61272 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ae233392-97f4-44a4-ac68-6a04e8b61272 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b31ff795-33f9-46ab-9abe-d5c35655f495 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b31ff795-33f9-46ab-9abe-d5c35655f495 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b31ff795-33f9-46ab-9abe-d5c35655f495 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b40ae522-462d-4a51-84bf-6c87ba43144c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b40ae522-462d-4a51-84bf-6c87ba43144c new file mode 100644 index 00000000000000..7ed4b7229e7797 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b40ae522-462d-4a51-84bf-6c87ba43144c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b56e42a8-7fb6-41e3-8014-1945ac5a3b0b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b56e42a8-7fb6-41e3-8014-1945ac5a3b0b new file mode 100644 index 00000000000000..95b1a6eeff5b84 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/b56e42a8-7fb6-41e3-8014-1945ac5a3b0b differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/bacd5e3c-d8dc-4b22-a006-c4b244e9b971 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/bacd5e3c-d8dc-4b22-a006-c4b244e9b971 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/bacd5e3c-d8dc-4b22-a006-c4b244e9b971 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/bcc393d8-6a41-4a81-8553-b5934af6d401 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/bcc393d8-6a41-4a81-8553-b5934af6d401 new file mode 100644 index 00000000000000..58f09e694b37f0 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/bcc393d8-6a41-4a81-8553-b5934af6d401 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c139387a-072f-4d4d-a1d3-befd96664480 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c139387a-072f-4d4d-a1d3-befd96664480 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c139387a-072f-4d4d-a1d3-befd96664480 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c1c8b265-bad3-450e-824f-067ef47e7827 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c1c8b265-bad3-450e-824f-067ef47e7827 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/c1c8b265-bad3-450e-824f-067ef47e7827 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ccec00d0-f5d5-4914-8d3b-d2a35c4bc1fd b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ccec00d0-f5d5-4914-8d3b-d2a35c4bc1fd new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ccec00d0-f5d5-4914-8d3b-d2a35c4bc1fd @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d18f7d4f-2fa2-4d17-98bb-6cef47d6727b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d18f7d4f-2fa2-4d17-98bb-6cef47d6727b new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d18f7d4f-2fa2-4d17-98bb-6cef47d6727b @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d4edb439-a170-4dac-b979-bebbcec9e1e8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d4edb439-a170-4dac-b979-bebbcec9e1e8 new file mode 100644 index 00000000000000..268aefd0ede0dc Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d4edb439-a170-4dac-b979-bebbcec9e1e8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d72b5a6b-9257-4022-92a6-6fa017a597de b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d72b5a6b-9257-4022-92a6-6fa017a597de new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/d72b5a6b-9257-4022-92a6-6fa017a597de @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e014c2fe-bbd7-43d8-862b-cc8587090e10 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e014c2fe-bbd7-43d8-862b-cc8587090e10 new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e014c2fe-bbd7-43d8-862b-cc8587090e10 @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e0262725-8956-4c47-a422-30fbe07be4fb b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e0262725-8956-4c47-a422-30fbe07be4fb new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e0262725-8956-4c47-a422-30fbe07be4fb @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e0c2b3ea-eada-47f3-8006-6a755abdfb7d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e0c2b3ea-eada-47f3-8006-6a755abdfb7d new file mode 100644 index 00000000000000..132db090fa2dd5 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e0c2b3ea-eada-47f3-8006-6a755abdfb7d differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e5ef7089-6903-460b-8d27-79dedce80e6e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e5ef7089-6903-460b-8d27-79dedce80e6e new file mode 100644 index 00000000000000..0e36dbd4cdcd04 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/e5ef7089-6903-460b-8d27-79dedce80e6e @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/eb095a55-f2f7-429e-af83-18710f80d06e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/eb095a55-f2f7-429e-af83-18710f80d06e new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/eb095a55-f2f7-429e-af83-18710f80d06e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ed943c2b-f675-4504-84e4-8085fd654c5d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ed943c2b-f675-4504-84e4-8085fd654c5d new file mode 100644 index 00000000000000..a24db4bf2c4327 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ed943c2b-f675-4504-84e4-8085fd654c5d differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ee3f8736-d574-4d69-8f0c-1ae2b30b050f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ee3f8736-d574-4d69-8f0c-1ae2b30b050f new file mode 100644 index 00000000000000..f039bcfd2160ae Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ee3f8736-d574-4d69-8f0c-1ae2b30b050f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/efb842f1-03a4-4407-9c39-2e7af130d3f5 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/efb842f1-03a4-4407-9c39-2e7af130d3f5 new file mode 100644 index 00000000000000..253d0f256554b9 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/efb842f1-03a4-4407-9c39-2e7af130d3f5 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f32012a4-c5f2-4be5-bae9-e9b83d0bebd1 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f32012a4-c5f2-4be5-bae9-e9b83d0bebd1 new file mode 100644 index 00000000000000..a727e93a9b86e2 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f32012a4-c5f2-4be5-bae9-e9b83d0bebd1 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f3f6ef36-d2d8-48b3-8ba1-1fa8169a2447 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f3f6ef36-d2d8-48b3-8ba1-1fa8169a2447 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f3f6ef36-d2d8-48b3-8ba1-1fa8169a2447 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f61d7a83-ccc8-4afb-8976-9290ba8b035a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f61d7a83-ccc8-4afb-8976-9290ba8b035a new file mode 100644 index 00000000000000..b9aa5a855b21d5 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f61d7a83-ccc8-4afb-8976-9290ba8b035a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f75af3d2-d7eb-40cb-a1af-3a8e15de4d7f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f75af3d2-d7eb-40cb-a1af-3a8e15de4d7f new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/f75af3d2-d7eb-40cb-a1af-3a8e15de4d7f @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fba42b4d-7bfe-44dc-874a-79e91d0c97be b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fba42b4d-7bfe-44dc-874a-79e91d0c97be new file mode 100644 index 00000000000000..121a25b7e65442 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fba42b4d-7bfe-44dc-874a-79e91d0c97be differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fd754104-527e-4a75-9dee-2445886f6df7 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fd754104-527e-4a75-9dee-2445886f6df7 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/fd754104-527e-4a75-9dee-2445886f6df7 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ff75b468-a725-47e6-a410-c823acceda2a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ff75b468-a725-47e6-a410-c823acceda2a new file mode 100644 index 00000000000000..dfb020f3e5b1de Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-checkpoint/ff75b468-a725-47e6-a410-c823acceda2a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/03556267-d317-4389-b2c5-cb6617380c53 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/03556267-d317-4389-b2c5-cb6617380c53 new file mode 100644 index 00000000000000..08039e57e8f254 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/03556267-d317-4389-b2c5-cb6617380c53 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/03e98b55-2752-4826-8058-5f7cad4dc8f8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/03e98b55-2752-4826-8058-5f7cad4dc8f8 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/03e98b55-2752-4826-8058-5f7cad4dc8f8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/05a3364c-418a-4c99-bc58-5f6ceeac38e0 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/05a3364c-418a-4c99-bc58-5f6ceeac38e0 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/05a3364c-418a-4c99-bc58-5f6ceeac38e0 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/070bb171-8a9a-4004-a8fc-3db8d05fc83a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/070bb171-8a9a-4004-a8fc-3db8d05fc83a new file mode 100644 index 00000000000000..4073ae19dda269 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/070bb171-8a9a-4004-a8fc-3db8d05fc83a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/086b1ce2-8f44-42ad-85e5-85d3cfa8c57f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/086b1ce2-8f44-42ad-85e5-85d3cfa8c57f new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/086b1ce2-8f44-42ad-85e5-85d3cfa8c57f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/09f2f282-2230-4b03-923b-5f2c68ad1e05 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/09f2f282-2230-4b03-923b-5f2c68ad1e05 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/09f2f282-2230-4b03-923b-5f2c68ad1e05 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0b5f99eb-04ae-4c30-a75c-7a2ccc9598d4 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0b5f99eb-04ae-4c30-a75c-7a2ccc9598d4 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0b5f99eb-04ae-4c30-a75c-7a2ccc9598d4 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0f22c723-413f-49b7-835b-320cdc92cb4f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0f22c723-413f-49b7-835b-320cdc92cb4f new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/0f22c723-413f-49b7-835b-320cdc92cb4f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/150f4489-f3a5-4904-a8f5-173cb35e20ff b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/150f4489-f3a5-4904-a8f5-173cb35e20ff new file mode 100644 index 00000000000000..efb26a1f31cb85 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/150f4489-f3a5-4904-a8f5-173cb35e20ff differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/16ae77d4-2921-416e-b8fa-47d5f5361524 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/16ae77d4-2921-416e-b8fa-47d5f5361524 new file mode 100644 index 00000000000000..719e9240ecdd88 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/16ae77d4-2921-416e-b8fa-47d5f5361524 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/18fb8fde-3c37-4241-8da7-f7d2cc95dd1e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/18fb8fde-3c37-4241-8da7-f7d2cc95dd1e new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/18fb8fde-3c37-4241-8da7-f7d2cc95dd1e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1a5ff059-f7cc-4a08-9268-ea271230f700 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1a5ff059-f7cc-4a08-9268-ea271230f700 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1a5ff059-f7cc-4a08-9268-ea271230f700 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1cc27bb1-7d76-4735-a3c8-11f5ee6b230e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1cc27bb1-7d76-4735-a3c8-11f5ee6b230e new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/1cc27bb1-7d76-4735-a3c8-11f5ee6b230e @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/2415c309-e747-40fd-a1e7-459ce0464ea7 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/2415c309-e747-40fd-a1e7-459ce0464ea7 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/2415c309-e747-40fd-a1e7-459ce0464ea7 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/2709bf7b-c933-422d-aa8e-4cecea552b5a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/2709bf7b-c933-422d-aa8e-4cecea552b5a new file mode 100644 index 00000000000000..8afff7c13214d1 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/2709bf7b-c933-422d-aa8e-4cecea552b5a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/29c885c5-5589-4326-9ef4-c549ab5a0390 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/29c885c5-5589-4326-9ef4-c549ab5a0390 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/29c885c5-5589-4326-9ef4-c549ab5a0390 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3291a378-5337-4a40-9458-8d7df6d10de0 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3291a378-5337-4a40-9458-8d7df6d10de0 new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3291a378-5337-4a40-9458-8d7df6d10de0 @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/348a6733-e54e-470e-836a-5073245deade b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/348a6733-e54e-470e-836a-5073245deade new file mode 100644 index 00000000000000..9c8a59b7912046 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/348a6733-e54e-470e-836a-5073245deade differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/35417483-2bb7-4f9b-aa9e-552c88aed0de b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/35417483-2bb7-4f9b-aa9e-552c88aed0de new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/35417483-2bb7-4f9b-aa9e-552c88aed0de @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/35e45c86-b8fd-46b5-85e6-832acc3f493c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/35e45c86-b8fd-46b5-85e6-832acc3f493c new file mode 100644 index 00000000000000..36cfb5e2e87d90 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/35e45c86-b8fd-46b5-85e6-832acc3f493c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d509781-f361-42fc-a7ea-954a22768146 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d509781-f361-42fc-a7ea-954a22768146 new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d509781-f361-42fc-a7ea-954a22768146 @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d678140-61cb-4c68-9ba0-f89c0c4fe939 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d678140-61cb-4c68-9ba0-f89c0c4fe939 new file mode 100644 index 00000000000000..70a57d340c1441 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/3d678140-61cb-4c68-9ba0-f89c0c4fe939 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/41748b1a-6316-4530-b896-b51a989e89b3 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/41748b1a-6316-4530-b896-b51a989e89b3 new file mode 100644 index 00000000000000..e01cb660d6e2ef Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/41748b1a-6316-4530-b896-b51a989e89b3 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/449b6ac3-3ba0-416d-a4d3-178da0fe8c77 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/449b6ac3-3ba0-416d-a4d3-178da0fe8c77 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/449b6ac3-3ba0-416d-a4d3-178da0fe8c77 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/47160329-3ea0-4f05-85bd-70a87d4f3627 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/47160329-3ea0-4f05-85bd-70a87d4f3627 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/47160329-3ea0-4f05-85bd-70a87d4f3627 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/47ee7a83-1e54-417c-85c7-91b72b9b8c37 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/47ee7a83-1e54-417c-85c7-91b72b9b8c37 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/47ee7a83-1e54-417c-85c7-91b72b9b8c37 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4a754c38-a466-4175-9a7a-6f126005430e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4a754c38-a466-4175-9a7a-6f126005430e new file mode 100644 index 00000000000000..03b29f9e9e1d0e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4a754c38-a466-4175-9a7a-6f126005430e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4bafff49-50ea-469a-a2c3-527f61e25d31 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4bafff49-50ea-469a-a2c3-527f61e25d31 new file mode 100644 index 00000000000000..ac2fe2cb87bee1 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/4bafff49-50ea-469a-a2c3-527f61e25d31 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/50202154-7876-435a-9ff8-34204b1e19da b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/50202154-7876-435a-9ff8-34204b1e19da new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/50202154-7876-435a-9ff8-34204b1e19da @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/505b515d-57da-48eb-94d5-e391e2fd516e b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/505b515d-57da-48eb-94d5-e391e2fd516e new file mode 100644 index 00000000000000..4a0a7fcc12321e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/505b515d-57da-48eb-94d5-e391e2fd516e differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/51d89b34-07e7-4bab-bd29-2216fddb3caa b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/51d89b34-07e7-4bab-bd29-2216fddb3caa new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/51d89b34-07e7-4bab-bd29-2216fddb3caa differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/5b1a53ac-0ad7-4861-8681-5eb44ba9705a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/5b1a53ac-0ad7-4861-8681-5eb44ba9705a new file mode 100644 index 00000000000000..dfb020f3e5b1de Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/5b1a53ac-0ad7-4861-8681-5eb44ba9705a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/5bdaf88d-2d20-43a4-8306-0279d5c8744c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/5bdaf88d-2d20-43a4-8306-0279d5c8744c new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/5bdaf88d-2d20-43a4-8306-0279d5c8744c @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/62ef218e-909f-4e75-8f51-d39a30023388 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/62ef218e-909f-4e75-8f51-d39a30023388 new file mode 100644 index 00000000000000..6635d2aa28240f Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/62ef218e-909f-4e75-8f51-d39a30023388 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/645fb0e7-0ef4-4f23-8f9e-8196a2970e15 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/645fb0e7-0ef4-4f23-8f9e-8196a2970e15 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/645fb0e7-0ef4-4f23-8f9e-8196a2970e15 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6ac0ed05-5d16-4887-8b26-7dcd3b966967 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6ac0ed05-5d16-4887-8b26-7dcd3b966967 new file mode 100644 index 00000000000000..dfb020f3e5b1de Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6ac0ed05-5d16-4887-8b26-7dcd3b966967 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6ebe22d9-dc15-4be3-b2a4-1653125675af b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6ebe22d9-dc15-4be3-b2a4-1653125675af new file mode 100644 index 00000000000000..0361f4db3db261 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/6ebe22d9-dc15-4be3-b2a4-1653125675af differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/73ba66fa-7390-464e-8ddb-e71d430490ca b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/73ba66fa-7390-464e-8ddb-e71d430490ca new file mode 100644 index 00000000000000..6ebbd86136d5ac Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/73ba66fa-7390-464e-8ddb-e71d430490ca differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/776ccf69-6562-4a5d-9967-302b4ece524f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/776ccf69-6562-4a5d-9967-302b4ece524f new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/776ccf69-6562-4a5d-9967-302b4ece524f @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7eaea792-d4cc-438b-9f74-85166e55cf00 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7eaea792-d4cc-438b-9f74-85166e55cf00 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7eaea792-d4cc-438b-9f74-85166e55cf00 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7f75438b-7d6d-4ddc-a263-f8cf860b6eb8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7f75438b-7d6d-4ddc-a263-f8cf860b6eb8 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/7f75438b-7d6d-4ddc-a263-f8cf860b6eb8 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/814d4827-402c-4e78-89e8-b7e00cd5dbe8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/814d4827-402c-4e78-89e8-b7e00cd5dbe8 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/814d4827-402c-4e78-89e8-b7e00cd5dbe8 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/84c0fb0f-7fba-4420-a016-bc24fa634f9b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/84c0fb0f-7fba-4420-a016-bc24fa634f9b new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/84c0fb0f-7fba-4420-a016-bc24fa634f9b differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/8561dd76-162f-4fb5-a4f9-9347202551c5 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/8561dd76-162f-4fb5-a4f9-9347202551c5 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/8561dd76-162f-4fb5-a4f9-9347202551c5 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/935aaf85-d5a7-4f09-af90-969d38c5dd68 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/935aaf85-d5a7-4f09-af90-969d38c5dd68 new file mode 100644 index 00000000000000..8511b30b679b77 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/935aaf85-d5a7-4f09-af90-969d38c5dd68 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/95906873-6d01-477f-9635-1ad2793ba481 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/95906873-6d01-477f-9635-1ad2793ba481 new file mode 100644 index 00000000000000..21f8ecb6def9f3 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/95906873-6d01-477f-9635-1ad2793ba481 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/98257f69-3296-4878-aa9c-2a163d8cb054 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/98257f69-3296-4878-aa9c-2a163d8cb054 new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/98257f69-3296-4878-aa9c-2a163d8cb054 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9a6ce57c-e2f8-4997-8db1-d7375ba5c460 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9a6ce57c-e2f8-4997-8db1-d7375ba5c460 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/9a6ce57c-e2f8-4997-8db1-d7375ba5c460 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/_metadata new file mode 100644 index 00000000000000..85a00a82d815e4 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a0fbfc64-ee05-4b65-bd97-4e3f0bf99fce b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a0fbfc64-ee05-4b65-bd97-4e3f0bf99fce new file mode 100644 index 00000000000000..961b6a2fc68c33 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a0fbfc64-ee05-4b65-bd97-4e3f0bf99fce differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a189e40c-9258-41f6-8ce0-255377864817 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a189e40c-9258-41f6-8ce0-255377864817 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a189e40c-9258-41f6-8ce0-255377864817 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a76017f0-1ad5-4319-b708-b15571374e64 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a76017f0-1ad5-4319-b708-b15571374e64 new file mode 100644 index 00000000000000..f0fe6700d66363 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a76017f0-1ad5-4319-b708-b15571374e64 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a8804acc-bdd7-438b-ba59-c4675c52a058 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a8804acc-bdd7-438b-ba59-c4675c52a058 new file mode 100644 index 00000000000000..713f13b551a558 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a8804acc-bdd7-438b-ba59-c4675c52a058 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a9a379c6-a867-4868-89f0-26140cf5b239 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a9a379c6-a867-4868-89f0-26140cf5b239 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/a9a379c6-a867-4868-89f0-26140cf5b239 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/aaf53af4-642a-4210-a448-e2c18d0db5de b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/aaf53af4-642a-4210-a448-e2c18d0db5de new file mode 100644 index 00000000000000..4f65f1dfc3655e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/aaf53af4-642a-4210-a448-e2c18d0db5de differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/aafbb7c7-566f-4d29-bb41-d12b72dbcbb1 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/aafbb7c7-566f-4d29-bb41-d12b72dbcbb1 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/aafbb7c7-566f-4d29-bb41-d12b72dbcbb1 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/af3112be-ef01-4097-b74e-b07eb0ec5323 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/af3112be-ef01-4097-b74e-b07eb0ec5323 new file mode 100644 index 00000000000000..274f614f68f8df Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/af3112be-ef01-4097-b74e-b07eb0ec5323 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/afb50700-193c-4af8-9a4d-b9de4c10ce9c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/afb50700-193c-4af8-9a4d-b9de4c10ce9c new file mode 100644 index 00000000000000..b7fa4cd558dd3c Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/afb50700-193c-4af8-9a4d-b9de4c10ce9c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b5cccf36-4ecb-4af2-aca7-4d6d5a634a4d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b5cccf36-4ecb-4af2-aca7-4d6d5a634a4d new file mode 100644 index 00000000000000..0e36dbd4cdcd04 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b5cccf36-4ecb-4af2-aca7-4d6d5a634a4d @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b66b2a07-f7ca-4f65-b3a3-707931a98fd4 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b66b2a07-f7ca-4f65-b3a3-707931a98fd4 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/b66b2a07-f7ca-4f65-b3a3-707931a98fd4 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c1100c71-5003-4482-bf79-729522742cc8 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c1100c71-5003-4482-bf79-729522742cc8 new file mode 100644 index 00000000000000..0c27c593421756 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c1100c71-5003-4482-bf79-729522742cc8 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c6e8b907-5478-47c5-9312-07b765266a6f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c6e8b907-5478-47c5-9312-07b765266a6f new file mode 100644 index 00000000000000..c4f2c4c98e9ef7 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c6e8b907-5478-47c5-9312-07b765266a6f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c769407a-edea-49d0-baa4-d5daf8615a40 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c769407a-edea-49d0-baa4-d5daf8615a40 new file mode 100644 index 00000000000000..d810f56c44d79e Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/c769407a-edea-49d0-baa4-d5daf8615a40 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/cf1e716c-ed57-40b5-85b2-6cd298f3548c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/cf1e716c-ed57-40b5-85b2-6cd298f3548c new file mode 100644 index 00000000000000..8647747ea1fc75 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/cf1e716c-ed57-40b5-85b2-6cd298f3548c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/cf80c6cb-5402-45a0-a85c-c2c55b414cf3 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/cf80c6cb-5402-45a0-a85c-c2c55b414cf3 new file mode 100644 index 00000000000000..474e5548fb8365 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/cf80c6cb-5402-45a0-a85c-c2c55b414cf3 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d0d60567-280c-4575-a4b5-0c0070644b27 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d0d60567-280c-4575-a4b5-0c0070644b27 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d0d60567-280c-4575-a4b5-0c0070644b27 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d3088d8c-2125-4785-b90f-5ef3b399fb42 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d3088d8c-2125-4785-b90f-5ef3b399fb42 new file mode 100644 index 00000000000000..6fa414733aae57 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d3088d8c-2125-4785-b90f-5ef3b399fb42 @@ -0,0 +1,317 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d42488e5-a66c-4fec-82b9-61ea4becd459 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d42488e5-a66c-4fec-82b9-61ea4becd459 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/d42488e5-a66c-4fec-82b9-61ea4becd459 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/da2cf4b5-4677-4b06-abbd-47425720a028 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/da2cf4b5-4677-4b06-abbd-47425720a028 new file mode 100644 index 00000000000000..be94c22b3e8779 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/da2cf4b5-4677-4b06-abbd-47425720a028 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dac0243f-a584-4fbc-a7a7-e64178d45a62 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dac0243f-a584-4fbc-a7a7-e64178d45a62 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dac0243f-a584-4fbc-a7a7-e64178d45a62 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dd9c1ff3-8b3e-4808-8c94-9010f4d7cb32 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dd9c1ff3-8b3e-4808-8c94-9010f4d7cb32 new file mode 100644 index 00000000000000..17019a4a2ffc91 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/dd9c1ff3-8b3e-4808-8c94-9010f4d7cb32 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e0fc93c9-38c2-4158-870b-7d1526bd43a1 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e0fc93c9-38c2-4158-870b-7d1526bd43a1 new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e0fc93c9-38c2-4158-870b-7d1526bd43a1 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e287854c-2d28-43a1-b135-3dba557fd931 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e287854c-2d28-43a1-b135-3dba557fd931 new file mode 100644 index 00000000000000..14b19dafd9d683 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e287854c-2d28-43a1-b135-3dba557fd931 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e562bd54-e2e2-433f-b949-5862c99e14ec b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e562bd54-e2e2-433f-b949-5862c99e14ec new file mode 100644 index 00000000000000..667ac8709a9d3d Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/e562bd54-e2e2-433f-b949-5862c99e14ec differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ed34b883-5715-4d7c-b615-2047f373e577 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ed34b883-5715-4d7c-b615-2047f373e577 new file mode 100644 index 00000000000000..528264718dd445 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ed34b883-5715-4d7c-b615-2047f373e577 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ee044c64-31e8-4095-a544-d9237a43ba34 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ee044c64-31e8-4095-a544-d9237a43ba34 new file mode 100644 index 00000000000000..b5eca0e986ea54 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ee044c64-31e8-4095-a544-d9237a43ba34 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ee29cbb0-871b-4c85-86d5-ecc26d97c6c9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ee29cbb0-871b-4c85-86d5-ecc26d97c6c9 new file mode 100644 index 00000000000000..0e36dbd4cdcd04 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/ee29cbb0-871b-4c85-86d5-ecc26d97c6c9 @@ -0,0 +1,429 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f0e14849-57ed-4e18-86a9-7055c7ce842a b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f0e14849-57ed-4e18-86a9-7055c7ce842a new file mode 100644 index 00000000000000..c3eaa30eeda694 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f0e14849-57ed-4e18-86a9-7055c7ce842a differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f2665fcd-edaa-4615-ace5-2c6585d452f3 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f2665fcd-edaa-4615-ace5-2c6585d452f3 new file mode 100644 index 00000000000000..aa5bb8ea50905a --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f2665fcd-edaa-4615-ace5-2c6585d452f3 @@ -0,0 +1 @@ +MANIFEST-000005 diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f593321c-a39c-43de-b237-d8d3b33f2000 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f593321c-a39c-43de-b237-d8d3b33f2000 new file mode 100644 index 00000000000000..00c77db5b298c5 --- /dev/null +++ b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f593321c-a39c-43de-b237-d8d3b33f2000 @@ -0,0 +1,541 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=8.10.0 + options_file_version=1.1 + +[DBOptions] + max_background_flushes=-1 + compaction_readahead_size=2097152 + strict_bytes_per_sync=false + wal_bytes_per_sync=0 + max_open_files=-1 + stats_history_buffer_size=1048576 + max_total_wal_size=0 + stats_persist_period_sec=600 + stats_dump_period_sec=0 + avoid_flush_during_shutdown=true + max_subcompactions=1 + bytes_per_sync=0 + delayed_write_rate=16777216 + max_background_compactions=-1 + max_background_jobs=2 + delete_obsolete_files_period_micros=21600000000 + writable_file_max_buffer_size=1048576 + file_checksum_gen_factory=nullptr + allow_data_in_errors=false + max_bgerror_resume_count=2147483647 + best_efforts_recovery=false + write_dbid_to_manifest=false + atomic_flush=false + manual_wal_flush=false + two_write_queues=false + avoid_flush_during_recovery=false + dump_malloc_stats=false + info_log_level=INFO_LEVEL + write_thread_slow_yield_usec=3 + unordered_write=false + allow_ingest_behind=false + fail_if_options_file_error=true + persist_stats_to_disk=false + WAL_ttl_seconds=0 + bgerror_resume_retry_interval=1000000 + allow_concurrent_memtable_write=true + paranoid_checks=true + WAL_size_limit_MB=0 + lowest_used_cache_tier=kNonVolatileBlockTier + keep_log_file_num=4 + table_cache_numshardbits=6 + max_file_opening_threads=16 + random_access_max_buffer_size=1048576 + log_readahead_size=0 + enable_pipelined_write=false + wal_recovery_mode=kPointInTimeRecovery + db_write_buffer_size=0 + allow_2pc=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + recycle_log_file_num=0 + db_host_id=__hostname__ + track_and_verify_wals_in_manifest=false + use_fsync=false + wal_compression=kNoCompression + compaction_verify_record_count=true + error_if_exists=false + manifest_preallocation_size=4194304 + is_fd_close_on_exec=true + enable_write_thread_adaptive_yield=true + enable_thread_tracking=false + avoid_unnecessary_blocking_io=false + allow_fallocate=true + max_log_file_size=26214400 + advise_random_on_open=true + create_missing_column_families=false + max_write_batch_group_size_bytes=1048576 + use_adaptive_mutex=false + wal_filter=nullptr + create_if_missing=true + enforce_single_del_contracts=true + allow_mmap_writes=false + access_hint_on_compaction_start=NORMAL + verify_sst_unique_id_in_manifest=true + log_file_time_to_roll=0 + use_direct_io_for_flush_and_compaction=false + flush_verify_memtable_count=true + max_manifest_file_size=1073741824 + write_thread_max_yield_usec=100 + use_direct_reads=false + allow_mmap_reads=false + + +[CFOptions "default"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "default"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/processing_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/processing_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "_timer_state/event_timer"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "_timer_state/event_timer"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + + +[CFOptions "state-name"] + bottommost_file_compaction_delay=0 + memtable_protection_bytes_per_key=0 + compression_per_level=kSnappyCompression + bottommost_compression=kDisableCompressionOption + sample_for_compression=0 + blob_garbage_collection_age_cutoff=0.250000 + blob_compression_type=kNoCompression + prepopulate_blob_cache=kDisable + blob_compaction_readahead_size=0 + level0_stop_writes_trigger=36 + min_blob_size=0 + last_level_temperature=kUnknown + compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;compression_size_percent=-1;max_size_amplification_percent=200;incremental=false;max_merge_width=4294967295;size_ratio=1;} + target_file_size_base=67108864 + ignore_max_compaction_bytes_for_input=true + memtable_whole_key_filtering=false + blob_file_starting_level=0 + soft_pending_compaction_bytes_limit=68719476736 + max_write_buffer_number=2 + ttl=2592000 + compaction_options_fifo={file_temperature_age_thresholds=;allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + check_flush_compaction_key_order=true + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + enable_blob_garbage_collection=false + arena_block_size=1048576 + bottommost_compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + target_file_size_multiplier=1 + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + blob_garbage_collection_force_threshold=1.000000 + enable_blob_files=false + level0_slowdown_writes_trigger=20 + compression=kSnappyCompression + level0_file_num_compaction_trigger=4 + block_protection_bytes_per_key=0 + prefix_extractor=nullptr + max_bytes_for_level_multiplier=10.000000 + write_buffer_size=67108864 + disable_auto_compactions=false + max_compaction_bytes=1677721600 + memtable_max_range_deletions=0 + compression_opts={use_zstd_dict_trainer=true;enabled=false;zstd_max_train_bytes=0;parallel_threads=1;max_compressed_bytes_per_kb=896;checksum=false;max_dict_bytes=0;strategy=0;max_dict_buffer_bytes=0;level=32767;window_bits=-14;} + hard_pending_compaction_bytes_limit=274877906944 + blob_file_size=268435456 + periodic_compaction_seconds=2592000 + paranoid_file_checks=false + experimental_mempurge_threshold=0.000000 + memtable_prefix_bloom_size_ratio=0.000000 + max_bytes_for_level_base=268435456 + max_sequential_skip_in_iterations=8 + report_bg_io_stats=false + sst_partitioner_factory=nullptr + compaction_pri=kMinOverlappingRatio + compaction_style=kCompactionStyleLevel + compaction_filter_factory=nullptr + compaction_filter=nullptr + memtable_factory=SkipListFactory + comparator=leveldb.BytewiseComparator + bloom_locality=0 + min_write_buffer_number_to_merge=1 + table_factory=BlockBasedTable + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + optimize_filters_for_hits=false + default_temperature=kUnknown + preserve_internal_time_seconds=0 + force_consistency_checks=true + merge_operator={id=StringAppendTESTOperator;delimiter=,;} + num_levels=7 + level_compaction_dynamic_file_size=true + memtable_insert_with_hint_prefix_extractor=nullptr + level_compaction_dynamic_level_bytes=false + persist_user_defined_timestamps=true + preclude_last_level_data_seconds=0 + inplace_update_support=false + +[TableOptions/BlockBasedTable "state-name"] + num_file_reads_for_auto_readahead=2 + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + read_amp_bytes_per_bit=0 + verify_compression=false + format_version=5 + optimize_filters_for_memory=false + partition_filters=false + detect_filter_construct_corruption=false + initial_auto_readahead_size=8192 + max_auto_readahead_size=262144 + enable_index_compression=true + checksum=kXXH3 + index_block_restart_interval=1 + pin_top_level_index_and_filter=true + block_align=false + block_size=4096 + index_type=kBinarySearch + filter_policy=nullptr + metadata_block_size=4096 + no_block_cache=false + index_shortening=kShortenSeparators + whole_key_filtering=true + block_size_deviation=10 + data_block_index_type=kDataBlockBinarySearch + data_block_hash_table_util_ratio=0.750000 + cache_index_and_filter_blocks=true + prepopulate_block_cache=kDisable + block_restart_interval=16 + pin_l0_filter_and_index_blocks_in_cache=true + cache_index_and_filter_blocks_with_high_priority=true + flush_block_policy_factory=FlushBlockBySizePolicyFactory + diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f6c86361-507f-4ca3-8caa-7c7ce9a436d2 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f6c86361-507f-4ca3-8caa-7c7ce9a436d2 new file mode 100644 index 00000000000000..70c34f06f2e247 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f6c86361-507f-4ca3-8caa-7c7ce9a436d2 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f723fcb2-45bd-4b08-a2a9-d6a50c867e4d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f723fcb2-45bd-4b08-a2a9-d6a50c867e4d new file mode 100644 index 00000000000000..c99a8b47d4ae4f Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f723fcb2-45bd-4b08-a2a9-d6a50c867e4d differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f80c5516-caec-4cdf-b6f6-2ebdc70048ce b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f80c5516-caec-4cdf-b6f6-2ebdc70048ce new file mode 100644 index 00000000000000..0354c08367e34a Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint-native/f80c5516-caec-4cdf-b6f6-2ebdc70048ce differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/09acd476-c8f9-4ca1-8930-81c553fee23d b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/09acd476-c8f9-4ca1-8930-81c553fee23d new file mode 100644 index 00000000000000..5e924258b13ca8 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/09acd476-c8f9-4ca1-8930-81c553fee23d differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/154ce638-772d-46d3-a024-281019dd952f b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/154ce638-772d-46d3-a024-281019dd952f new file mode 100644 index 00000000000000..3e4b43753dbe30 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/154ce638-772d-46d3-a024-281019dd952f differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1a63ad9c-6dae-4cb6-b5de-3af680c67fb9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1a63ad9c-6dae-4cb6-b5de-3af680c67fb9 new file mode 100644 index 00000000000000..3ca8fe8552489a Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1a63ad9c-6dae-4cb6-b5de-3af680c67fb9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1b156e31-3761-4b44-b4e0-de8d66daafb9 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1b156e31-3761-4b44-b4e0-de8d66daafb9 new file mode 100644 index 00000000000000..7dd2ab6466d0b0 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1b156e31-3761-4b44-b4e0-de8d66daafb9 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1fb1db0e-a888-4276-8b9d-2b866f7f3f5b b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1fb1db0e-a888-4276-8b9d-2b866f7f3f5b new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/1fb1db0e-a888-4276-8b9d-2b866f7f3f5b differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/249b04db-dcd1-4cd7-a86c-2d8c8c07b508 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/249b04db-dcd1-4cd7-a86c-2d8c8c07b508 new file mode 100644 index 00000000000000..d1759b63def095 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/249b04db-dcd1-4cd7-a86c-2d8c8c07b508 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/50bc8b6b-0473-4858-9cba-c6d9ad6b6b3c b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/50bc8b6b-0473-4858-9cba-c6d9ad6b6b3c new file mode 100644 index 00000000000000..2bceb5cf93a5e6 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/50bc8b6b-0473-4858-9cba-c6d9ad6b6b3c differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/651cc0dc-210b-47e4-82ef-b6597840cdfc b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/651cc0dc-210b-47e4-82ef-b6597840cdfc new file mode 100644 index 00000000000000..3ca8fe8552489a Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/651cc0dc-210b-47e4-82ef-b6597840cdfc differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/7e01a7dd-0775-4b68-bb05-1656ca68db63 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/7e01a7dd-0775-4b68-bb05-1656ca68db63 new file mode 100644 index 00000000000000..966ef8e59b6465 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/7e01a7dd-0775-4b68-bb05-1656ca68db63 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/986d29ea-a289-4d70-8955-a4955ba26596 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/986d29ea-a289-4d70-8955-a4955ba26596 new file mode 100644 index 00000000000000..899ac0d7170558 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/986d29ea-a289-4d70-8955-a4955ba26596 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/_metadata new file mode 100644 index 00000000000000..72f8a60c95da15 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/_metadata differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/a1c0337a-8fc4-4dee-9af9-3320ae772edb b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/a1c0337a-8fc4-4dee-9af9-3320ae772edb new file mode 100644 index 00000000000000..7190d92c836c07 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/a1c0337a-8fc4-4dee-9af9-3320ae772edb differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/b141d405-9cb3-4811-ae5f-ce95eb4cf972 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/b141d405-9cb3-4811-ae5f-ce95eb4cf972 new file mode 100644 index 00000000000000..899ac0d7170558 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/b141d405-9cb3-4811-ae5f-ce95eb4cf972 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/b601976c-7434-4928-8cfa-47a0688c75e5 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/b601976c-7434-4928-8cfa-47a0688c75e5 new file mode 100644 index 00000000000000..7bdcb98af9b491 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/b601976c-7434-4928-8cfa-47a0688c75e5 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/d21dd724-7b5c-48a7-8cb4-1ac66ecdbd01 b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/d21dd724-7b5c-48a7-8cb4-1ac66ecdbd01 new file mode 100644 index 00000000000000..0eb542e15445cc Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/d21dd724-7b5c-48a7-8cb4-1ac66ecdbd01 differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/d431a59d-f25c-471a-b389-12d46d10d4bd b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/d431a59d-f25c-471a-b389-12d46d10d4bd new file mode 100644 index 00000000000000..7bdcb98af9b491 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/d431a59d-f25c-471a-b389-12d46d10d4bd differ diff --git a/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/ee9ff9f9-03b7-4642-809c-dbeae15aefeb b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/ee9ff9f9-03b7-4642-809c-dbeae15aefeb new file mode 100644 index 00000000000000..b9b674eadd0cc1 Binary files /dev/null and b/flink-tests/src/test/resources/new-stateful-udf-migration-itcase-flink2.3-rocksdb-savepoint/ee9ff9f9-03b7-4642-809c-dbeae15aefeb differ diff --git a/flink-tests/src/test/resources/operatorstate/complexKeyed-flink2.3/_metadata b/flink-tests/src/test/resources/operatorstate/complexKeyed-flink2.3/_metadata new file mode 100644 index 00000000000000..5fee184c1b84dc Binary files /dev/null and b/flink-tests/src/test/resources/operatorstate/complexKeyed-flink2.3/_metadata differ diff --git a/flink-tests/src/test/resources/operatorstate/nonKeyed-flink2.3/_metadata b/flink-tests/src/test/resources/operatorstate/nonKeyed-flink2.3/_metadata new file mode 100644 index 00000000000000..6b19f4cdee6f57 Binary files /dev/null and b/flink-tests/src/test/resources/operatorstate/nonKeyed-flink2.3/_metadata differ diff --git a/pom.xml b/pom.xml index 0a38cfe7c056af..670753e010bcb6 100644 --- a/pom.xml +++ b/pom.xml @@ -973,6 +973,27 @@ under the License. false + + spotless-ratchet + + + spotless.ratchetFrom + + + + + + + com.diffplug.spotless + spotless-maven-plugin + + ${spotless.ratchetFrom} + + + + + + scala-2.12