From b41e752c3f4cb9a80f657ab4927fdaec9fc1e931 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Fri, 17 Jul 2026 18:04:21 +1000 Subject: [PATCH 1/4] feat(int_engine_new_payload): correlate gloas payloads via execution_payload events On gloas (ePBS) networks the beacon block no longer embeds the execution payload, so fct_block_head.execution_payload_block_hash is empty and every snooper observation was dropped for lack of slot context - the engine timings tables stayed empty on glamsterdam devnets even with raw data flowing. Map each revealed payload's block hash to its beacon block root via the beacon_api_eth_v1_events_execution_payload SSE events and coalesce that into block_context. The events table only exists on networks running gloas-era xatu, so it is referenced through EXTERNAL_CLUSTER behind a GLOAS_PAYLOAD_EVENTS_DATABASE env gate instead of a declared dependency: deployments opt in by setting the env to the raw database holding the table, and everywhere else the template renders the exact same SQL as before. Verified against live data: glamsterdam-devnet-7 correlates 99.7% of 23k engine events over 2h (42 nodes); on mainnet the gated-off rendering returns checksum-identical results to the current model. Claude-Session: https://claude.ai/code/session_01J2ShnSwGwH4iZuXXwbLgvP --- .../int_engine_new_payload.sql | 54 ++++++++++++++----- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/models/transformations/int_engine_new_payload.sql b/models/transformations/int_engine_new_payload.sql index b5d14295..28006850 100644 --- a/models/transformations/int_engine_new_payload.sql +++ b/models/transformations/int_engine_new_payload.sql @@ -19,26 +19,54 @@ dependencies: INSERT INTO `{{ .self.database }}`.`{{ .self.table }}` WITH +{{ if .env.GLOAS_PAYLOAD_EVENTS_DATABASE }} +-- Gloas (ePBS): the beacon block no longer embeds the execution payload, so +-- fct_block_head carries no execution_payload_block_hash. The execution_payload +-- SSE events map each revealed payload's block hash to its beacon block root. +-- Referenced via EXTERNAL_CLUSTER instead of a declared dependency because the +-- table only exists on networks running gloas-era xatu; deployments opt in by +-- setting GLOAS_PAYLOAD_EVENTS_DATABASE to the raw database holding the table. +payload_events AS ( + SELECT + block_root, + any(block_hash) AS payload_block_hash + FROM cluster('{{ .env.EXTERNAL_CLUSTER }}', `{{ .env.GLOAS_PAYLOAD_EVENTS_DATABASE }}`.`beacon_api_eth_v1_events_execution_payload{{ .clickhouse.local_suffix }}`) + WHERE meta_network_name = '{{ .env.NETWORK }}' + AND slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE + AND fromUnixTimestamp({{ .bounds.end }}) + INTERVAL 5 MINUTE + AND block_hash != '' + GROUP BY block_root +), +{{ end }} -- Get slot context and block metadata from fct_block_head -- This provides CL context (slot, epoch, block_root, proposer_index) that execution_engine lacks -- Join on execution_payload_block_hash to correlate EL block hash with CL slot block_context AS ( SELECT - slot, - slot_start_date_time, - epoch, - epoch_start_date_time, - block_root, - parent_root, - proposer_index, - execution_payload_block_hash, - block_total_bytes, - block_total_bytes_compressed, - block_version - FROM {{ index .dep "{{transformation}}" "fct_block_head" "helpers" "from" }} FINAL + bh.slot AS slot, + bh.slot_start_date_time AS slot_start_date_time, + bh.epoch AS epoch, + bh.epoch_start_date_time AS epoch_start_date_time, + bh.block_root AS block_root, + bh.parent_root AS parent_root, + bh.proposer_index AS proposer_index, +{{ if .env.GLOAS_PAYLOAD_EVENTS_DATABASE }} + -- Pre-gloas blocks carry their payload hash; gloas blocks fall back to + -- the payload observed for their block root on the SSE layer + coalesce(nullif(bh.execution_payload_block_hash, ''), nullif(pe.payload_block_hash, '')) AS execution_payload_block_hash, +{{ else }} + bh.execution_payload_block_hash AS execution_payload_block_hash, +{{ end }} + bh.block_total_bytes AS block_total_bytes, + bh.block_total_bytes_compressed AS block_total_bytes_compressed, + bh.block_version AS block_version + FROM {{ index .dep "{{transformation}}" "fct_block_head" "helpers" "from" }} AS bh FINAL +{{ if .env.GLOAS_PAYLOAD_EVENTS_DATABASE }} + GLOBAL LEFT JOIN payload_events pe ON bh.block_root = pe.block_root +{{ end }} -- Use wider window to ensure we catch all blocks that might match engine events -- Engine events use event_date_time which may differ from slot_start_date_time - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE + WHERE bh.slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE AND fromUnixTimestamp({{ .bounds.end }}) + INTERVAL 5 MINUTE AND execution_payload_block_hash IS NOT NULL AND execution_payload_block_hash != '' From f16af615f42fb144288e4647be37564d9ffb6088 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Fri, 17 Jul 2026 18:10:04 +1000 Subject: [PATCH 2/4] refactor(int_engine_new_payload): drop the env gate, declare the dependency Correlation now works on every network with no opt-in: the execution_payload events external is a declared dependency, OR-grouped with beacon_api_eth_v2_beacon_block so networks where the table is empty or absent schedule unaffected, and the coalesce prefers the block's own embedded payload hash so mixed pre/post-gloas histories work in one query. Requires the gloas event-table migrations to exist on a network's raw database before this model rolls there; the glamsterdam devnets already have them. Claude-Session: https://claude.ai/code/session_01J2ShnSwGwH4iZuXXwbLgvP --- ...on_api_eth_v1_events_execution_payload.sql | 32 +++++++++++++++++++ .../int_engine_new_payload.sql | 23 ++++++------- 2 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 models/external/beacon_api_eth_v1_events_execution_payload.sql diff --git a/models/external/beacon_api_eth_v1_events_execution_payload.sql b/models/external/beacon_api_eth_v1_events_execution_payload.sql new file mode 100644 index 00000000..afc1c8af --- /dev/null +++ b/models/external/beacon_api_eth_v1_events_execution_payload.sql @@ -0,0 +1,32 @@ +--- +table: beacon_api_eth_v1_events_execution_payload +cache: + incremental_scan_interval: 5s + full_scan_interval: 24h +interval: + type: slot +lag: 12 +--- +SELECT + {{ if .cache.is_incremental_scan }} + '{{ .cache.previous_min }}' as min, + {{ else }} + toUnixTimestamp(min(slot_start_date_time)) as min, + {{ end }} + toUnixTimestamp(max(slot_start_date_time)) as max +FROM {{ .self.helpers.from }} +WHERE + meta_network_name = '{{ .env.NETWORK }}' + + -- previous_max if incremental scan and is set, otherwise default/env + {{- $ts := default "0" .env.EXTERNAL_MODEL_MIN_TIMESTAMP -}} + {{- if .cache.is_incremental_scan -}} + {{- if .cache.previous_max -}} + {{- $ts = .cache.previous_max -}} + {{- end -}} + {{- end }} + AND slot_start_date_time >= fromUnixTimestamp({{ $ts }}) + {{- if .cache.is_incremental_scan }} + AND slot_start_date_time <= fromUnixTimestamp({{ $ts }}) + {{ default "100000" .env.EXTERNAL_MODEL_SCAN_SIZE_TIMESTAMP }} + {{- end }} + diff --git a/models/transformations/int_engine_new_payload.sql b/models/transformations/int_engine_new_payload.sql index 28006850..4fa6c998 100644 --- a/models/transformations/int_engine_new_payload.sql +++ b/models/transformations/int_engine_new_payload.sql @@ -15,29 +15,30 @@ dependencies: - - "{{external}}.execution_engine_new_payload" - "{{external}}.consensus_engine_api_new_payload" - "{{transformation}}.fct_block_head" + # Gloas payload correlation source. OR-grouped with the beacon block table + # (already required transitively via fct_block_head) so networks where the + # execution_payload events table is empty or absent schedule unaffected. + - - "{{external}}.beacon_api_eth_v1_events_execution_payload" + - "{{external}}.beacon_api_eth_v2_beacon_block" --- INSERT INTO `{{ .self.database }}`.`{{ .self.table }}` WITH -{{ if .env.GLOAS_PAYLOAD_EVENTS_DATABASE }} -- Gloas (ePBS): the beacon block no longer embeds the execution payload, so --- fct_block_head carries no execution_payload_block_hash. The execution_payload --- SSE events map each revealed payload's block hash to its beacon block root. --- Referenced via EXTERNAL_CLUSTER instead of a declared dependency because the --- table only exists on networks running gloas-era xatu; deployments opt in by --- setting GLOAS_PAYLOAD_EVENTS_DATABASE to the raw database holding the table. +-- fct_block_head carries no execution_payload_block_hash there. The +-- execution_payload SSE events map each revealed payload's block hash to its +-- beacon block root. Empty on pre-gloas networks, where it contributes nothing. payload_events AS ( SELECT block_root, any(block_hash) AS payload_block_hash - FROM cluster('{{ .env.EXTERNAL_CLUSTER }}', `{{ .env.GLOAS_PAYLOAD_EVENTS_DATABASE }}`.`beacon_api_eth_v1_events_execution_payload{{ .clickhouse.local_suffix }}`) + FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_execution_payload" "helpers" "from" }} WHERE meta_network_name = '{{ .env.NETWORK }}' AND slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE AND fromUnixTimestamp({{ .bounds.end }}) + INTERVAL 5 MINUTE AND block_hash != '' GROUP BY block_root ), -{{ end }} -- Get slot context and block metadata from fct_block_head -- This provides CL context (slot, epoch, block_root, proposer_index) that execution_engine lacks -- Join on execution_payload_block_hash to correlate EL block hash with CL slot @@ -50,20 +51,14 @@ block_context AS ( bh.block_root AS block_root, bh.parent_root AS parent_root, bh.proposer_index AS proposer_index, -{{ if .env.GLOAS_PAYLOAD_EVENTS_DATABASE }} -- Pre-gloas blocks carry their payload hash; gloas blocks fall back to -- the payload observed for their block root on the SSE layer coalesce(nullif(bh.execution_payload_block_hash, ''), nullif(pe.payload_block_hash, '')) AS execution_payload_block_hash, -{{ else }} - bh.execution_payload_block_hash AS execution_payload_block_hash, -{{ end }} bh.block_total_bytes AS block_total_bytes, bh.block_total_bytes_compressed AS block_total_bytes_compressed, bh.block_version AS block_version FROM {{ index .dep "{{transformation}}" "fct_block_head" "helpers" "from" }} AS bh FINAL -{{ if .env.GLOAS_PAYLOAD_EVENTS_DATABASE }} GLOBAL LEFT JOIN payload_events pe ON bh.block_root = pe.block_root -{{ end }} -- Use wider window to ensure we catch all blocks that might match engine events -- Engine events use event_date_time which may differ from slot_start_date_time WHERE bh.slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE From c34da71d4571e829bd704faf8df62c794f9d7399 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Fri, 17 Jul 2026 18:50:15 +1000 Subject: [PATCH 3/4] fix: drop semicolon from comment that truncated the rendered SQL CBT's statement splitter cuts on semicolons without comment awareness. Claude-Session: https://claude.ai/code/session_01J2ShnSwGwH4iZuXXwbLgvP --- models/transformations/int_engine_new_payload.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/transformations/int_engine_new_payload.sql b/models/transformations/int_engine_new_payload.sql index 4fa6c998..69d87487 100644 --- a/models/transformations/int_engine_new_payload.sql +++ b/models/transformations/int_engine_new_payload.sql @@ -51,7 +51,7 @@ block_context AS ( bh.block_root AS block_root, bh.parent_root AS parent_root, bh.proposer_index AS proposer_index, - -- Pre-gloas blocks carry their payload hash; gloas blocks fall back to + -- Pre-gloas blocks carry their payload hash. Gloas blocks fall back to -- the payload observed for their block root on the SSE layer coalesce(nullif(bh.execution_payload_block_hash, ''), nullif(pe.payload_block_hash, '')) AS execution_payload_block_hash, bh.block_total_bytes AS block_total_bytes, From 92f15159d83a5e63b44c3a8118c4b3565aec9a95 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Fri, 17 Jul 2026 19:14:58 +1000 Subject: [PATCH 4/4] fix: enrich fct_block_head payload hash at the source instead The engine fct models correlate against fct_block_head independently of int_engine_new_payload, so the int-level patch left them empty on gloas. Coalesce the payload hash into fct_block_head from the execution_payload SSE events and revert the int model - every consumer of the column is repaired at once. Claude-Session: https://claude.ai/code/session_01J2ShnSwGwH4iZuXXwbLgvP --- models/transformations/fct_block_head.sql | 29 +++++++++-- .../int_engine_new_payload.sql | 49 +++++-------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/models/transformations/fct_block_head.sql b/models/transformations/fct_block_head.sql index 644a79be..3825b90e 100644 --- a/models/transformations/fct_block_head.sql +++ b/models/transformations/fct_block_head.sql @@ -14,16 +14,36 @@ tags: - head dependencies: - "{{external}}.beacon_api_eth_v2_beacon_block" + # Gloas payload-hash source. OR-grouped with the beacon block table so + # networks where the execution_payload events table is empty or absent + # schedule unaffected. + - - "{{external}}.beacon_api_eth_v1_events_execution_payload" + - "{{external}}.beacon_api_eth_v2_beacon_block" --- INSERT INTO `{{ .self.database }}`.`{{ .self.table }}` +WITH +-- Gloas (ePBS): the beacon block no longer embeds the execution payload, so +-- its execution_payload_block_hash is empty in the block table. The +-- execution_payload SSE events map each revealed payload's block hash to its +-- beacon block root. Empty on pre-gloas networks, where it contributes nothing. +payload_events AS ( + SELECT + block_root, + any(block_hash) AS payload_block_hash + FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_execution_payload" "helpers" "from" }} + WHERE meta_network_name = '{{ .env.NETWORK }}' + AND slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + AND block_hash != '' + GROUP BY block_root +) SELECT fromUnixTimestamp({{ .task.start }}) as updated_date_time, argMax(slot, updated_date_time) AS slot, slot_start_date_time, argMax(epoch, updated_date_time) AS epoch, argMax(epoch_start_date_time, updated_date_time) AS epoch_start_date_time, - block_root, + bb.block_root AS block_root, argMax(block_version, updated_date_time) AS block_version, argMax(block_total_bytes, updated_date_time) AS block_total_bytes, argMax(block_total_bytes_compressed, updated_date_time) AS block_total_bytes_compressed, @@ -32,7 +52,9 @@ SELECT argMax(proposer_index, updated_date_time) AS proposer_index, argMax(eth1_data_block_hash, updated_date_time) AS eth1_data_block_hash, argMax(eth1_data_deposit_root, updated_date_time) AS eth1_data_deposit_root, - argMax(execution_payload_block_hash, updated_date_time) AS execution_payload_block_hash, + -- Pre-gloas blocks carry their payload hash. Gloas blocks fall back to + -- the payload observed for their block root on the SSE layer + coalesce(nullif(argMax(execution_payload_block_hash, updated_date_time), ''), any(pe.payload_block_hash), '') AS execution_payload_block_hash, argMax(execution_payload_block_number, updated_date_time) AS execution_payload_block_number, argMax(execution_payload_fee_recipient, updated_date_time) AS execution_payload_fee_recipient, argMax(execution_payload_base_fee_per_gas, updated_date_time) AS execution_payload_base_fee_per_gas, @@ -45,7 +67,8 @@ SELECT argMax(execution_payload_transactions_count, updated_date_time) AS execution_payload_transactions_count, argMax(execution_payload_transactions_total_bytes, updated_date_time) AS execution_payload_transactions_total_bytes, argMax(execution_payload_transactions_total_bytes_compressed, updated_date_time) AS execution_payload_transactions_total_bytes_compressed -FROM {{ index .dep "{{external}}" "beacon_api_eth_v2_beacon_block" "helpers" "from" }} FINAL +FROM {{ index .dep "{{external}}" "beacon_api_eth_v2_beacon_block" "helpers" "from" }} AS bb FINAL +GLOBAL LEFT JOIN payload_events pe ON bb.block_root = pe.block_root WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' GROUP BY slot_start_date_time, block_root diff --git a/models/transformations/int_engine_new_payload.sql b/models/transformations/int_engine_new_payload.sql index 69d87487..b5d14295 100644 --- a/models/transformations/int_engine_new_payload.sql +++ b/models/transformations/int_engine_new_payload.sql @@ -15,53 +15,30 @@ dependencies: - - "{{external}}.execution_engine_new_payload" - "{{external}}.consensus_engine_api_new_payload" - "{{transformation}}.fct_block_head" - # Gloas payload correlation source. OR-grouped with the beacon block table - # (already required transitively via fct_block_head) so networks where the - # execution_payload events table is empty or absent schedule unaffected. - - - "{{external}}.beacon_api_eth_v1_events_execution_payload" - - "{{external}}.beacon_api_eth_v2_beacon_block" --- INSERT INTO `{{ .self.database }}`.`{{ .self.table }}` WITH --- Gloas (ePBS): the beacon block no longer embeds the execution payload, so --- fct_block_head carries no execution_payload_block_hash there. The --- execution_payload SSE events map each revealed payload's block hash to its --- beacon block root. Empty on pre-gloas networks, where it contributes nothing. -payload_events AS ( - SELECT - block_root, - any(block_hash) AS payload_block_hash - FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_execution_payload" "helpers" "from" }} - WHERE meta_network_name = '{{ .env.NETWORK }}' - AND slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE - AND fromUnixTimestamp({{ .bounds.end }}) + INTERVAL 5 MINUTE - AND block_hash != '' - GROUP BY block_root -), -- Get slot context and block metadata from fct_block_head -- This provides CL context (slot, epoch, block_root, proposer_index) that execution_engine lacks -- Join on execution_payload_block_hash to correlate EL block hash with CL slot block_context AS ( SELECT - bh.slot AS slot, - bh.slot_start_date_time AS slot_start_date_time, - bh.epoch AS epoch, - bh.epoch_start_date_time AS epoch_start_date_time, - bh.block_root AS block_root, - bh.parent_root AS parent_root, - bh.proposer_index AS proposer_index, - -- Pre-gloas blocks carry their payload hash. Gloas blocks fall back to - -- the payload observed for their block root on the SSE layer - coalesce(nullif(bh.execution_payload_block_hash, ''), nullif(pe.payload_block_hash, '')) AS execution_payload_block_hash, - bh.block_total_bytes AS block_total_bytes, - bh.block_total_bytes_compressed AS block_total_bytes_compressed, - bh.block_version AS block_version - FROM {{ index .dep "{{transformation}}" "fct_block_head" "helpers" "from" }} AS bh FINAL - GLOBAL LEFT JOIN payload_events pe ON bh.block_root = pe.block_root + slot, + slot_start_date_time, + epoch, + epoch_start_date_time, + block_root, + parent_root, + proposer_index, + execution_payload_block_hash, + block_total_bytes, + block_total_bytes_compressed, + block_version + FROM {{ index .dep "{{transformation}}" "fct_block_head" "helpers" "from" }} FINAL -- Use wider window to ensure we catch all blocks that might match engine events -- Engine events use event_date_time which may differ from slot_start_date_time - WHERE bh.slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 5 MINUTE AND fromUnixTimestamp({{ .bounds.end }}) + INTERVAL 5 MINUTE AND execution_payload_block_hash IS NOT NULL AND execution_payload_block_hash != ''