From ab5b5497d0c264cee412a69d3e38f32f8a86e18c Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Thu, 16 Jul 2026 14:46:04 +1000 Subject: [PATCH 1/3] fix(models): stop dead libp2p gossip feed stalling block models libp2p_gossipsub_beacon_block is published by a pipeline deployed independently of the beacon API sentries. When it stops (e.g. a sidecar predating a fork's gossip changes, as happened on glamsterdam-devnet-7 at gloas activation), fct_block_first_seen_by_node and fct_block_proposer_head stall forever because it is a hard dependency. OR-group it with beacon_api_eth_v1_events_block so the fresher source gates forwardfill. --- models/transformations/fct_block_first_seen_by_node.sql | 8 ++++++-- models/transformations/fct_block_proposer_head.sql | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/models/transformations/fct_block_first_seen_by_node.sql b/models/transformations/fct_block_first_seen_by_node.sql index 778deca3..cd898c0c 100644 --- a/models/transformations/fct_block_first_seen_by_node.sql +++ b/models/transformations/fct_block_first_seen_by_node.sql @@ -12,9 +12,13 @@ tags: - block dependencies: - "{{external}}.beacon_api_eth_v1_events_block_gossip" - - "{{external}}.beacon_api_eth_v1_events_block" - "{{external}}.beacon_api_eth_v1_events_head" - - "{{external}}.libp2p_gossipsub_beacon_block" + # The libp2p pipeline is deployed independently of the beacon API sentries + # and can stop publishing (e.g. a sidecar that predates a fork's gossip + # changes). OR-grouping it with the block event keeps a dead gossip feed + # from stalling forwardfill; whichever source is fresher gates the interval. + - - "{{external}}.beacon_api_eth_v1_events_block" + - "{{external}}.libp2p_gossipsub_beacon_block" --- INSERT INTO `{{ .self.database }}`.`{{ .self.table }}` diff --git a/models/transformations/fct_block_proposer_head.sql b/models/transformations/fct_block_proposer_head.sql index 5b66323d..4a1736b4 100644 --- a/models/transformations/fct_block_proposer_head.sql +++ b/models/transformations/fct_block_proposer_head.sql @@ -14,10 +14,14 @@ tags: - head dependencies: - "{{external}}.beacon_api_eth_v1_events_block_gossip" - - "{{external}}.beacon_api_eth_v1_events_block" - - "{{external}}.beacon_api_eth_v1_proposer_duty" - "{{external}}.canonical_beacon_proposer_duty" - - "{{external}}.libp2p_gossipsub_beacon_block" + # The libp2p pipeline is deployed independently of the beacon API sentries + # and can stop publishing (e.g. a sidecar that predates a fork's gossip + # changes). OR-grouping it with the block event keeps a dead gossip feed + # from stalling forwardfill; whichever source is fresher gates the interval. + - - "{{external}}.beacon_api_eth_v1_events_block" + - "{{external}}.libp2p_gossipsub_beacon_block" --- INSERT INTO `{{ .self.database }}`.`{{ .self.table }}` From db92da15aeda50614e42087cc49bb614db94d0e9 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Thu, 16 Jul 2026 14:46:24 +1000 Subject: [PATCH 2/3] ci(docker): publish images for release/** branches Per-devnet release branches (release/glamsterdam-devnet-7) carry models for raw tables that only exist on those devnets, so they cannot ship to mainnet via master. Publishing branch-ref tags (release-glamsterdam-devnet-7) lets devnet deployments pin them, mirroring xatu's release branch convention. --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b304c6ec..dfe965d6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,10 @@ on: push: branches: - master + # Per-devnet release branches (e.g. release/glamsterdam-devnet-7) publish + # persistent branch-ref tags like release-glamsterdam-devnet-7 so devnet + # deployments can pin models that must not ship to mainnet yet. + - 'release/**' workflow_dispatch: inputs: cbt_tag: From d417bbeda34dad79640ff3e0eb80663a438a0b46 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Fri, 17 Jul 2026 18:04:26 +1000 Subject: [PATCH 3/3] fix(models): re-scan trailing 90s so late rows repair processed slots The OR-group lets forwardfill run at the fresher source's pace, so rows from the slower source can land after their slot's interval was already processed. Widen every data scan 90s behind the interval start so subsequent tasks recompute the trailing slots with the fuller data; ReplacingMergeTree(updated_date_time) keeps the newest version. The window must widen every source uniformly - recomputing a slot from a subset of sources would replace a complete row with a worse one. The duty-completeness gate in fct_block_proposer_head keeps the exact interval: its 12s-grid arithmetic assumes the scan window equals the interval, and trailing slots were gated by their own task. --- .../transformations/fct_block_first_seen_by_node.sql | 8 ++++---- models/transformations/fct_block_proposer_head.sql | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/models/transformations/fct_block_first_seen_by_node.sql b/models/transformations/fct_block_first_seen_by_node.sql index cd898c0c..4804e9c7 100644 --- a/models/transformations/fct_block_first_seen_by_node.sql +++ b/models/transformations/fct_block_first_seen_by_node.sql @@ -45,7 +45,7 @@ WITH combined_events AS ( meta_consensus_version, meta_consensus_implementation FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_block" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' UNION ALL @@ -72,7 +72,7 @@ WITH combined_events AS ( meta_consensus_version, meta_consensus_implementation FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_block_gossip" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' UNION ALL @@ -99,7 +99,7 @@ WITH combined_events AS ( meta_consensus_version, meta_consensus_implementation FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_head" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' UNION ALL @@ -135,7 +135,7 @@ WITH combined_events AS ( '' END AS meta_consensus_implementation FROM {{ index .dep "{{external}}" "libp2p_gossipsub_beacon_block" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' ) SELECT diff --git a/models/transformations/fct_block_proposer_head.sql b/models/transformations/fct_block_proposer_head.sql index 4a1736b4..1c58523e 100644 --- a/models/transformations/fct_block_proposer_head.sql +++ b/models/transformations/fct_block_proposer_head.sql @@ -51,7 +51,7 @@ WITH proposer_duties AS ( proposer_validator_index, proposer_pubkey FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_proposer_duty" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' UNION ALL SELECT @@ -62,7 +62,7 @@ WITH proposer_duties AS ( proposer_validator_index, proposer_pubkey FROM {{ index .dep "{{external}}" "canonical_beacon_proposer_duty" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' ) ), @@ -129,7 +129,7 @@ block_gossip AS ( epoch_start_date_time, block as block_root FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_block_gossip" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' ), @@ -141,7 +141,7 @@ block_events AS ( epoch_start_date_time, block as block_root FROM {{ index .dep "{{external}}" "beacon_api_eth_v1_events_block" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' ), @@ -154,7 +154,7 @@ gossipsub_blocks AS ( block as block_root, proposer_index FROM {{ index .dep "{{external}}" "libp2p_gossipsub_beacon_block" "helpers" "from" }} FINAL - WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) AND fromUnixTimestamp({{ .bounds.end }}) + WHERE slot_start_date_time BETWEEN fromUnixTimestamp({{ .bounds.start }}) - INTERVAL 90 SECOND AND fromUnixTimestamp({{ .bounds.end }}) AND meta_network_name = '{{ .env.NETWORK }}' ),