From d3da683b03c0e5bb0bba0802767301f27ef384ea Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 15:08:14 +0000 Subject: [PATCH 1/7] Alert on Piri's chain head and PDP proving gauges Four rules in the appliance group over the gauges Piri's PDP pipeline exports (piri_chain_head_*, piri_pdp_proofset_*, piri_pdp_task_last_success_timestamp_seconds), starting from the PromQL in Piri's monitoring docs: - the chain head is more than five minutes old (critical); - Piri is reporting but its chain scheduler has never applied a head (warning), gated on the database-backed gauges so a Piri too old to export the head does not match; - a proof set is past its challenge window, with the current epoch extrapolated from the wall clock so a stale head still fires, and a `for` long enough to ride out curio scheduling the next period just after the window closes (critical); - no successful PDPv0_Prove in 1.5 proving periods (warning). They select Piri's series by job="forge/piri" and the appliance label, since the host exporter's service_name in appliance_matcher does not apply to them. The chain-notification log rule stays until the alerting stages run a Piri with these gauges, and its comment now says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ Co-Authored-By: Petra Jaros --- terraform/envs/grafana/alerts.tf | 403 ++++++++++++++++++++++++++++++- 1 file changed, 400 insertions(+), 3 deletions(-) diff --git a/terraform/envs/grafana/alerts.tf b/terraform/envs/grafana/alerts.tf index 9533279f..935810f9 100644 --- a/terraform/envs/grafana/alerts.tf +++ b/terraform/envs/grafana/alerts.tf @@ -1,16 +1,18 @@ # Grafana alert rules for Forge, in their own folder. folders.tf sets out why # these are git only where the dashboards beside them are not. # -# Five rules, in two groups so Central and the appliances can evaluate at +# Nine rules, in two groups so Central and the appliances can evaluate at # different intervals: CloudWatch publishes once a minute, the appliance host # exporter is scraped once a minute but nothing here needs that resolution. # -# Three of the five came from rules built by hand in the UI and exported; the +# Three of the nine came from rules built by hand in the UI and exported; the # shapes below follow that export rather than the provider's documentation, # which is why the expression stages address __expr__ and put their own refId in # the condition's query.params. The fourth is FIL-1209's disk rule. The fifth, # Piri's chain notifications, is the only one that reads logs rather than -# metrics. +# metrics. The last four read the chain head and proving gauges Piri exports +# over OTLP, and between them replace that log rule once every alerting stage +# runs a Piri that has them. # # Of the seven alerts under FIL-1145 only FIL-1209 states a threshold. The rest # say "too high", or defer to an SLO that has not been written (FIL-1242), or @@ -85,6 +87,15 @@ locals { # appliance matcher narrows it to the alerting stages. piri_log_matcher = "appliance=~\"(${local.stages})-.*\", service_name=~\"appliance-.*-piri\"" + # Piri's OTLP metrics, and the target_info series that carries its resource + # attributes. Alloy's OTLP conversion sets job to service.namespace/ + # service.name, forge/piri since Piri reports the forge namespace, and the + # remote writer's external_labels stamp appliance, region and node on every + # series it sends, target_info included (infra-nodes + # nodes/dev/platform/config/alloy/config.alloy). appliance_matcher cannot be + # reused: its service_name is the host exporter's. + piri_metric_matcher = "appliance=~\"(${local.stages})-.*\", job=\"forge/piri\"" + # Fields every Prometheus query stage carries. `instant` picks one sample per # series and needs no reduce before the threshold; a range query does. query_defaults = { @@ -515,6 +526,12 @@ resource "grafana_rule_group" "appliance" { # no_data_state is OK for the reason the 5xx rule gives: count_over_time # returns a series only for a node that logged the line, so an empty result is # the healthy state. An instant query, so no reduce stage. + # + # Superseded by "Piri's chain head is stale" and the three rules after it, + # which read the head and the proving schedule directly, once the alerting + # stages run a Piri that exports those gauges. Kept until then, since a Piri + # without them leaves this rule the only one watching the chain; delete it + # after that rather than run both. rule { name = "Piri has stopped receiving chain notifications" condition = "B" @@ -588,6 +605,386 @@ resource "grafana_rule_group" "appliance" { }) } } + + # The four rules below read gauges Piri exports from its PDP pipeline (piri + # docs/content/operator-guide/monitoring.md, "PDP Proving Health"), starting + # from the PromQL given there. They are grouped by node as well as region for + # the reason the disk rule gives. Filecoin epochs are thirty seconds, which is + # where every 30 below comes from. + + # The timestamp of the last tipset Piri's chain scheduler applied, against the + # wall clock. Five minutes is ten tipsets, far past what null rounds or a slow + # block produce, so this is Lotus stalled or unreachable, or Piri's + # subscription to it stuck. + # + # `for` is 5m, the shortest pending period a group evaluated every five + # minutes can give: one more evaluation to confirm the first. The threshold + # already holds five minutes of grace. + # + # no_data_state is OK: a Piri that has not seen a head since it started, or + # one that exports no chain gauges at all, has no series here. The first is + # the next rule's job; the second is a Piri older than these metrics. + rule { + name = "Piri's chain head is stale" + condition = "B" + for = "5m" + no_data_state = "OK" + exec_err_state = "Error" + + annotations = { + summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) has not seen a new chain head for more than five minutes" + description = "The last tipset Piri's chain scheduler applied is more than five minutes old. Filecoin produces one every thirty seconds, so the Lotus it reads the chain from is stalled, out of sync or unreachable, or Piri's subscription to it is stuck, and Piri cannot schedule or submit proofs. Check the sync status of that Lotus (`lotus sync wait` or `lotus chain head` on the host that owns it) and run `piri status` in the Piri container." + runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" + } + + labels = { + team = "forge" + component = "appliance" + severity = "critical" + } + + data { + ref_id = "A" + datasource_uid = var.prometheus_datasource_uid + + relative_time_range { + from = 600 + to = 0 + } + + model = jsonencode(merge(local.query_defaults, { + refId = "A" + instant = true + range = false + intervalMs = 1000 + expr = <<-PROMQL + time() - max by (appliance, region, node) ( + piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}} + ) + PROMQL + })) + } + + data { + ref_id = "B" + query_type = "expression" + datasource_uid = "__expr__" + + relative_time_range { + from = 0 + to = 0 + } + + model = jsonencode({ + refId = "B" + type = "threshold" + expression = "A" + datasource = { type = "__expr__", uid = "__expr__" } + intervalMs = 1000 + maxDataPoints = 43200 + conditions = [{ + type = "query" + operator = { type = "and" } + query = { params = ["B"] } + reducer = { type = "last", params = [] } + evaluator = { type = "gt", params = [300] } + }] + }) + } + } + + # Piri is reporting but its chain scheduler has never applied a head. The head + # gauges appear only once the first tipset arrives after start-up, which with + # a working Lotus is within seconds, so a Piri that restarted against a Lotus + # it cannot reach has no head series at all and the rule above never sees it. + # + # target_info is the evidence Piri is up: it carries the same job and + # appliance, region and node labels as every other Piri series, so the same + # matcher applies to it. On its own it would also match every Piri too old to + # export the head, which is every Piri until these metrics are deployed. So + # the node must also export one of the gauges Piri reads from its database, + # which do not depend on the chain: they prove the running Piri has the + # metrics, and so that the missing head means something. A node with no proof + # sets and no successful proof yet is left out, and has nothing to lose. + # + # Five-minute windows are ten of Piri's thirty-second pushes. `for` is 10m, so + # a restart that takes a moment to reach Lotus does not fire. + # + # no_data_state is OK, and not NoData, even though "no head" is what the rule + # is about: the `unless` is what finds the missing head, so the result is + # empty on every healthy node, and NoData would page for the whole fleet. It + # also means a Piri that has stopped reporting entirely, target_info and all, + # is not caught here; nothing in this file watches for that yet. + rule { + name = "Piri is reporting but has no chain head" + condition = "B" + for = "10m" + no_data_state = "OK" + exec_err_state = "Error" + + annotations = { + summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) is running but has not seen a chain head" + description = "Piri is exporting metrics but its chain scheduler has not applied a single tipset since it started, so it cannot schedule or submit proofs. Usually the Lotus it reads the chain from is unreachable or was down when Piri started. Check that Lotus is up and synced (`lotus sync wait` or `lotus chain head` on the host that owns it), that Piri's Lotus endpoint and token are right, and run `piri status` in the Piri container." + runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" + } + + labels = { + team = "forge" + component = "appliance" + severity = "warning" + } + + data { + ref_id = "A" + datasource_uid = var.prometheus_datasource_uid + + relative_time_range { + from = 600 + to = 0 + } + + model = jsonencode(merge(local.query_defaults, { + refId = "A" + instant = true + range = false + intervalMs = 1000 + expr = <<-PROMQL + max by (appliance, region, node) ( + present_over_time(target_info{${local.piri_metric_matcher}}[5m]) + ) + and + ( + max by (appliance, region, node) ( + present_over_time(piri_pdp_task_last_success_timestamp_seconds{${local.piri_metric_matcher}}[5m]) + ) + or + max by (appliance, region, node) ( + present_over_time(piri_pdp_proofset_next_challenge_epoch{${local.piri_metric_matcher}}[5m]) + ) + ) + unless + max by (appliance, region, node) ( + present_over_time(piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}}[5m]) + ) + PROMQL + })) + } + + data { + ref_id = "B" + query_type = "expression" + datasource_uid = "__expr__" + + relative_time_range { + from = 0 + to = 0 + } + + model = jsonencode({ + refId = "B" + type = "threshold" + expression = "A" + datasource = { type = "__expr__", uid = "__expr__" } + intervalMs = 1000 + maxDataPoints = 43200 + conditions = [{ + type = "query" + operator = { type = "and" } + query = { params = ["B"] } + reducer = { type = "last", params = [] } + evaluator = { type = "gt", params = [0] } + }] + }) + } + } + + # A proof set whose challenge window has closed without its next proving + # period being scheduled: epochs past the window's close, per proof set. The + # current epoch is the last head plus the wall-clock time since it, so this + # still fires when the head itself is stale. + # + # Some overshoot is normal. Curio's NextProvingPeriod watcher + # (tasks/pdpv0/task_next_pp.go) only queues the task once a head reaches + # prove_at_epoch + challenge_window, and the task moves prove_at_epoch on + # when it sends nextProvingPeriod, without waiting for it to land. Add + # harmonytask's three-second poll, a handful of contract reads and Piri's + # thirty-second push, and a healthy proof set is a few epochs past its window + # for a minute or two each period, which an evaluation can catch. `for` is + # 10m: three evaluations in a row, twenty epochs, which that never reaches, + # while a scheduling task that is failing, or held back by the hundred-epoch + # backoff curio applies after a failed attempt, stays past the window for + # longer than that. + # + # no_data_state is OK: a node with no proof sets, or a proof set with no next + # challenge (briefly between periods, or proving disabled), has no series. + rule { + name = "Piri proof set is past its challenge window" + condition = "B" + for = "10m" + no_data_state = "OK" + exec_err_state = "Error" + + annotations = { + summary = "Proof set {{ $labels.proof_set }} on {{ $labels.node }} ({{ $labels.region }}) is past its challenge window" + description = "The proof set's challenge window closed more than ten minutes ago and Piri has not scheduled its next proving period, so this period's proof is missed or at risk and the proof set may be faulted on chain. Check that the Lotus Piri reads the chain from is synced, run `piri status`, and read the proof set's on-chain state with `piri client pdp proofset state` in the Piri container; the PDPv0_Prove and PDPv0_ProvPeriod task logs say why it failed." + runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" + } + + labels = { + team = "forge" + component = "appliance" + severity = "critical" + } + + data { + ref_id = "A" + datasource_uid = var.prometheus_datasource_uid + + relative_time_range { + from = 600 + to = 0 + } + + model = jsonencode(merge(local.query_defaults, { + refId = "A" + instant = true + range = false + intervalMs = 1000 + expr = <<-PROMQL + ( + max by (appliance, region, node) (piri_chain_head_epoch{${local.piri_metric_matcher}}) + + (time() - max by (appliance, region, node) (piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}})) / 30 + ) + - on (appliance, region, node) group_right + max by (appliance, region, node, proof_set) ( + piri_pdp_proofset_next_challenge_epoch{${local.piri_metric_matcher}} + + piri_pdp_proofset_challenge_window_epochs{${local.piri_metric_matcher}} + ) + PROMQL + })) + } + + data { + ref_id = "B" + query_type = "expression" + datasource_uid = "__expr__" + + relative_time_range { + from = 0 + to = 0 + } + + model = jsonencode({ + refId = "B" + type = "threshold" + expression = "A" + datasource = { type = "__expr__", uid = "__expr__" } + intervalMs = 1000 + maxDataPoints = 43200 + conditions = [{ + type = "query" + operator = { type = "and" } + query = { params = ["B"] } + reducer = { type = "last", params = [] } + evaluator = { type = "gt", params = [0] } + }] + }) + } + } + + # The backstop: no successful PDPv0_Prove on the node in one and a half + # proving periods. The value is the time since the last success in proving + # periods, so the threshold is 1.5. + # + # Per node, not per proof set: one proof set proving keeps it quiet while + # another does not, which is the rule above's job. What this adds is the case + # that rule cannot see. A proof set curio gives up on after repeated failures + # has its next challenge cleared and drops out of the proof set gauges + # altogether. The period is read over three days for that reason, so it + # outlasts one and a half of the longest period (2880 epochs, a day) after + # the gauge is gone, and still lapses on its own once proving on the node has + # ended for good. + # + # `for` is 10m. The threshold is the grace; this only confirms it. + # + # no_data_state is OK: a node that has never proved, or has no proof sets in + # three days, has no series. + rule { + name = "Piri has not proved in 1.5 proving periods" + condition = "B" + for = "10m" + no_data_state = "OK" + exec_err_state = "Error" + + annotations = { + summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) has not submitted a successful proof in 1.5 proving periods" + description = "No PDPv0_Prove task has succeeded on this node for more than one and a half proving periods, so at least one proving period has passed without a proof. Check that the Lotus Piri reads the chain from is synced, run `piri status`, and check each proof set with `piri client pdp proofset state` in the Piri container; the PDPv0_Prove task logs say why proofs are failing." + runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" + } + + labels = { + team = "forge" + component = "appliance" + severity = "warning" + } + + data { + ref_id = "A" + datasource_uid = var.prometheus_datasource_uid + + relative_time_range { + from = 600 + to = 0 + } + + model = jsonencode(merge(local.query_defaults, { + refId = "A" + instant = true + range = false + intervalMs = 1000 + expr = <<-PROMQL + ( + time() - max by (appliance, region, node) ( + piri_pdp_task_last_success_timestamp_seconds{${local.piri_metric_matcher}, task_name="PDPv0_Prove"} + ) + ) + / on (appliance, region, node) + ( + 30 * max by (appliance, region, node) ( + max_over_time(piri_pdp_proofset_proving_period_epochs{${local.piri_metric_matcher}}[3d]) + ) + ) + PROMQL + })) + } + + data { + ref_id = "B" + query_type = "expression" + datasource_uid = "__expr__" + + relative_time_range { + from = 0 + to = 0 + } + + model = jsonencode({ + refId = "B" + type = "threshold" + expression = "A" + datasource = { type = "__expr__", uid = "__expr__" } + intervalMs = 1000 + maxDataPoints = 43200 + conditions = [{ + type = "query" + operator = { type = "and" } + query = { params = ["B"] } + reducer = { type = "last", params = [] } + evaluator = { type = "gt", params = [1.5] } + }] + }) + } + } } # Not written, and why. Each needs a number or a decision that is not in the From de03aad9bf18f30b2703b40a906829542388e68d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 20:38:01 +0000 Subject: [PATCH 2/7] grafana: don't count or name the Piri rules in their comments Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ Co-Authored-By: Petra Jaros --- terraform/envs/grafana/alerts.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/envs/grafana/alerts.tf b/terraform/envs/grafana/alerts.tf index 6bf8a979..544c6adb 100644 --- a/terraform/envs/grafana/alerts.tf +++ b/terraform/envs/grafana/alerts.tf @@ -653,8 +653,8 @@ resource "grafana_rule_group" "appliance" { # returns a series only for a node that logged the line, so an empty result is # the healthy state. An instant query, so no reduce stage. # - # Superseded by "Piri's chain head is stale" and the three rules after it, - # which read the head and the proving schedule directly, once the alerting + # Superseded by the rules below that read Piri's chain head and proving + # schedule directly, once the alerting # stages run a Piri that exports those gauges. Kept until then, since a Piri # without them leaves this rule the only one watching the chain; delete it # after that rather than run both. @@ -732,7 +732,7 @@ resource "grafana_rule_group" "appliance" { } } - # The four rules below read gauges Piri exports from its PDP pipeline (piri + # The rules below read gauges Piri exports from its PDP pipeline (piri # docs/content/operator-guide/monitoring.md, "PDP Proving Health"), starting # from the PromQL given there. They are grouped by node as well as region for # the reason the disk rule gives. Filecoin epochs are thirty seconds, which is From c996e1b787de904242c4a154f3ed11ce9c4ad7d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 17:44:56 +0000 Subject: [PATCH 3/7] List the Piri metric rules in the observability alert table The table names every rule in alerts.tf, and #183 brought it up to date for #167's. Add the four this branch introduces. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ Co-Authored-By: Petra Jaros --- docs/observability.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/observability.md b/docs/observability.md index e10800f9..90eac268 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -83,6 +83,10 @@ appliance label. | Appliance has stopped reporting | Forge Regions | deploy stamp | FIL-1163 | | Appliance free disk space below 40% | Forge Regions | node exporter | FIL-1209 | | Piri has stopped receiving chain notifications | Forge Regions | Piri's logs, Loki | FIL-1383 | +| Piri's chain head is stale | Forge Regions | Piri's metrics | FIL-1383 | +| Piri is reporting but has no chain head | Forge Regions | Piri's metrics | FIL-1383 | +| Piri proof set is past its challenge window | Forge Regions | Piri's metrics | FIL-1383 | +| Piri has not proved in 1.5 proving periods | Forge Regions | Piri's metrics | FIL-1383 | | Postgres is refusing connections | Forge Regions | Postgres's logs, Loki | FIL-1163 | | Appliance 5xx rate too high | Forge Regions | Caddy | FIL-1163 | | Appliance container is not running | Forge Regions containers | cAdvisor, staging | FIL-1163 | From 615791e1ac07e6c2221164adbe2f31ad7978e7bc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 18:12:23 +0000 Subject: [PATCH 4/7] Label the Piri metric rules team_name, as #189 did the rest #189 renamed the routing label from team to team_name on every rule in main. These four were written before it and still carried team, so a route matching team_name would have missed them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ Co-Authored-By: Petra Jaros --- terraform/envs/grafana/alerts.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/envs/grafana/alerts.tf b/terraform/envs/grafana/alerts.tf index 63774620..c9fd1ec9 100644 --- a/terraform/envs/grafana/alerts.tf +++ b/terraform/envs/grafana/alerts.tf @@ -982,7 +982,7 @@ resource "grafana_rule_group" "appliance" { } labels = { - team = "forge" + team_name = "forge" component = "appliance" severity = "critical" } @@ -1073,7 +1073,7 @@ resource "grafana_rule_group" "appliance" { } labels = { - team = "forge" + team_name = "forge" component = "appliance" severity = "warning" } @@ -1175,7 +1175,7 @@ resource "grafana_rule_group" "appliance" { } labels = { - team = "forge" + team_name = "forge" component = "appliance" severity = "critical" } @@ -1267,7 +1267,7 @@ resource "grafana_rule_group" "appliance" { } labels = { - team = "forge" + team_name = "forge" component = "appliance" severity = "warning" } From ce80842657bbd510b6ad04bb1c91455d9b6b57c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:34:53 +0000 Subject: [PATCH 5/7] grafana: align the Piri proving rules with piri#152's reviewed semantics fil-forge/piri#152 went through five review rounds that changed what its gauges mean and which alerts they support, and these rules were written against its first version. Two of them claimed more than the metrics say: - "Piri proof set is past its challenge window" could not see a missed proof, because Curio schedules the next period as soon as the window closes whether or not the proof landed. It is now "Piri's proving period has not advanced", catching scheduling that has stopped (a stuck head, the task engine not running), and leaves out proof sets in a failure backoff, as #152 documents, so one rejected transaction does not page twice. `for` drops to 5m, the few minutes #152 recommends. - "Piri has not proved in 1.5 proving periods" read a gauge that counts any Prove run without a retryable error, including rejected and late ones. It is now "Piri's Prove task has not run in 1.5 proving periods", reads the period over a day as #152 does, and is critical as #152's table recommends. Two rules are added for the failures #152 added gauges for: - "Piri proving is failing": increase of the per-proof-set failure count over an hour. The level stays raised for up to a proving period after any revert, so it is not a paging condition; increase, not delta, so a reset followed by a new failure still counts. - "Piri proof set is unrecoverable": increase of the unrecoverable count over an hour. Piri does not run Curio's data set deletion, so the level never falls on its own and would page forever. The chain head stale comment now gives #152's null round and slow handler caveat, and "Piri is reporting but has no chain head" is gated on the unrecoverable count, which Piri reports on every collection, so a node with no proof sets is covered too. Every group_left/group_right carries () on the same line, the parse failure #152's fourth round found. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Petra Jaros Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ --- docs/observability.md | 6 +- terraform/envs/grafana/alerts.tf | 361 ++++++++++++++++++++++++------- 2 files changed, 291 insertions(+), 76 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index b97ab2f4..df20f69a 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -94,8 +94,10 @@ appliance label. | Piri has stopped receiving chain notifications | Forge Regions | Piri's logs, Loki | FIL-1383 | | Piri's chain head is stale | Forge Regions | Piri's metrics | FIL-1383 | | Piri is reporting but has no chain head | Forge Regions | Piri's metrics | FIL-1383 | -| Piri proof set is past its challenge window | Forge Regions | Piri's metrics | FIL-1383 | -| Piri has not proved in 1.5 proving periods | Forge Regions | Piri's metrics | FIL-1383 | +| Piri's proving period has not advanced | Forge Regions | Piri's metrics | FIL-1383 | +| Piri proving is failing | Forge Regions | Piri's metrics | FIL-1383 | +| Piri proof set is unrecoverable | Forge Regions | Piri's metrics | FIL-1383 | +| Piri's Prove task has not run in 1.5 proving periods | Forge Regions | Piri's metrics | FIL-1383 | | Postgres is refusing connections | Forge Regions | Postgres's logs, Loki | FIL-1163 | | Appliance 5xx rate too high | Forge Regions | Caddy | FIL-1163 | | Appliance container is not running | Forge Regions containers | cAdvisor, staging | FIL-1163 | diff --git a/terraform/envs/grafana/alerts.tf b/terraform/envs/grafana/alerts.tf index d1d8e624..9b56ef4d 100644 --- a/terraform/envs/grafana/alerts.tf +++ b/terraform/envs/grafana/alerts.tf @@ -1418,20 +1418,31 @@ resource "grafana_rule_group" "appliance" { } } - # The rules below read gauges Piri exports from its PDP pipeline (piri - # docs/content/operator-guide/monitoring.md, "PDP Proving Health"), starting - # from the PromQL given there. They are grouped by node as well as region for - # the reason the disk rule gives. Filecoin epochs are thirty seconds, which is - # where every 30 below comes from. + # The rules below read gauges Piri exports from its PDP pipeline, and follow + # the alerts piri's docs/content/operator-guide/monitoring.md ("PDP Proving + # Health") sets out: one failure for each rule to page on. They are grouped by + # node as well as region for the reason the disk rule gives, and by proof set + # where the failure is a proof set's. Filecoin epochs are thirty seconds, + # which is where every 30 below comes from. + # + # The gauges are what Curio has recorded in its database, not chain state, so + # none of these confirms that a proof landed. Two failures go unseen: a proof + # Curio sent that then failed on chain, and a Prove task that woke after its + # challenge window had closed. Curio records both as successful task runs and + # leaves its failure count alone. + # + # The proving failures page (critical): a missed proof is a fault on chain, + # which costs the provider, and waiting for someone to notice costs more. # The timestamp of the last tipset Piri's chain scheduler applied, against the - # wall clock. Five minutes is ten tipsets, far past what null rounds or a slow - # block produce, so this is Lotus stalled or unreachable, or Piri's - # subscription to it stuck. + # wall clock: Lotus stalled or unreachable, or Piri's subscription to it + # stuck. Piri cannot schedule or submit proofs without a moving head. # - # `for` is 5m, the shortest pending period a group evaluated every five - # minutes can give: one more evaluation to confirm the first. The threshold - # already holds five minutes of grace. + # Five minutes is ten tipsets, but a run of null rounds can leave that long + # between tipsets, and a slow chain scheduler handler that runs before the + # head is recorded can delay it. `for` is 5m, the shortest pending period a + # group evaluated every five minutes can give, so the head must stay stale + # across two evaluations, about ten minutes in all, before it fires. # # no_data_state is OK: a Piri that has not seen a head since it started, or # one that exports no chain gauges at all, has no series here. The first is @@ -1514,14 +1525,16 @@ resource "grafana_rule_group" "appliance" { # target_info is the evidence Piri is up: it carries the same job and # appliance, region and node labels as every other Piri series, so the same # matcher applies to it. On its own it would also match every Piri too old to - # export the head, which is every Piri until these metrics are deployed. So - # the node must also export one of the gauges Piri reads from its database, - # which do not depend on the chain: they prove the running Piri has the - # metrics, and so that the missing head means something. A node with no proof - # sets and no successful proof yet is left out, and has nothing to lose. + # export the head. So the node must also export + # piri_pdp_proofsets_unrecoverable, which Piri reads from its database on + # every collection, proof sets or none, and which does not depend on the + # chain: it shows the running Piri has these metrics, and so that the missing + # head means something. # # Five-minute windows are ten of Piri's thirty-second pushes. `for` is 10m, so - # a restart that takes a moment to reach Lotus does not fire. + # a restart that takes a moment to reach Lotus does not fire. Warning, not + # critical: until the first head arrives Piri has not missed anything yet, and + # the proving rules below page if it goes on long enough to cost a proof. # # no_data_state is OK, and not NoData, even though "no head" is what the rule # is about: the `unless` is what finds the missing head, so the result is @@ -1566,17 +1579,11 @@ resource "grafana_rule_group" "appliance" { max by (appliance, region, node) ( present_over_time(target_info{${local.piri_metric_matcher}}[5m]) ) - and - ( - max by (appliance, region, node) ( - present_over_time(piri_pdp_task_last_success_timestamp_seconds{${local.piri_metric_matcher}}[5m]) - ) - or - max by (appliance, region, node) ( - present_over_time(piri_pdp_proofset_next_challenge_epoch{${local.piri_metric_matcher}}[5m]) - ) + and on (appliance, region, node) + max by (appliance, region, node) ( + present_over_time(piri_pdp_proofsets_unrecoverable{${local.piri_metric_matcher}}[5m]) ) - unless + unless on (appliance, region, node) max by (appliance, region, node) ( present_over_time(piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}}[5m]) ) @@ -1612,35 +1619,42 @@ resource "grafana_rule_group" "appliance" { } } - # A proof set whose challenge window has closed without its next proving - # period being scheduled: epochs past the window's close, per proof set. The - # current epoch is the last head plus the wall-clock time since it, so this - # still fires when the head itself is stale. - # - # Some overshoot is normal. Curio's NextProvingPeriod watcher - # (tasks/pdpv0/task_next_pp.go) only queues the task once a head reaches - # prove_at_epoch + challenge_window, and the task moves prove_at_epoch on - # when it sends nextProvingPeriod, without waiting for it to land. Add - # harmonytask's three-second poll, a handful of contract reads and Piri's - # thirty-second push, and a healthy proof set is a few epochs past its window - # for a minute or two each period, which an evaluation can catch. `for` is - # 10m: three evaluations in a row, twenty epochs, which that never reaches, - # while a scheduling task that is failing, or held back by the hundred-epoch - # backoff curio applies after a failed attempt, stays past the window for - # longer than that. - # - # no_data_state is OK: a node with no proof sets, or a proof set with no next - # challenge (briefly between periods, or proving disabled), has no series. + # A proof set whose challenge window, as Curio recorded it, has closed without + # Curio scheduling the next proving period, for a reason other than a failure + # backoff. The value is epochs past the window's close. The current epoch is + # the last head plus the wall-clock time since it, so this still fires when + # the head itself is stale. + # + # It catches proving-period scheduling that has stopped: a stuck head, or the + # task engine not running. It does not catch a missed proof. Curio schedules + # the next period as soon as the window closes (tasks/pdpv0/task_next_pp.go), + # whether or not this period's proof landed, and moves the challenge epoch on + # when it sends that transaction rather than when it lands. + # + # A proof set in a failure backoff is left out by the `unless`: Curio holds + # its scheduling back on purpose until the backoff deadline passes the head, + # and "Piri proving is failing" has already paged for the failure that set + # it. The deadline stays recorded after it passes, so the comparison is + # against the same extrapolated head, and the set comes back into this rule + # once the head is past it. + # + # A healthy proof set is past its window for a minute or two each period, + # until the scheduling task runs and Piri's next push reports it. `for` is 5m, + # the shortest a five-minute group gives: two evaluations in a row cannot both + # land in that. + # + # no_data_state is OK: a node with no proof sets, or a proof set with proving + # disabled or marked unrecoverable, has no schedule series. rule { - name = "Piri proof set is past its challenge window" + name = "Piri's proving period has not advanced" condition = "B" - for = "10m" + for = "5m" no_data_state = "OK" exec_err_state = "Error" annotations = { - summary = "Proof set {{ $labels.proof_set }} on {{ $labels.node }} ({{ $labels.region }}) is past its challenge window" - description = "The proof set's challenge window closed more than ten minutes ago and Piri has not scheduled its next proving period, so this period's proof is missed or at risk and the proof set may be faulted on chain. Check that the Lotus Piri reads the chain from is synced, run `piri status`, and read the proof set's on-chain state with `piri client pdp proofset state` in the Piri container; the PDPv0_Prove and PDPv0_ProvPeriod task logs say why it failed." + summary = "Proof set {{ $labels.proof_set }} on {{ $labels.node }} ({{ $labels.region }}) has not had its next proving period scheduled" + description = "The proof set's challenge window has closed and Piri has not scheduled its next proving period, and it is not waiting out a failure backoff, so proving for it has stopped. Usually the chain head has stopped moving or Piri's PDP task engine is not running. Check that the Lotus Piri reads the chain from is synced, run `piri status`, and read the proof set's state with `piri client pdp proofset state` in the Piri container; the PDPv0_ProvPeriod task logs say whether scheduling ran." runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" } @@ -1667,13 +1681,118 @@ resource "grafana_rule_group" "appliance" { intervalMs = 1000 expr = <<-PROMQL ( - max by (appliance, region, node) (piri_chain_head_epoch{${local.piri_metric_matcher}}) - + (time() - max by (appliance, region, node) (piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}})) / 30 + ( + max by (appliance, region, node) (piri_chain_head_epoch{${local.piri_metric_matcher}}) + + (time() - max by (appliance, region, node) (piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}})) / 30 + ) + - on (appliance, region, node) group_right() + max by (appliance, region, node, proof_set) ( + piri_pdp_proofset_next_challenge_epoch{${local.piri_metric_matcher}} + + piri_pdp_proofset_challenge_window_epochs{${local.piri_metric_matcher}} + ) ) - - on (appliance, region, node) group_right + unless on (appliance, region, node, proof_set) + ( + max by (appliance, region, node, proof_set) ( + piri_pdp_proofset_next_prove_attempt_epoch{${local.piri_metric_matcher}} + ) + > on (appliance, region, node) group_left() + ( + max by (appliance, region, node) (piri_chain_head_epoch{${local.piri_metric_matcher}}) + + (time() - max by (appliance, region, node) (piri_chain_head_timestamp_seconds{${local.piri_metric_matcher}})) / 30 + ) + ) + PROMQL + })) + } + + data { + ref_id = "B" + query_type = "expression" + datasource_uid = "__expr__" + + relative_time_range { + from = 0 + to = 0 + } + + model = jsonencode({ + refId = "B" + type = "threshold" + expression = "A" + datasource = { type = "__expr__", uid = "__expr__" } + intervalMs = 1000 + maxDataPoints = 43200 + conditions = [{ + type = "query" + operator = { type = "and" } + query = { params = ["B"] } + reducer = { type = "last", params = [] } + evaluator = { type = "gt", params = [0] } + }] + }) + } + } + + # Curio has had a proving transaction for a proof set rejected within the + # last hour. A rejected prove is not retried within its period, so that + # period's proof was missed; a rejected scheduling transaction is retried + # after a backoff, and Curio gives up on the set after repeated failures (the + # next rule). + # + # The gauge is the count of rejections since the set's last successful prove + # send, and the level is not a paging condition. Only a successful prove send + # resets it, so it stays above zero for up to a proving period after any + # rejection, even when the retry that follows succeeds, and indefinitely if + # proving is then disabled for the set. So the rule fires on a rise instead. + # `increase`, not `delta`, because the count behaves as a counter whose only + # drop is the reset to zero: a reset followed by a new failure within the + # hour is still counted as a rise, where `delta` would net it to zero. + # + # It fires for about an hour per new failure, so during a long backoff it + # pages after each failure rather than throughout. `for` is 0m: the hour + # window is the debounce, and a single rejection is worth a page. + # + # no_data_state is OK: every proof set reports the count, zero included, so + # an empty result means no proof sets or a Piri older than the metric, and a + # healthy set evaluates to zero. + rule { + name = "Piri proving is failing" + condition = "B" + for = "0m" + no_data_state = "OK" + exec_err_state = "Error" + + annotations = { + summary = "Proof set {{ $labels.proof_set }} on {{ $labels.node }} ({{ $labels.region }}) had a proving transaction rejected" + description = "Curio, inside Piri, has had a proving transaction for this proof set rejected by the contract in the last hour. A rejected prove means this proving period's proof was missed; a rejected scheduling transaction is retried after a backoff, and repeated failures make Curio give up on the proof set. Check the Piri container's logs for the PDPv0_Prove or PDPv0_ProvPeriod error, check the proof set with `piri client pdp proofset state`, and check the wallet has FIL for gas." + runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" + } + + labels = { + team_name = "forge" + component = "appliance" + stage = local.appliance_stage_label + severity = "critical" + } + + data { + ref_id = "A" + datasource_uid = var.prometheus_datasource_uid + + relative_time_range { + from = 3600 + to = 0 + } + + model = jsonencode(merge(local.query_defaults, { + refId = "A" + instant = true + range = false + intervalMs = 1000 + expr = <<-PROMQL max by (appliance, region, node, proof_set) ( - piri_pdp_proofset_next_challenge_epoch{${local.piri_metric_matcher}} - + piri_pdp_proofset_challenge_window_epochs{${local.piri_metric_matcher}} + increase(piri_pdp_proofset_consecutive_prove_failures{${local.piri_metric_matcher}}[1h]) ) PROMQL })) @@ -1707,33 +1826,127 @@ resource "grafana_rule_group" "appliance" { } } - # The backstop: no successful PDPv0_Prove on the node in one and a half - # proving periods. The value is the time since the last success in proving - # periods, so the threshold is 1.5. + # Curio has given up proving a proof set after an unrecoverable failure. + # Piri stops reporting that set's own series and counts it in + # piri_pdp_proofsets_unrecoverable instead, per node. # - # Per node, not per proof set: one proof set proving keeps it quiet while - # another does not, which is the rule above's job. What this adds is the case - # that rule cannot see. A proof set curio gives up on after repeated failures - # has its next challenge cleared and drops out of the proof set gauges - # altogether. The period is read over three days for that reason, so it - # outlasts one and a half of the longest period (2880 epochs, a day) after - # the gauge is gone, and still lapses on its own once proving on the node has - # ended for good. + # The count does not go down on its own: Piri does not run Curio's data set + # deletion, so the row stays until someone removes it by hand. A rule on the + # level would page forever after the first one, so this fires on a rise, for + # about an hour per newly unrecoverable set. `for` is 0m for the reason the + # rule above gives. Two things follow from reading a rise: # - # `for` is 10m. The threshold is the grace; this only confirms it. + # - A set that was already unrecoverable when a Piri with this metric first + # reported is never paged for; graph the gauge to see the current state. + # - `increase` reads any drop as a counter reset, so removing one row by + # hand while others remain fires once, for an hour. # - # no_data_state is OK: a node that has never proved, or has no proof sets in - # three days, has no series. + # no_data_state is OK: Piri reports the count, zero included, on every + # collection, so an empty result is a Piri older than the metric, and a + # healthy node evaluates to zero. rule { - name = "Piri has not proved in 1.5 proving periods" + name = "Piri proof set is unrecoverable" + condition = "B" + for = "0m" + no_data_state = "OK" + exec_err_state = "Error" + + annotations = { + summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) has stopped proving a proof set after an unrecoverable failure" + description = "Curio, inside Piri, has marked a proof set unrecoverable in the last hour and will not prove it again, so its data is no longer being proved on chain. Check the Piri container's logs for why Curio gave up on it, and check each proof set with `piri client pdp proofset state`. Piri does not delete the proof set, so it stays counted until it is dealt with by hand." + runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" + } + + labels = { + team_name = "forge" + component = "appliance" + stage = local.appliance_stage_label + severity = "critical" + } + + data { + ref_id = "A" + datasource_uid = var.prometheus_datasource_uid + + relative_time_range { + from = 3600 + to = 0 + } + + model = jsonencode(merge(local.query_defaults, { + refId = "A" + instant = true + range = false + intervalMs = 1000 + expr = <<-PROMQL + max by (appliance, region, node) ( + increase(piri_pdp_proofsets_unrecoverable{${local.piri_metric_matcher}}[1h]) + ) + PROMQL + })) + } + + data { + ref_id = "B" + query_type = "expression" + datasource_uid = "__expr__" + + relative_time_range { + from = 0 + to = 0 + } + + model = jsonencode({ + refId = "B" + type = "threshold" + expression = "A" + datasource = { type = "__expr__", uid = "__expr__" } + intervalMs = 1000 + maxDataPoints = 43200 + conditions = [{ + type = "query" + operator = { type = "and" } + query = { params = ["B"] } + reducer = { type = "last", params = [] } + evaluator = { type = "gt", params = [0] } + }] + }) + } + } + + # The backstop: no PDPv0_Prove run has finished on the node in one and a half + # proving periods. The value is the time since the last one in proving + # periods, so the threshold is 1.5. + # + # The gauge is the last run that finished without a retryable error, not the + # last proof that landed. It includes runs that woke too late, found proving + # disabled or had their proof rejected, so it says the task engine is running + # Prove, not that proofs are landing; the rules above cover rejections. What + # it adds is a Prove task that keeps failing with a retryable error, or never + # runs, while scheduling carries on, which nothing else here sees. + # + # Per node, not per proof set: with several proof sets, proving one keeps it + # quiet. The period is the longest seen over a day, which bridges a missed + # collection. A node whose last proof set is disabled or unrecoverable loses + # the period series within the day, before the threshold, so this lapses + # rather than repeat the unrecoverable page. + # + # Critical, as the piri docs recommend: a Prove task that does not run misses + # every proof on the node. `for` is 10m. The threshold is the grace; this only + # confirms it. + # + # no_data_state is OK: a node that has never finished a Prove run, or has had + # no scheduled proof set in a day, has no series. + rule { + name = "Piri's Prove task has not run in 1.5 proving periods" condition = "B" for = "10m" no_data_state = "OK" exec_err_state = "Error" annotations = { - summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) has not submitted a successful proof in 1.5 proving periods" - description = "No PDPv0_Prove task has succeeded on this node for more than one and a half proving periods, so at least one proving period has passed without a proof. Check that the Lotus Piri reads the chain from is synced, run `piri status`, and check each proof set with `piri client pdp proofset state` in the Piri container; the PDPv0_Prove task logs say why proofs are failing." + summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) has not finished a Prove task run in 1.5 proving periods" + description = "No PDPv0_Prove task run has finished on this node for more than one and a half proving periods, so Piri is not sending proofs and its proof sets are missing their proving periods. Check that the Lotus Piri reads the chain from is synced, run `piri status`, and check each proof set with `piri client pdp proofset state` in the Piri container; the PDPv0_Prove task logs say why it is not finishing." runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" } @@ -1741,7 +1954,7 @@ resource "grafana_rule_group" "appliance" { team_name = "forge" component = "appliance" stage = local.appliance_stage_label - severity = "warning" + severity = "critical" } data { @@ -1767,7 +1980,7 @@ resource "grafana_rule_group" "appliance" { / on (appliance, region, node) ( 30 * max by (appliance, region, node) ( - max_over_time(piri_pdp_proofset_proving_period_epochs{${local.piri_metric_matcher}}[3d]) + max_over_time(piri_pdp_proofset_proving_period_epochs{${local.piri_metric_matcher}}[1d]) ) ) PROMQL From a89fbb4f12a14db900119b3d9ea137ddf86c8cca Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:42:11 +0000 Subject: [PATCH 6/7] grafana: page on a Piri with no head, and retire the Loki stopgap Review round 1 on this PR: - "Piri is reporting but has no chain head" was a warning on the grounds that the proving rules would page if it lasted. None of them can fire without a head series or a proving send, so the only later page came after a proof was missed. It is critical now, like "chain head is stale" for the same cause. - The Loki rule on Piri's resubscribe line was to be deleted once these gauges replaced it, but this PR kept it, so merging at the gate would page twice for a stuck Lotus. It goes, with its matcher and docs row; the PR merges only once every alerting stage runs a Piri with the gauges. - Comments: the shortest non-zero pending period, the assumption behind the Prove-task rule lapsing before its threshold, and a note that one cause can raise several of these over time. The docs table is realigned. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Petra Jaros Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ --- docs/observability.md | 37 ++++----- terraform/envs/grafana/alerts.tf | 137 +++++-------------------------- 2 files changed, 38 insertions(+), 136 deletions(-) diff --git a/docs/observability.md b/docs/observability.md index df20f69a..3e6e2c83 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -84,25 +84,24 @@ The rules watch staging and prod, the stages listed in `alert_stages` in `fc-` names, and an appliance rule matches the `-` appliance label. -| Rule | Group | Source | Ticket | -| ----------------------------------- | ------------------------- | ----------------- | -------- | -| Service has no healthy hosts | Forge Central | ALB, CloudWatch | FIL-1151 | -| Service 5xx errors | Forge Central | ALB, CloudWatch | FIL-1207 | -| Provision Lambda errors | Forge Central | Lambda, CloudWatch| FIL-1151 | -| Appliance has stopped reporting | Forge Regions | deploy stamp | FIL-1163 | -| Appliance free disk space below 40% | Forge Regions | node exporter | FIL-1209 | -| Piri has stopped receiving chain notifications | Forge Regions | Piri's logs, Loki | FIL-1383 | -| Piri's chain head is stale | Forge Regions | Piri's metrics | FIL-1383 | -| Piri is reporting but has no chain head | Forge Regions | Piri's metrics | FIL-1383 | -| Piri's proving period has not advanced | Forge Regions | Piri's metrics | FIL-1383 | -| Piri proving is failing | Forge Regions | Piri's metrics | FIL-1383 | -| Piri proof set is unrecoverable | Forge Regions | Piri's metrics | FIL-1383 | -| Piri's Prove task has not run in 1.5 proving periods | Forge Regions | Piri's metrics | FIL-1383 | -| Postgres is refusing connections | Forge Regions | Postgres's logs, Loki | FIL-1163 | -| Appliance 5xx rate too high | Forge Regions | Caddy | FIL-1163 | -| Appliance container is not running | Forge Regions containers | cAdvisor, staging | FIL-1163 | -| OpenBao cannot unseal | Forge Regions outages | OpenBao's logs, Loki | FIL-1164 | -| Appliance site is failing most requests | Forge Regions outages | Caddy | FIL-1164 | +| Rule | Group | Source | Ticket | +| ---------------------------------------------------- | ------------------------ | --------------------- | -------- | +| Service has no healthy hosts | Forge Central | ALB, CloudWatch | FIL-1151 | +| Service 5xx errors | Forge Central | ALB, CloudWatch | FIL-1207 | +| Provision Lambda errors | Forge Central | Lambda, CloudWatch | FIL-1151 | +| Appliance has stopped reporting | Forge Regions | deploy stamp | FIL-1163 | +| Appliance free disk space below 40% | Forge Regions | node exporter | FIL-1209 | +| Piri's chain head is stale | Forge Regions | Piri's metrics | FIL-1383 | +| Piri is reporting but has no chain head | Forge Regions | Piri's metrics | FIL-1383 | +| Piri's proving period has not advanced | Forge Regions | Piri's metrics | FIL-1383 | +| Piri proving is failing | Forge Regions | Piri's metrics | FIL-1383 | +| Piri proof set is unrecoverable | Forge Regions | Piri's metrics | FIL-1383 | +| Piri's Prove task has not run in 1.5 proving periods | Forge Regions | Piri's metrics | FIL-1383 | +| Postgres is refusing connections | Forge Regions | Postgres's logs, Loki | FIL-1163 | +| Appliance 5xx rate too high | Forge Regions | Caddy | FIL-1163 | +| Appliance container is not running | Forge Regions containers | cAdvisor, staging | FIL-1163 | +| OpenBao cannot unseal | Forge Regions outages | OpenBao's logs, Loki | FIL-1164 | +| Appliance site is failing most requests | Forge Regions outages | Caddy | FIL-1164 | ## Logs diff --git a/terraform/envs/grafana/alerts.tf b/terraform/envs/grafana/alerts.tf index 9b56ef4d..010bfedc 100644 --- a/terraform/envs/grafana/alerts.tf +++ b/terraform/envs/grafana/alerts.tf @@ -94,12 +94,6 @@ locals { # a superset, and one that needs no per-node edit. filesystem_matcher = "${local.appliance_matcher}, fstype!~\"tmpfs|vfat|squashfs|overlay\", mountpoint!~\"/boot.*\"" - # Piri's container log stream. Alloy names it appliance---piri - # from the Compose service and stamps appliance, region and node on it - # (infra-nodes nodes/dev/platform/config/alloy/config.alloy), so the same - # appliance matcher narrows it to the alerting stages. - piri_log_matcher = "appliance=~\"(${local.stages})-.*\", service_name=~\"appliance-.*-piri\"" - # Ingot's OTLP metrics. Alloy names them appliance---ingot # from Ingot's service.name and stamps appliance, region and node on them, # the same as Piri's (infra-nodes docs/observability.md). @@ -763,109 +757,6 @@ resource "grafana_rule_group" "appliance" { } } - # A stopgap for a Lotus whose head has stopped moving. Piri's chain scheduler - # (curio lib/chainsched) ticks every five minutes and, when no head change has - # arrived since the last tick, logs - # - # no notifications received in 5m0s, resubscribing to ChainNotify - # - # and resubscribes. A stuck Lotus answers the new subscription with its current - # tipset and nothing after, so the line recurs roughly every ten minutes for as - # long as the head is stuck. Nothing else Piri exports says so: it stays up, - # answers its health check and keeps its metrics flowing, and only its proofs - # stop. A Lotus that is down outright fails the subscription instead, and logs - # something else. - # - # Any line in fifteen minutes is a match, since with a ten-minute cadence that - # window always holds one while the head is stuck. `for` is 15m so a single - # line does not fire on its own: fifteen minutes of pending needs a second line - # inside the window, which is about twenty minutes of a head that has not - # moved. Filecoin produces a tipset every thirty seconds, so that is well past - # anything a healthy chain does. - # - # no_data_state is OK for the reason the 5xx rule gives: count_over_time - # returns a series only for a node that logged the line, so an empty result is - # the healthy state. An instant query, so no reduce stage. - # - # Superseded by the rules below that read Piri's chain head and proving - # schedule directly, once the alerting stages run a Piri that exports those - # gauges. Kept until then, since a Piri without them leaves this rule the only - # one watching the chain; delete it after that rather than run both. - rule { - name = "Piri has stopped receiving chain notifications" - condition = "B" - for = "15m" - no_data_state = "OK" - exec_err_state = "Error" - - annotations = { - summary = "Piri on {{ $labels.node }} ({{ $labels.region }}) has stopped receiving chain head changes" - description = "Piri's chain scheduler has logged \"no notifications received ... resubscribing to ChainNotify\" repeatedly for fifteen minutes: the Lotus it reads the chain from is up but its head is not moving, so Piri cannot schedule or submit proofs. Check the sync status of that Lotus (`lotus sync wait` or `lotus chain head` on the host that owns it) and run `piri status` in the Piri container." - runbook_url = "https://github.com/fil-forge/infra-nodes/blob/main/docs/RUNBOOK.md#when-something-is-wrong" - } - - labels = { - team_name = "forge" - component = "appliance" - stage = local.appliance_stage_label - severity = "critical" - } - - data { - ref_id = "A" - datasource_uid = var.loki_datasource_uid - - relative_time_range { - from = 900 - to = 0 - } - - model = jsonencode({ - refId = "A" - editorMode = "code" - queryType = "instant" - intervalMs = 1000 - maxDataPoints = 43200 - expr = <<-LOGQL - sum by (appliance, region, node) ( - count_over_time( - {${local.piri_log_matcher}} - |~ "no notifications received in .* resubscribing to ChainNotify" - [15m] - ) - ) - LOGQL - }) - } - - data { - ref_id = "B" - query_type = "expression" - datasource_uid = "__expr__" - - relative_time_range { - from = 0 - to = 0 - } - - model = jsonencode({ - refId = "B" - type = "threshold" - expression = "A" - datasource = { type = "__expr__", uid = "__expr__" } - intervalMs = 1000 - maxDataPoints = 43200 - conditions = [{ - type = "query" - operator = { type = "and" } - query = { params = ["B"] } - reducer = { type = "last", params = [] } - evaluator = { type = "gt", params = [0] } - }] - }) - } - } - # Postgres refusing connections. Seen for real in the 2026-09-24 load test: # staging Postgres refused 3,629 connections in 27 minutes, every multipart # object's CompleteMultipartUpload failed at least once, and nothing alerted. @@ -1433,6 +1324,15 @@ resource "grafana_rule_group" "appliance" { # # The proving failures page (critical): a missed proof is a fault on chain, # which costs the provider, and waiting for someone to notice costs more. + # + # One cause can raise more than one of these over time: a head that stays + # stuck is first "chain head is stale", then "proving period has not + # advanced", then "Prove task has not run". Each names the failure as it has + # become by then. + # + # They replace the earlier Loki rule on Piri's "no notifications received" + # resubscribe line, which watched for a stuck head before Piri exported one. + # Running both would page twice for the same stuck Lotus. # The timestamp of the last tipset Piri's chain scheduler applied, against the # wall clock: Lotus stalled or unreachable, or Piri's subscription to it @@ -1440,8 +1340,8 @@ resource "grafana_rule_group" "appliance" { # # Five minutes is ten tipsets, but a run of null rounds can leave that long # between tipsets, and a slow chain scheduler handler that runs before the - # head is recorded can delay it. `for` is 5m, the shortest pending period a - # group evaluated every five minutes can give, so the head must stay stale + # head is recorded can delay it. `for` is 5m, the shortest non-zero pending + # period a group evaluated every five minutes can give, so the head must stay stale # across two evaluations, about ten minutes in all, before it fires. # # no_data_state is OK: a Piri that has not seen a head since it started, or @@ -1532,9 +1432,11 @@ resource "grafana_rule_group" "appliance" { # head means something. # # Five-minute windows are ten of Piri's thirty-second pushes. `for` is 10m, so - # a restart that takes a moment to reach Lotus does not fire. Warning, not - # critical: until the first head arrives Piri has not missed anything yet, and - # the proving rules below page if it goes on long enough to cost a proof. + # a restart that takes a moment to reach Lotus does not fire. Critical, as + # "chain head is stale" is for the same cause: none of the proving rules below + # can fire for a node with no head, since they compare against it or wait on a + # proving send that never happens, so the only later page would be "Prove task + # has not run", which comes after a proof has been missed. # # no_data_state is OK, and not NoData, even though "no head" is what the rule # is about: the `unless` is what finds the missing head, so the result is @@ -1558,7 +1460,7 @@ resource "grafana_rule_group" "appliance" { team_name = "forge" component = "appliance" stage = local.appliance_stage_label - severity = "warning" + severity = "critical" } data { @@ -1928,8 +1830,9 @@ resource "grafana_rule_group" "appliance" { # Per node, not per proof set: with several proof sets, proving one keeps it # quiet. The period is the longest seen over a day, which bridges a missed # collection. A node whose last proof set is disabled or unrecoverable loses - # the period series within the day, before the threshold, so this lapses - # rather than repeat the unrecoverable page. + # the period series within the day, which is before the threshold while 1.5 + # proving periods is longer than a day, as with mainnet's 2880 epochs; then + # this lapses rather than repeat the unrecoverable page. # # Critical, as the piri docs recommend: a Prove task that does not run misses # every proof on the node. `for` is 10m. The threshold is the grace; this only From 2e4e8e8a578414d826e73b969e7a77fd642ed4e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:46:41 +0000 Subject: [PATCH 7/7] grafana: tidy the Piri alert comments Give both five-minute pending periods the same wording, rewrap an overlong line, move the container log stream naming onto the Postgres matcher now that the Piri log matcher is gone, and say that every Piri chain and proving rule pages. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Petra Jaros Claude-Session: https://claude.ai/code/session_01JCF8gyb8jNRmHmWqQmVhRJ --- terraform/envs/grafana/alerts.tf | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/terraform/envs/grafana/alerts.tf b/terraform/envs/grafana/alerts.tf index 010bfedc..59c83674 100644 --- a/terraform/envs/grafana/alerts.tf +++ b/terraform/envs/grafana/alerts.tf @@ -108,7 +108,10 @@ locals { # reused: its service_name is the host exporter's. piri_metric_matcher = "appliance=~\"(${local.stages})-.*\", job=\"forge/piri\"" - # Postgres's container log stream, same scheme. Anchored, so this is the + # Postgres's container log stream. Alloy names each container's stream + # appliance--- from the Compose service and stamps + # appliance, region and node on it (infra-nodes + # nodes/dev/platform/config/alloy/config.alloy). Anchored, so this is the # postgres service and not postgres-init. postgres_log_matcher = "appliance=~\"(${local.stages})-.*\", service_name=~\"appliance-.*-postgres\"" @@ -1322,8 +1325,9 @@ resource "grafana_rule_group" "appliance" { # challenge window had closed. Curio records both as successful task runs and # leaves its failure count alone. # - # The proving failures page (critical): a missed proof is a fault on chain, - # which costs the provider, and waiting for someone to notice costs more. + # All of these page (critical). Each one means proofs are failing or about to: + # a missed proof is a fault on chain, which costs the provider, and waiting + # for someone to notice costs more. # # One cause can raise more than one of these over time: a head that stays # stuck is first "chain head is stale", then "proving period has not @@ -1341,8 +1345,8 @@ resource "grafana_rule_group" "appliance" { # Five minutes is ten tipsets, but a run of null rounds can leave that long # between tipsets, and a slow chain scheduler handler that runs before the # head is recorded can delay it. `for` is 5m, the shortest non-zero pending - # period a group evaluated every five minutes can give, so the head must stay stale - # across two evaluations, about ten minutes in all, before it fires. + # period a group evaluated every five minutes can give, so the head must stay + # stale across two evaluations, about ten minutes in all, before it fires. # # no_data_state is OK: a Piri that has not seen a head since it started, or # one that exports no chain gauges at all, has no series here. The first is @@ -1542,8 +1546,8 @@ resource "grafana_rule_group" "appliance" { # # A healthy proof set is past its window for a minute or two each period, # until the scheduling task runs and Piri's next push reports it. `for` is 5m, - # the shortest a five-minute group gives: two evaluations in a row cannot both - # land in that. + # the shortest non-zero pending period a five-minute group gives: two + # evaluations in a row cannot both land in that. # # no_data_state is OK: a node with no proof sets, or a proof set with proving # disabled or marked unrecoverable, has no schedule series.