|
| 1 | +package metrics |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "math/big" |
| 7 | + "time" |
| 8 | + |
| 9 | + "github.com/ethereum/go-ethereum/common" |
| 10 | + "github.com/spf13/cobra" |
| 11 | + "go.opentelemetry.io/otel/attribute" |
| 12 | + "go.opentelemetry.io/otel/metric" |
| 13 | + |
| 14 | + "github.com/storacha/forgectl/cli/config" |
| 15 | + "github.com/storacha/forgectl/pkg/services/inspector" |
| 16 | + "github.com/storacha/forgectl/pkg/telemetry" |
| 17 | + "github.com/storacha/forgectl/pkg/telemetry/metrics" |
| 18 | +) |
| 19 | + |
| 20 | +var ( |
| 21 | + faultsPayerFlag string |
| 22 | + faultsOtlpEndpoint string |
| 23 | + faultsOtlpInsecure bool |
| 24 | +) |
| 25 | + |
| 26 | +var faultsCmd = &cobra.Command{ |
| 27 | + Use: "faults", |
| 28 | + Short: "Export proof fault metrics via OTLP", |
| 29 | + Long: `Export proof fault metrics via OTLP to a collector. |
| 30 | +
|
| 31 | +Metrics exported: |
| 32 | + - forgectl_rail_missed_periods: Count of proving periods where proof was missed (per rail/dataset) |
| 33 | +
|
| 34 | +This command collects metrics once and exits, making it suitable for cron jobs.`, |
| 35 | + RunE: runFaultsMetrics, |
| 36 | +} |
| 37 | + |
| 38 | +func init() { |
| 39 | + faultsCmd.Flags().StringVar(&faultsPayerFlag, "payer", "", "Payer address to monitor (required)") |
| 40 | + cobra.CheckErr(faultsCmd.MarkFlagRequired("payer")) |
| 41 | + |
| 42 | + faultsCmd.Flags().StringVar(&faultsOtlpEndpoint, "otlp-endpoint", "", "OTLP HTTP endpoint (required, e.g., localhost:4318)") |
| 43 | + cobra.CheckErr(faultsCmd.MarkFlagRequired("otlp-endpoint")) |
| 44 | + |
| 45 | + faultsCmd.Flags().BoolVar(&faultsOtlpInsecure, "otlp-insecure", false, "Use insecure connection for OTLP") |
| 46 | +} |
| 47 | + |
| 48 | +type faultMetrics struct { |
| 49 | + railMissedPeriods *telemetry.Float64Gauge |
| 50 | +} |
| 51 | + |
| 52 | +func runFaultsMetrics(cmd *cobra.Command, args []string) error { |
| 53 | + if !common.IsHexAddress(faultsPayerFlag) { |
| 54 | + return fmt.Errorf("invalid payer address: %s", faultsPayerFlag) |
| 55 | + } |
| 56 | + payerAddr := common.HexToAddress(faultsPayerFlag) |
| 57 | + |
| 58 | + ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) |
| 59 | + defer cancel() |
| 60 | + |
| 61 | + cfg, err := config.LoadReadOnly() |
| 62 | + if err != nil { |
| 63 | + return fmt.Errorf("failed to load config: %w", err) |
| 64 | + } |
| 65 | + |
| 66 | + // Initialize telemetry with a short publish interval for one-shot collection |
| 67 | + tel, err := telemetry.New( |
| 68 | + ctx, |
| 69 | + "production", |
| 70 | + "forgectl", |
| 71 | + "0.0.1", |
| 72 | + payerAddr.Hex(), |
| 73 | + metrics.Config{ |
| 74 | + Collectors: []metrics.CollectorConfig{ |
| 75 | + { |
| 76 | + Endpoint: faultsOtlpEndpoint, |
| 77 | + Insecure: faultsOtlpInsecure, |
| 78 | + PublishInterval: 5 * time.Second, |
| 79 | + }, |
| 80 | + }, |
| 81 | + }, |
| 82 | + ) |
| 83 | + if err != nil { |
| 84 | + return fmt.Errorf("failed to initialize telemetry: %w", err) |
| 85 | + } |
| 86 | + defer func() { |
| 87 | + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 88 | + defer shutdownCancel() |
| 89 | + if err := tel.Shutdown(shutdownCtx); err != nil { |
| 90 | + log.Errorw("failed to shutdown telemetry", "error", err) |
| 91 | + } |
| 92 | + }() |
| 93 | + |
| 94 | + // Create metrics |
| 95 | + meter := tel.Metrics.Meter("forgectl") |
| 96 | + fm, err := newFaultMetrics(meter) |
| 97 | + if err != nil { |
| 98 | + return fmt.Errorf("failed to create metrics: %w", err) |
| 99 | + } |
| 100 | + |
| 101 | + // Initialize inspector |
| 102 | + inspctr, err := inspector.New(inspector.Config{ |
| 103 | + ClientEndpoint: cfg.RPCUrl, |
| 104 | + PaymentsContractAddress: cfg.PaymentsAddr(), |
| 105 | + ServiceContractAddress: cfg.ServiceAddr(), |
| 106 | + ProviderRegistryAddress: cfg.ServiceRegistryAddr(), |
| 107 | + SessionKeyRegistryAddress: cfg.SessionKeyRegistryAddr(), |
| 108 | + TokenAddress: cfg.TokenAddr(), |
| 109 | + PDPVerifierAddress: cfg.VerifierAddr(), |
| 110 | + }) |
| 111 | + if err != nil { |
| 112 | + return fmt.Errorf("failed to create inspector: %w", err) |
| 113 | + } |
| 114 | + |
| 115 | + tokenAddr := cfg.TokenAddr() |
| 116 | + |
| 117 | + log.Infow("collecting fault metrics", |
| 118 | + "payer", payerAddr.Hex(), |
| 119 | + "endpoint", faultsOtlpEndpoint, |
| 120 | + ) |
| 121 | + |
| 122 | + // Collect and record metrics |
| 123 | + if err := collectFaultMetrics(ctx, inspctr, fm, tokenAddr, payerAddr); err != nil { |
| 124 | + return fmt.Errorf("failed to collect fault metrics: %w", err) |
| 125 | + } |
| 126 | + |
| 127 | + log.Info("fault metrics collected successfully") |
| 128 | + return nil |
| 129 | +} |
| 130 | + |
| 131 | +func newFaultMetrics(meter metric.Meter) (*faultMetrics, error) { |
| 132 | + railMissedPeriods, err := telemetry.NewFloat64Gauge(meter, "forgectl_rail_missed_periods", "Count of proving periods where proof was missed", "periods") |
| 133 | + if err != nil { |
| 134 | + return nil, err |
| 135 | + } |
| 136 | + |
| 137 | + return &faultMetrics{ |
| 138 | + railMissedPeriods: railMissedPeriods, |
| 139 | + }, nil |
| 140 | +} |
| 141 | + |
| 142 | +func collectFaultMetrics( |
| 143 | + ctx context.Context, |
| 144 | + inspctr *inspector.Service, |
| 145 | + fm *faultMetrics, |
| 146 | + tokenAddr, payerAddr common.Address, |
| 147 | +) error { |
| 148 | + fetchCtx, cancel := context.WithTimeout(ctx, 30*time.Second) |
| 149 | + defer cancel() |
| 150 | + |
| 151 | + status, err := inspctr.PaymentsStatus(fetchCtx, tokenAddr, payerAddr, 0, 100) |
| 152 | + if err != nil { |
| 153 | + return fmt.Errorf("failed to fetch payment status: %w", err) |
| 154 | + } |
| 155 | + |
| 156 | + // Record per-rail missed proving periods |
| 157 | + maxProvingPeriod := status.MaxProvingPeriod |
| 158 | + for _, payee := range status.Payees { |
| 159 | + for _, rail := range payee.Rails { |
| 160 | + if rail.LifetimeTotalEpochs != nil && rail.LifetimeProvenEpochs != nil && maxProvingPeriod > 0 && rail.DataSetId != nil { |
| 161 | + missedEpochs := new(big.Int).Sub(rail.LifetimeTotalEpochs, rail.LifetimeProvenEpochs) |
| 162 | + // Floor division: definite count of fully missed proving periods |
| 163 | + missedPeriods := missedEpochs.Uint64() / maxProvingPeriod |
| 164 | + |
| 165 | + railAttrs := []attribute.KeyValue{ |
| 166 | + attribute.String("payer", payerAddr.Hex()), |
| 167 | + attribute.String("payee", payee.Address.Hex()), |
| 168 | + attribute.String("dataset_id", rail.DataSetId.String()), |
| 169 | + } |
| 170 | + fm.railMissedPeriods.Record(ctx, float64(missedPeriods), railAttrs...) |
| 171 | + |
| 172 | + log.Infow("recorded rail fault metrics", |
| 173 | + "payee", payee.Address.Hex(), |
| 174 | + "dataset_id", rail.DataSetId.String(), |
| 175 | + "missed_periods", missedPeriods, |
| 176 | + ) |
| 177 | + } |
| 178 | + } |
| 179 | + } |
| 180 | + |
| 181 | + return nil |
| 182 | +} |
0 commit comments