Skip to content

Commit d0eb75b

Browse files
committed
chain validation and fix command
1 parent e5f7511 commit d0eb75b

File tree

12 files changed

+1227
-38
lines changed

12 files changed

+1227
-38
lines changed

cmd/root.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ func init() {
198198
viper.BindPFlag("publisher.events.topic0Filter", rootCmd.PersistentFlags().Lookup("publisher-events-topic0Filter"))
199199
rootCmd.AddCommand(orchestratorCmd)
200200
rootCmd.AddCommand(apiCmd)
201+
rootCmd.AddCommand(validateCmd)
201202
}
202203

203204
func initConfig() {

cmd/validate.go

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
package cmd
2+
3+
import (
4+
"crypto/tls"
5+
"fmt"
6+
"math/big"
7+
"strconv"
8+
9+
"github.com/ClickHouse/clickhouse-go/v2"
10+
"github.com/rs/zerolog/log"
11+
"github.com/spf13/cobra"
12+
config "github.com/thirdweb-dev/indexer/configs"
13+
"github.com/thirdweb-dev/indexer/internal/rpc"
14+
"github.com/thirdweb-dev/indexer/internal/storage"
15+
"github.com/thirdweb-dev/indexer/internal/validation"
16+
)
17+
18+
var (
19+
validateCmd = &cobra.Command{
20+
Use: "validate",
21+
Short: "TBD",
22+
Long: "TBD",
23+
Run: func(cmd *cobra.Command, args []string) {
24+
RunValidate(cmd, args)
25+
},
26+
}
27+
)
28+
29+
func RunValidate(cmd *cobra.Command, args []string) {
30+
batchSize := big.NewInt(1000)
31+
fixBatchSize := 0 // default is no batch size
32+
if len(args) > 0 {
33+
batchSizeFromArgs, err := strconv.Atoi(args[0])
34+
if err != nil {
35+
log.Fatal().Err(err).Msg("Failed to parse batch size")
36+
}
37+
if batchSizeFromArgs < 1 {
38+
batchSizeFromArgs = 1
39+
}
40+
batchSize = big.NewInt(int64(batchSizeFromArgs))
41+
log.Info().Msgf("Using batch size %d from args", batchSize)
42+
}
43+
if len(args) > 1 {
44+
fixBatchSizeFromArgs, err := strconv.Atoi(args[1])
45+
if err != nil {
46+
log.Fatal().Err(err).Msg("Failed to parse fix batch size")
47+
}
48+
fixBatchSize = fixBatchSizeFromArgs
49+
}
50+
log.Debug().Msgf("Batch size: %d, fix batch size: %d", batchSize, fixBatchSize)
51+
batchSize = new(big.Int).Sub(batchSize, big.NewInt(1)) // -1 because range ends are inclusive
52+
53+
rpcClient, err := rpc.Initialize()
54+
if err != nil {
55+
log.Fatal().Err(err).Msg("Failed to initialize RPC")
56+
}
57+
log.Info().Msgf("Running validation for chain %d", rpcClient.GetChainID())
58+
59+
s, err := storage.NewStorageConnector(&config.Cfg.Storage)
60+
if err != nil {
61+
log.Fatal().Err(err).Msg("Failed to initialize storage")
62+
}
63+
cursor, err := validation.InitCursor(rpcClient.GetChainID(), s)
64+
if err != nil {
65+
log.Fatal().Err(err).Msg("Failed to initialize cursor")
66+
}
67+
log.Debug().Msgf("Cursor initialized for chain %d, starting from block %d", rpcClient.GetChainID(), cursor.LastScannedBlockNumber)
68+
69+
conn, err := clickhouse.Open(&clickhouse.Options{
70+
Addr: []string{fmt.Sprintf("%s:%d", config.Cfg.Storage.Main.Clickhouse.Host, config.Cfg.Storage.Main.Clickhouse.Port)},
71+
Protocol: clickhouse.Native,
72+
TLS: &tls.Config{},
73+
Auth: clickhouse.Auth{
74+
Username: config.Cfg.Storage.Main.Clickhouse.Username,
75+
Password: config.Cfg.Storage.Main.Clickhouse.Password,
76+
},
77+
Settings: func() clickhouse.Settings {
78+
settings := clickhouse.Settings{
79+
"do_not_merge_across_partitions_select_final": "1",
80+
"use_skip_indexes_if_final": "1",
81+
"optimize_move_to_prewhere_if_final": "1",
82+
"async_insert": "1",
83+
"wait_for_async_insert": "1",
84+
}
85+
return settings
86+
}(),
87+
})
88+
if err != nil {
89+
log.Fatal().Err(err).Msg("Failed to connect to ClickHouse")
90+
}
91+
defer conn.Close()
92+
93+
startBlock := new(big.Int).Add(cursor.LastScannedBlockNumber, big.NewInt(1))
94+
95+
for startBlock.Cmp(cursor.MaxBlockNumber) <= 0 {
96+
batchEndBlock := new(big.Int).Add(startBlock, batchSize)
97+
if batchEndBlock.Cmp(cursor.MaxBlockNumber) > 0 {
98+
batchEndBlock = new(big.Int).Set(cursor.MaxBlockNumber)
99+
}
100+
101+
log.Info().Msgf("Validating batch of blocks from %s to %s", startBlock.String(), batchEndBlock.String())
102+
err := validateAndFixRange(rpcClient, s, conn, startBlock, batchEndBlock, fixBatchSize)
103+
if err != nil {
104+
log.Fatal().Err(err).Msgf("failed to validate range %v-%v", startBlock, batchEndBlock)
105+
}
106+
107+
startBlock = new(big.Int).Add(batchEndBlock, big.NewInt(1))
108+
cursor.Update(batchEndBlock)
109+
}
110+
}
111+
112+
/**
113+
* Validates a range of blocks (end and start are inclusive) for a given chain and fixes any problems it finds
114+
*/
115+
func validateAndFixRange(rpcClient rpc.IRPCClient, s storage.IStorage, conn clickhouse.Conn, startBlock *big.Int, endBlock *big.Int, fixBatchSize int) error {
116+
chainId := rpcClient.GetChainID()
117+
err := validation.FindAndRemoveDuplicates(conn, chainId, startBlock, endBlock)
118+
if err != nil {
119+
log.Fatal().Err(err).Msg("Failed to find and fix duplicates")
120+
}
121+
122+
err = validation.FindAndFixGaps(rpcClient, s, conn, chainId, startBlock, endBlock)
123+
if err != nil {
124+
log.Fatal().Err(err).Msg("Failed to find and fix gaps")
125+
}
126+
127+
err = validation.ValidateAndFixBlocks(rpcClient, s, conn, startBlock, endBlock, fixBatchSize)
128+
if err != nil {
129+
log.Fatal().Err(err).Msg("Failed to validate and fix blocks")
130+
}
131+
132+
log.Debug().Msgf("Validation complete for range %v-%v", startBlock, endBlock)
133+
return nil
134+
}

go.mod

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/ethereum/go-ethereum v1.14.8
88
github.com/gin-gonic/gin v1.10.0
99
github.com/gorilla/schema v1.4.1
10+
github.com/holiman/uint256 v1.3.1
1011
github.com/prometheus/client_golang v1.20.4
1112
github.com/rs/zerolog v1.33.0
1213
github.com/spf13/cobra v1.8.1
@@ -20,6 +21,7 @@ require (
2021

2122
require (
2223
github.com/ClickHouse/ch-go v0.63.1 // indirect
24+
github.com/DataDog/zstd v1.4.5 // indirect
2325
github.com/KyleBanks/depth v1.2.1 // indirect
2426
github.com/Microsoft/go-winio v0.6.2 // indirect
2527
github.com/andybalholm/brotli v1.1.1 // indirect
@@ -31,16 +33,25 @@ require (
3133
github.com/cespare/xxhash/v2 v2.3.0 // indirect
3234
github.com/cloudwego/base64x v0.1.4 // indirect
3335
github.com/cloudwego/iasm v0.2.0 // indirect
36+
github.com/cockroachdb/errors v1.11.3 // indirect
37+
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
38+
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
39+
github.com/cockroachdb/pebble v1.1.1 // indirect
40+
github.com/cockroachdb/redact v1.1.5 // indirect
41+
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
3442
github.com/consensys/bavard v0.1.13 // indirect
3543
github.com/consensys/gnark-crypto v0.12.1 // indirect
3644
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
45+
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
3746
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
3847
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
3948
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
4049
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
4150
github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
51+
github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect
4252
github.com/fsnotify/fsnotify v1.7.0 // indirect
4353
github.com/gabriel-vasile/mimetype v1.4.7 // indirect
54+
github.com/getsentry/sentry-go v0.27.0 // indirect
4455
github.com/gin-contrib/sse v0.1.0 // indirect
4556
github.com/go-faster/city v1.0.1 // indirect
4657
github.com/go-faster/errors v0.7.1 // indirect
@@ -53,26 +64,32 @@ require (
5364
github.com/go-playground/universal-translator v0.18.1 // indirect
5465
github.com/go-playground/validator/v10 v10.23.0 // indirect
5566
github.com/goccy/go-json v0.10.4 // indirect
67+
github.com/gofrs/flock v0.8.1 // indirect
68+
github.com/gogo/protobuf v1.3.2 // indirect
5669
github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
70+
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
5771
github.com/google/uuid v1.6.0 // indirect
5872
github.com/gorilla/websocket v1.4.2 // indirect
5973
github.com/hashicorp/hcl v1.0.0 // indirect
60-
github.com/holiman/uint256 v1.3.1 // indirect
6174
github.com/inconshreveable/mousetrap v1.1.0 // indirect
6275
github.com/josharian/intern v1.0.0 // indirect
6376
github.com/json-iterator/go v1.1.12 // indirect
6477
github.com/klauspost/compress v1.17.11 // indirect
6578
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
79+
github.com/kr/pretty v0.3.1 // indirect
80+
github.com/kr/text v0.2.0 // indirect
6681
github.com/leodido/go-urn v1.4.0 // indirect
6782
github.com/magiconair/properties v1.8.7 // indirect
6883
github.com/mailru/easyjson v0.7.7 // indirect
6984
github.com/mattn/go-colorable v0.1.13 // indirect
7085
github.com/mattn/go-isatty v0.0.20 // indirect
86+
github.com/mattn/go-runewidth v0.0.13 // indirect
7187
github.com/mitchellh/mapstructure v1.5.0 // indirect
7288
github.com/mmcloughlin/addchain v0.4.0 // indirect
7389
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
7490
github.com/modern-go/reflect2 v1.0.2 // indirect
7591
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
92+
github.com/olekukonko/tablewriter v0.0.5 // indirect
7693
github.com/paulmach/orb v0.11.1 // indirect
7794
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
7895
github.com/pierrec/lz4/v4 v4.1.22 // indirect
@@ -81,6 +98,8 @@ require (
8198
github.com/prometheus/client_model v0.6.1 // indirect
8299
github.com/prometheus/common v0.55.0 // indirect
83100
github.com/prometheus/procfs v0.15.1 // indirect
101+
github.com/rivo/uniseg v0.2.0 // indirect
102+
github.com/rogpeppe/go-internal v1.11.0 // indirect
84103
github.com/sagikazarmark/locafero v0.4.0 // indirect
85104
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
86105
github.com/segmentio/asm v1.2.0 // indirect
@@ -93,6 +112,7 @@ require (
93112
github.com/stretchr/objx v0.5.2 // indirect
94113
github.com/subosito/gotenv v1.6.0 // indirect
95114
github.com/supranational/blst v0.3.11 // indirect
115+
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
96116
github.com/tklauser/go-sysconf v0.3.12 // indirect
97117
github.com/tklauser/numcpus v0.6.1 // indirect
98118
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect

0 commit comments

Comments
 (0)