diff --git a/cmd/root.go b/cmd/root.go index f47a2e9..8e664bc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -198,6 +198,7 @@ func init() { viper.BindPFlag("publisher.events.topic0Filter", rootCmd.PersistentFlags().Lookup("publisher-events-topic0Filter")) rootCmd.AddCommand(orchestratorCmd) rootCmd.AddCommand(apiCmd) + rootCmd.AddCommand(validateCmd) } func initConfig() { diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 0000000..faecf19 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,134 @@ +package cmd + +import ( + "crypto/tls" + "fmt" + "math/big" + "strconv" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + config "github.com/thirdweb-dev/indexer/configs" + "github.com/thirdweb-dev/indexer/internal/rpc" + "github.com/thirdweb-dev/indexer/internal/storage" + "github.com/thirdweb-dev/indexer/internal/validation" +) + +var ( + validateCmd = &cobra.Command{ + Use: "validate", + Short: "TBD", + Long: "TBD", + Run: func(cmd *cobra.Command, args []string) { + RunValidate(cmd, args) + }, + } +) + +func RunValidate(cmd *cobra.Command, args []string) { + batchSize := big.NewInt(1000) + fixBatchSize := 0 // default is no batch size + if len(args) > 0 { + batchSizeFromArgs, err := strconv.Atoi(args[0]) + if err != nil { + log.Fatal().Err(err).Msg("Failed to parse batch size") + } + if batchSizeFromArgs < 1 { + batchSizeFromArgs = 1 + } + batchSize = big.NewInt(int64(batchSizeFromArgs)) + log.Info().Msgf("Using batch size %d from args", batchSize) + } + if len(args) > 1 { + fixBatchSizeFromArgs, err := strconv.Atoi(args[1]) + if err != nil { + log.Fatal().Err(err).Msg("Failed to parse fix batch size") + } + fixBatchSize = fixBatchSizeFromArgs + } + log.Debug().Msgf("Batch size: %d, fix batch size: %d", batchSize, fixBatchSize) + batchSize = new(big.Int).Sub(batchSize, big.NewInt(1)) // -1 because range ends are inclusive + + rpcClient, err := rpc.Initialize() + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize RPC") + } + log.Info().Msgf("Running validation for chain %d", rpcClient.GetChainID()) + + s, err := storage.NewStorageConnector(&config.Cfg.Storage) + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize storage") + } + cursor, err := validation.InitCursor(rpcClient.GetChainID(), s) + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize cursor") + } + log.Debug().Msgf("Cursor initialized for chain %d, starting from block %d", rpcClient.GetChainID(), cursor.LastScannedBlockNumber) + + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{fmt.Sprintf("%s:%d", config.Cfg.Storage.Main.Clickhouse.Host, config.Cfg.Storage.Main.Clickhouse.Port)}, + Protocol: clickhouse.Native, + TLS: &tls.Config{}, + Auth: clickhouse.Auth{ + Username: config.Cfg.Storage.Main.Clickhouse.Username, + Password: config.Cfg.Storage.Main.Clickhouse.Password, + }, + Settings: func() clickhouse.Settings { + settings := clickhouse.Settings{ + "do_not_merge_across_partitions_select_final": "1", + "use_skip_indexes_if_final": "1", + "optimize_move_to_prewhere_if_final": "1", + "async_insert": "1", + "wait_for_async_insert": "1", + } + return settings + }(), + }) + if err != nil { + log.Fatal().Err(err).Msg("Failed to connect to ClickHouse") + } + defer conn.Close() + + startBlock := new(big.Int).Add(cursor.LastScannedBlockNumber, big.NewInt(1)) + + for startBlock.Cmp(cursor.MaxBlockNumber) <= 0 { + batchEndBlock := new(big.Int).Add(startBlock, batchSize) + if batchEndBlock.Cmp(cursor.MaxBlockNumber) > 0 { + batchEndBlock = new(big.Int).Set(cursor.MaxBlockNumber) + } + + log.Info().Msgf("Validating batch of blocks from %s to %s", startBlock.String(), batchEndBlock.String()) + err := validateAndFixRange(rpcClient, s, conn, startBlock, batchEndBlock, fixBatchSize) + if err != nil { + log.Fatal().Err(err).Msgf("failed to validate range %v-%v", startBlock, batchEndBlock) + } + + startBlock = new(big.Int).Add(batchEndBlock, big.NewInt(1)) + cursor.Update(batchEndBlock) + } +} + +/** + * Validates a range of blocks (end and start are inclusive) for a given chain and fixes any problems it finds + */ +func validateAndFixRange(rpcClient rpc.IRPCClient, s storage.IStorage, conn clickhouse.Conn, startBlock *big.Int, endBlock *big.Int, fixBatchSize int) error { + chainId := rpcClient.GetChainID() + err := validation.FindAndRemoveDuplicates(conn, chainId, startBlock, endBlock) + if err != nil { + log.Fatal().Err(err).Msg("Failed to find and fix duplicates") + } + + err = validation.FindAndFixGaps(rpcClient, s, conn, chainId, startBlock, endBlock) + if err != nil { + log.Fatal().Err(err).Msg("Failed to find and fix gaps") + } + + err = validation.ValidateAndFixBlocks(rpcClient, s, conn, startBlock, endBlock, fixBatchSize) + if err != nil { + log.Fatal().Err(err).Msg("Failed to validate and fix blocks") + } + + log.Debug().Msgf("Validation complete for range %v-%v", startBlock, endBlock) + return nil +} diff --git a/go.mod b/go.mod index ac1ff73..5b7ccc2 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,13 @@ module github.com/thirdweb-dev/indexer -go 1.23 +go 1.23.0 require ( github.com/ClickHouse/clickhouse-go/v2 v2.30.1 - github.com/ethereum/go-ethereum v1.14.8 + github.com/ethereum/go-ethereum v1.15.11 github.com/gin-gonic/gin v1.10.0 github.com/gorilla/schema v1.4.1 + github.com/holiman/uint256 v1.3.2 github.com/prometheus/client_golang v1.20.4 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 @@ -20,27 +21,39 @@ require ( require ( github.com/ClickHouse/ch-go v0.63.1 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.10.0 // indirect + github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/bytedance/sonic v1.12.6 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v1.1.2 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect + github.com/consensys/bavard v0.1.27 // indirect + github.com/consensys/gnark-crypto v0.16.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect - github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect + github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/c-kzg-4844/v2 v2.1.0 // indirect + github.com/ethereum/go-verkle v0.2.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect @@ -53,26 +66,32 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.23.0 // indirect github.com/goccy/go-json v0.10.4 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.3.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect @@ -81,6 +100,8 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -92,25 +113,26 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.11 // indirect + github.com/supranational/blst v0.3.14 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twmb/franz-go/pkg/kmsg v1.9.0 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/urfave/cli/v2 v2.27.4 // indirect + github.com/urfave/cli/v2 v2.27.5 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/otel v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.12.0 // indirect - golang.org/x/crypto v0.33.0 // indirect + golang.org/x/crypto v0.35.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect - golang.org/x/net v0.35.0 // indirect + golang.org/x/net v0.36.0 // indirect golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.22.0 // indirect - golang.org/x/tools v0.25.0 // indirect + golang.org/x/tools v0.29.0 // indirect google.golang.org/protobuf v1.36.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 18c8cf2..78381d8 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= +github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -31,6 +33,8 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= @@ -39,22 +43,35 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/bavard v0.1.27 h1:j6hKUrGAy/H+gpNrpLU3I26n1yc+VMGmd6ID5+gAhOs= +github.com/consensys/bavard v0.1.27/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/gnark-crypto v0.16.0 h1:8Dl4eYmUWK9WmlP1Bj6je688gBRJCJbT8Mw4KoTAawo= +github.com/consensys/gnark-crypto v0.16.0/go.mod h1:Ke3j06ndtPTVvo++PhGNgvm+lgpLvzbcE2MqljY7diU= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-eth-kzg v1.3.0 h1:05GrhASN9kDAidaFJOda6A4BEvgvuXbazXg/0E3OOdI= +github.com/crate-crypto/go-eth-kzg v1.3.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= +github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -67,12 +84,20 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844/v2 v2.1.0 h1:gQropX9YFBhl3g4HYhwE70zq3IHFRgbbNPw0Shwzf5w= +github.com/ethereum/c-kzg-4844/v2 v2.1.0/go.mod h1:TC48kOKjJKPbN7C++qIgt0TJzZ70QznYR7Ob+WXl57E= github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig= github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= +github.com/ethereum/go-ethereum v1.15.11 h1:JK73WKeu0WC0O1eyX+mdQAVHUV+UR1a9VB/domDngBU= +github.com/ethereum/go-ethereum v1.15.11/go.mod h1:mf8YiHIb0GR4x4TipcvBUPxJLw1mFdmxzoDi11sDRoI= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= +github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= +github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= @@ -87,6 +112,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= @@ -119,10 +146,21 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -147,6 +185,9 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -177,6 +218,7 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -189,6 +231,7 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -206,8 +249,17 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= @@ -215,6 +267,9 @@ github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNH github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -230,8 +285,10 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -280,6 +337,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= +github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= @@ -305,6 +364,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8= github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= @@ -334,6 +395,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -341,9 +404,12 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -351,6 +417,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -358,9 +427,16 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -380,6 +456,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -388,6 +465,7 @@ golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -395,10 +473,20 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= @@ -407,10 +495,15 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/common/trace.go b/internal/common/trace.go index b12b076..86791c9 100644 --- a/internal/common/trace.go +++ b/internal/common/trace.go @@ -13,14 +13,14 @@ type Trace struct { TransactionHash string `json:"transaction_hash" ch:"transaction_hash"` TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"` Subtraces int64 `json:"subtraces" ch:"subtraces"` - TraceAddress []uint64 `json:"trace_address" ch:"trace_address"` + TraceAddress []int64 `json:"trace_address" ch:"trace_address"` TraceType string `json:"trace_type" ch:"type"` CallType string `json:"call_type" ch:"call_type"` Error string `json:"error" ch:"error"` FromAddress string `json:"from_address" ch:"from_address"` ToAddress string `json:"to_address" ch:"to_address"` - Gas *big.Int `json:"gas" ch:"gas"` - GasUsed *big.Int `json:"gas_used" ch:"gas_used"` + Gas uint64 `json:"gas" ch:"gas"` + GasUsed uint64 `json:"gas_used" ch:"gas_used"` Input string `json:"input" ch:"input"` Output string `json:"output" ch:"output"` Value *big.Int `json:"value" ch:"value"` @@ -34,27 +34,27 @@ type Trace struct { type RawTraces = []map[string]interface{} type TraceModel struct { - ChainId string `json:"chain_id"` - BlockNumber uint64 `json:"block_number"` - BlockHash string `json:"block_hash"` - BlockTimestamp uint64 `json:"block_timestamp"` - TransactionHash string `json:"transaction_hash"` - TransactionIndex uint64 `json:"transaction_index"` - Subtraces int64 `json:"subtraces"` - TraceAddress []uint64 `json:"trace_address"` - TraceType string `json:"trace_type"` - CallType string `json:"call_type"` - Error string `json:"error"` - FromAddress string `json:"from_address"` - ToAddress string `json:"to_address"` - Gas uint64 `json:"gas"` - GasUsed uint64 `json:"gas_used"` - Input string `json:"input"` - Output string `json:"output"` - Value uint64 `json:"value"` - Author string `json:"author"` - RewardType string `json:"reward_type"` - RefundAddress string `json:"refund_address"` + ChainId string `json:"chain_id"` + BlockNumber uint64 `json:"block_number"` + BlockHash string `json:"block_hash"` + BlockTimestamp uint64 `json:"block_timestamp"` + TransactionHash string `json:"transaction_hash"` + TransactionIndex uint64 `json:"transaction_index"` + Subtraces int64 `json:"subtraces"` + TraceAddress []int64 `json:"trace_address"` + TraceType string `json:"trace_type"` + CallType string `json:"call_type"` + Error string `json:"error"` + FromAddress string `json:"from_address"` + ToAddress string `json:"to_address"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gas_used"` + Input string `json:"input"` + Output string `json:"output"` + Value uint64 `json:"value"` + Author string `json:"author"` + RewardType string `json:"reward_type"` + RefundAddress string `json:"refund_address"` } func (t *Trace) Serialize() TraceModel { @@ -71,8 +71,8 @@ func (t *Trace) Serialize() TraceModel { Error: t.Error, FromAddress: t.FromAddress, ToAddress: t.ToAddress, - Gas: t.Gas.Uint64(), - GasUsed: t.GasUsed.Uint64(), + Gas: t.Gas, + GasUsed: t.GasUsed, Input: t.Input, Output: t.Output, Value: t.Value.Uint64(), diff --git a/internal/rpc/serializer.go b/internal/rpc/serializer.go index 01f6346..53c2f5e 100644 --- a/internal/rpc/serializer.go +++ b/internal/rpc/serializer.go @@ -379,8 +379,8 @@ func serializeTrace(chainId *big.Int, trace map[string]interface{}, block common Error: interfaceToString(trace["error"]), FromAddress: interfaceToString(action["from"]), ToAddress: interfaceToString(action["to"]), - Gas: hexToBigInt(action["gas"]), - GasUsed: hexToBigInt(result["gasUsed"]), + Gas: hexToUint64(action["gas"]), + GasUsed: hexToUint64(result["gasUsed"]), Input: interfaceToString(action["input"]), Output: interfaceToString(result["output"]), Value: hexToBigInt(action["value"]), @@ -399,15 +399,15 @@ func hexToBigInt(hex interface{}) *big.Int { return v } -func serializeTraceAddress(traceAddress interface{}) []uint64 { +func serializeTraceAddress(traceAddress interface{}) []int64 { if traceAddressSlice, ok := traceAddress.([]interface{}); ok { - var addresses []uint64 + var addresses []int64 for _, addr := range traceAddressSlice { - addresses = append(addresses, uint64(addr.(float64))) + addresses = append(addresses, int64(addr.(float64))) } return addresses } - return []uint64{} + return []int64{} } func hexToTime(hex interface{}) time.Time { diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index b552d36..fb93069 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -372,8 +372,8 @@ func (c *ClickHouseConnector) insertTraces(traces []common.Trace, opt InsertOpti trace.Error, trace.FromAddress, trace.ToAddress, - trace.Gas.Uint64(), - trace.GasUsed.Uint64(), + trace.Gas, + trace.GasUsed, trace.Input, trace.Output, trace.Value, @@ -471,6 +471,10 @@ func (c *ClickHouseConnector) GetAggregations(table string, qf QueryFilter) (Que if qf.ChainId != nil && qf.ChainId.Sign() > 0 { whereClauses = append(whereClauses, createFilterClause("chain_id", qf.ChainId.String())) } + blockNumbersClause := createBlockNumbersClause(qf.BlockNumbers) + if blockNumbersClause != "" { + whereClauses = append(whereClauses, blockNumbersClause) + } contractAddressClause := createContractAddressClause(table, qf.ContractAddress) if contractAddressClause != "" { whereClauses = append(whereClauses, contractAddressClause) @@ -929,7 +933,7 @@ func (c *ClickHouseConnector) InsertStagingData(data []common.BlockData) error { } func (c *ClickHouseConnector) GetStagingData(qf QueryFilter) ([]common.BlockData, error) { - query := fmt.Sprintf("SELECT data FROM %s.block_data WHERE block_number IN (%s) AND is_deleted = 0", + query := fmt.Sprintf("SELECT data FROM %s.block_data FINAL WHERE block_number IN (%s) AND is_deleted = 0", c.cfg.Database, getBlockNumbersStringArray(qf.BlockNumbers)) if qf.ChainId.Sign() != 0 { @@ -1280,8 +1284,8 @@ func (c *ClickHouseConnector) InsertBlockData(data []common.BlockData) error { trace.Error, trace.FromAddress, trace.ToAddress, - trace.Gas.Uint64(), - trace.GasUsed.Uint64(), + trace.Gas, + trace.GasUsed, trace.Input, trace.Output, trace.Value, diff --git a/internal/validation/cursor.go b/internal/validation/cursor.go new file mode 100644 index 0000000..d954446 --- /dev/null +++ b/internal/validation/cursor.go @@ -0,0 +1,72 @@ +package validation + +import ( + "encoding/json" + "fmt" + "math/big" + "os" + + "github.com/thirdweb-dev/indexer/internal/storage" +) + +type Cursor struct { + LastScannedBlockNumber *big.Int + MaxBlockNumber *big.Int + ChainId *big.Int +} + +func InitCursor(chainId *big.Int, storage storage.IStorage) (*Cursor, error) { + lastScannedBlock := getLastScannedBlock(chainId) + maxBlockNumber, err := storage.MainStorage.GetMaxBlockNumber(chainId) + if err != nil { + return nil, err + } + if maxBlockNumber == nil { + maxBlockNumber = big.NewInt(0) + } + if lastScannedBlock.Cmp(maxBlockNumber) >= 0 { + return nil, fmt.Errorf("last scanned block number is greater than or equal to max block number") + } + return &Cursor{ + LastScannedBlockNumber: lastScannedBlock, + MaxBlockNumber: maxBlockNumber, + ChainId: chainId, + }, nil +} + +func (c *Cursor) Update(blockNumber *big.Int) error { + cursorFile := fmt.Sprintf("validation_cursor_%s.json", c.ChainId.String()) + jsonData, err := json.Marshal(blockNumber.String()) + if err != nil { + return err + } + + err = os.WriteFile(cursorFile, jsonData, 0644) + if err != nil { + return err + } + c.LastScannedBlockNumber = blockNumber + return nil +} + +func getLastScannedBlock(chainId *big.Int) *big.Int { + cursorFile := fmt.Sprintf("validation_cursor_%s.json", chainId.String()) + if _, err := os.Stat(cursorFile); err != nil { + return big.NewInt(0) + } + + fileData, err := os.ReadFile(cursorFile) + if err != nil { + return big.NewInt(0) + } + + var lastBlock string + err = json.Unmarshal(fileData, &lastBlock) + if err != nil { + return big.NewInt(0) + } + + lastBlockBig := new(big.Int) + lastBlockBig.SetString(lastBlock, 10) + return lastBlockBig +} diff --git a/internal/validation/db.go b/internal/validation/db.go new file mode 100644 index 0000000..ccca9d9 --- /dev/null +++ b/internal/validation/db.go @@ -0,0 +1,257 @@ +package validation + +import ( + "context" + "fmt" + "math/big" + + "github.com/ClickHouse/clickhouse-go/v2" +) + +type Block struct { + ChainId *big.Int `json:"chain_id" ch:"chain_id"` + Number *big.Int `json:"block_number" ch:"block_number"` + TransactionsRoot string `json:"transactions_root" ch:"transactions_root"` + ReceiptsRoot string `json:"receipts_root" ch:"receipts_root"` + LogsBloom string `json:"logs_bloom" ch:"logs_bloom"` + TransactionCount uint64 `json:"transaction_count" ch:"transaction_count"` +} + +type Log struct { + ChainId *big.Int `json:"chain_id" ch:"chain_id"` + Number *big.Int `json:"block_number" ch:"block_number"` + Address string `json:"address" ch:"address"` + LogIndex uint64 `json:"log_index" ch:"log_index"` + Topic0 string `json:"topic_0" ch:"topic_0"` + Topic1 string `json:"topic_1" ch:"topic_1"` + Topic2 string `json:"topic_2" ch:"topic_2"` + Topic3 string `json:"topic_3" ch:"topic_3"` +} + +type Transaction struct { + ChainId *big.Int `json:"chain_id" ch:"chain_id"` + Number *big.Int `json:"block_number" ch:"block_number"` + Nonce uint64 `json:"nonce" ch:"nonce"` + TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"` + ToAddress string `json:"to_address" ch:"to_address"` + Value *big.Int `json:"value" ch:"value" swaggertype:"string"` + Gas uint64 `json:"gas" ch:"gas"` + GasPrice *big.Int `json:"gas_price" ch:"gas_price" swaggertype:"string"` + Data string `json:"data" ch:"data"` + MaxFeePerGas *big.Int `json:"max_fee_per_gas" ch:"max_fee_per_gas" swaggertype:"string"` + MaxPriorityFeePerGas *big.Int `json:"max_priority_fee_per_gas" ch:"max_priority_fee_per_gas" swaggertype:"string"` + MaxFeePerBlobGas *big.Int `json:"max_fee_per_blob_gas" ch:"max_fee_per_blob_gas" swaggertype:"string"` + BlobVersionedHashes []string `json:"blob_versioned_hashes" ch:"blob_versioned_hashes"` + TransactionType uint8 `json:"transaction_type" ch:"transaction_type"` + R *big.Int `json:"r" ch:"r" swaggertype:"string"` + S *big.Int `json:"s" ch:"s" swaggertype:"string"` + V *big.Int `json:"v" ch:"v" swaggertype:"string"` + AccessListJson *string `json:"access_list_json" ch:"access_list"` + AuthorizationListJson *string `json:"authorization_list_json" ch:"authorization_list"` + BlobGasUsed *uint64 `json:"blob_gas_used" ch:"blob_gas_used"` + BlobGasPrice *big.Int `json:"blob_gas_price" ch:"blob_gas_price" swaggertype:"string"` +} + +type BlockData struct { + Block Block + Transactions []Transaction + Logs []Log +} + +func FetchBlockDataFromDBForRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]BlockData, error) { + // Get blocks, logs and transactions concurrently + type blockResult struct { + blocks []Block + err error + } + + type logResult struct { + logMap map[string][]Log // blockNumber -> logs + err error + } + + type txResult struct { + txMap map[string][]Transaction // blockNumber -> transactions + err error + } + + blocksChan := make(chan blockResult) + logsChan := make(chan logResult) + txsChan := make(chan txResult) + + // Launch goroutines for concurrent fetching + go func() { + blocks, err := getBlocksForRange(conn, chainId, startBlock, endBlock) + blocksChan <- blockResult{blocks: blocks, err: err} + }() + + go func() { + logs, err := getLogsForRange(conn, chainId, startBlock, endBlock) + if err != nil { + logsChan <- logResult{err: err} + return + } + + // Pre-organize logs by block number + logMap := make(map[string][]Log) + for _, log := range logs { + blockNum := log.Number.String() + logMap[blockNum] = append(logMap[blockNum], log) + } + logsChan <- logResult{logMap: logMap} + }() + + go func() { + transactions, err := getTransactionsForRange(conn, chainId, startBlock, endBlock) + if err != nil { + txsChan <- txResult{err: err} + return + } + + // Pre-organize transactions by block number + txMap := make(map[string][]Transaction) + for _, tx := range transactions { + blockNum := tx.Number.String() + txMap[blockNum] = append(txMap[blockNum], tx) + } + txsChan <- txResult{txMap: txMap} + }() + + // Wait for all results + blocksResult := <-blocksChan + logsResult := <-logsChan + txsResult := <-txsChan + + // Check for errors + if blocksResult.err != nil { + return nil, fmt.Errorf("error fetching blocks: %v", blocksResult.err) + } + if logsResult.err != nil { + return nil, fmt.Errorf("error fetching logs: %v", logsResult.err) + } + if txsResult.err != nil { + return nil, fmt.Errorf("error fetching transactions: %v", txsResult.err) + } + + // Build BlockData slice + blockData := make([]BlockData, len(blocksResult.blocks)) + + // Build BlockData for each block + for i, block := range blocksResult.blocks { + blockNum := block.Number.String() + blockData[i] = BlockData{ + Block: block, + Logs: logsResult.logMap[blockNum], + Transactions: txsResult.txMap[blockNum], + } + } + + return blockData, nil +} + +func getBlocksForRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]Block, error) { + query := ` + SELECT + chain_id, + block_number, + transactions_root, + receipts_root, + logs_bloom, + transaction_count + FROM blocks FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + blocks := []Block{} + + for rows.Next() { + var block Block + err := rows.ScanStruct(&block) + if err != nil { + return nil, err + } + blocks = append(blocks, block) + } + return blocks, nil +} + +func getLogsForRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]Log, error) { + query := ` + SELECT + chain_id, + block_number, + log_index, + address, + topic_0, + topic_1, + topic_2, + topic_3 + FROM logs FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + logs := []Log{} + + for rows.Next() { + var log Log + err := rows.ScanStruct(&log) + if err != nil { + return nil, err + } + logs = append(logs, log) + } + return logs, nil +} + +func getTransactionsForRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]Transaction, error) { + query := ` + SELECT + chain_id, + block_number, + nonce, + transaction_index, + to_address, + value, + gas, + gas_price, + data, + max_fee_per_gas, + max_priority_fee_per_gas, + max_fee_per_blob_gas, + blob_versioned_hashes, + transaction_type, + r, + s, + v, + access_list, + authorization_list, + blob_gas_used, + blob_gas_price + FROM transactions FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + transactions := []Transaction{} + + for rows.Next() { + var transaction Transaction + err := rows.ScanStruct(&transaction) + if err != nil { + return nil, err + } + transactions = append(transactions, transaction) + } + return transactions, nil +} diff --git a/internal/validation/duplicates.go b/internal/validation/duplicates.go new file mode 100644 index 0000000..3e7da21 --- /dev/null +++ b/internal/validation/duplicates.go @@ -0,0 +1,237 @@ +package validation + +import ( + "context" + "fmt" + "math/big" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/rs/zerolog/log" +) + +type DuplicateTransaction struct { + BlockNumber *big.Int `json:"block_number" ch:"block_number"` + Hash string `json:"hash" ch:"hash"` +} + +type DuplicateLog struct { + BlockNumber *big.Int `json:"block_number" ch:"block_number"` + TxHash string `json:"transaction_hash" ch:"transaction_hash"` + LogIndex uint64 `json:"log_index" ch:"log_index"` +} + +func FindAndRemoveDuplicates(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) error { + duplicateBlockNumbers, err := findDuplicateBlocksInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(duplicateBlockNumbers) == 0 { + log.Debug().Msg("No duplicate blocks found in range") + } else { + log.Debug().Msgf("Found %d duplicate blocks in range %v-%v: %v", len(duplicateBlockNumbers), startBlock, endBlock, duplicateBlockNumbers) + removeDuplicateBlocks(conn, chainId, duplicateBlockNumbers) + } + + duplicateTransactions, err := findDuplicateTransactionsInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(duplicateTransactions) == 0 { + log.Debug().Msg("No duplicate transactions found in range") + } else { + log.Debug().Msgf("Found %d duplicate transactions in range %v-%v: %v", len(duplicateTransactions), startBlock, endBlock, duplicateTransactions) + removeDuplicateTransactions(conn, chainId, duplicateTransactions) + } + + duplicateLogs, err := findDuplicateLogsInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(duplicateLogs) == 0 { + log.Debug().Msg("No duplicate logs found in range") + } else { + log.Debug().Msgf("Found %d duplicate logs in range %v-%v: %v", len(duplicateLogs), startBlock, endBlock, duplicateLogs) + removeDuplicateLogs(conn, chainId, duplicateLogs) + } + + return nil +} + +func findDuplicateBlocksInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]*big.Int, error) { + query := `SELECT block_number + FROM default.blocks FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + GROUP BY block_number + HAVING sum(sign) != 1 + ORDER BY block_number; + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + blockNumbers := make([]*big.Int, 0) + + for rows.Next() { + var blockNumber *big.Int + err := rows.Scan(&blockNumber) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, blockNumber) + } + return blockNumbers, nil +} + +func findDuplicateTransactionsInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]DuplicateTransaction, error) { + query := `SELECT block_number, hash + FROM default.transactions FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + GROUP BY block_number, hash + HAVING sum(sign) != 1 + ORDER BY block_number; + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + duplicateTransactions := make([]DuplicateTransaction, 0) + + for rows.Next() { + var duplicateTransaction DuplicateTransaction + err := rows.ScanStruct(&duplicateTransaction) + if err != nil { + return nil, err + } + duplicateTransactions = append(duplicateTransactions, duplicateTransaction) + } + return duplicateTransactions, nil +} + +func findDuplicateLogsInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]DuplicateLog, error) { + query := `SELECT block_number, transaction_hash, log_index + FROM default.logs FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + GROUP BY block_number, transaction_hash, log_index + HAVING sum(sign) != 1 + ORDER BY block_number; + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + duplicateLogs := make([]DuplicateLog, 0) + + for rows.Next() { + var duplicateLog DuplicateLog + err := rows.ScanStruct(&duplicateLog) + if err != nil { + return nil, err + } + duplicateLogs = append(duplicateLogs, duplicateLog) + } + return duplicateLogs, nil +} + +func removeDuplicateBlocks(conn clickhouse.Conn, chainId *big.Int, duplicateBlockNumbers []*big.Int) error { + query := `WITH + to_be_inserted AS ( + SELECT chain_id, block_number, block_timestamp, hash, parent_hash, sha3_uncles, nonce, mix_hash, miner, state_root, + transactions_root, receipts_root, logs_bloom, size, extra_data, difficulty, total_difficulty, transaction_count, + gas_limit, gas_used, withdrawals_root, base_fee_per_gas, insert_timestamp, -sign as sign + FROM default.blocks FINAL + WHERE chain_id = ? AND block_number IN (?) + ) + INSERT INTO blocks ( + chain_id, block_number, block_timestamp, hash, parent_hash, sha3_uncles, nonce, mix_hash, miner, state_root, + transactions_root, receipts_root, logs_bloom, size, extra_data, difficulty, total_difficulty, transaction_count, + gas_limit, gas_used, withdrawals_root, base_fee_per_gas, insert_timestamp, sign + ) SELECT * from to_be_inserted + ` + err := conn.Exec(context.Background(), query, chainId, duplicateBlockNumbers) + if err != nil { + return err + } + return nil +} + +func removeDuplicateTransactions(conn clickhouse.Conn, chainId *big.Int, duplicateTransactions []DuplicateTransaction) error { + query := `WITH + to_be_inserted AS ( + SELECT chain_id, hash, nonce, block_hash, block_number, block_timestamp, transaction_index, from_address, to_address, value, gas, gas_price, data, function_selector, + max_fee_per_gas, max_priority_fee_per_gas, max_fee_per_blob_gas, blob_versioned_hashes, transaction_type, r, s, v, access_list, authorization_list, contract_address, + gas_used, cumulative_gas_used, effective_gas_price, blob_gas_used, blob_gas_price, logs_bloom, status, insert_timestamp, -sign as sign + FROM default.transactions FINAL + WHERE chain_id = ? AND block_number IN (?) AND hash IN (?) + ) + INSERT INTO transactions ( + chain_id, hash, nonce, block_hash, block_number, block_timestamp, transaction_index, from_address, to_address, value, gas, gas_price, data, function_selector, + max_fee_per_gas, max_priority_fee_per_gas, max_fee_per_blob_gas, blob_versioned_hashes, transaction_type, r, s, v, access_list, authorization_list, contract_address, + gas_used, cumulative_gas_used, effective_gas_price, blob_gas_used, blob_gas_price, logs_bloom, status, insert_timestamp, sign + ) SELECT * from to_be_inserted + ` + + const batchSize = 1000 + for i := 0; i < len(duplicateTransactions); i += batchSize { + end := i + batchSize + if end > len(duplicateTransactions) { + end = len(duplicateTransactions) + } + + batch := duplicateTransactions[i:end] + blockNumbers := make([]*big.Int, 0, len(batch)) + hashes := make([]string, 0, len(batch)) + + for _, duplicateTransaction := range batch { + blockNumbers = append(blockNumbers, duplicateTransaction.BlockNumber) + hashes = append(hashes, duplicateTransaction.Hash) + } + + err := conn.Exec(context.Background(), query, chainId, blockNumbers, hashes) + if err != nil { + return err + } + } + return nil +} + +func removeDuplicateLogs(conn clickhouse.Conn, chainId *big.Int, duplicateLogs []DuplicateLog) error { + const batchSize = 1000 + for i := 0; i < len(duplicateLogs); i += batchSize { + end := i + batchSize + if end > len(duplicateLogs) { + end = len(duplicateLogs) + } + + batch := duplicateLogs[i:end] + blockNumbers := make([]*big.Int, 0, len(batch)) + tuples := make([]string, 0, len(batch)) + + for _, duplicateLog := range batch { + blockNumbers = append(blockNumbers, duplicateLog.BlockNumber) + tuples = append(tuples, fmt.Sprintf("('%s', %d)", duplicateLog.TxHash, duplicateLog.LogIndex)) + } + + query := fmt.Sprintf(`WITH + to_be_inserted AS ( + SELECT chain_id, block_number, block_hash, block_timestamp, transaction_hash, transaction_index, log_index, address, + data, topic_0, topic_1, topic_2, topic_3, insert_timestamp, -sign as sign + FROM default.logs FINAL + WHERE chain_id = ? AND block_number IN (?) AND (transaction_hash, log_index) IN (%s) + ) + INSERT INTO logs ( + chain_id, block_number, block_hash, block_timestamp, transaction_hash, transaction_index, log_index, address, + data, topic_0, topic_1, topic_2, topic_3, insert_timestamp, sign + ) SELECT * from to_be_inserted + `, strings.Join(tuples, ",")) + + err := conn.Exec(context.Background(), query, chainId, blockNumbers) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/validation/gaps.go b/internal/validation/gaps.go new file mode 100644 index 0000000..d4f8ba0 --- /dev/null +++ b/internal/validation/gaps.go @@ -0,0 +1,94 @@ +package validation + +import ( + "context" + "math/big" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/rs/zerolog/log" + "github.com/thirdweb-dev/indexer/internal/orchestrator" + "github.com/thirdweb-dev/indexer/internal/rpc" + "github.com/thirdweb-dev/indexer/internal/storage" +) + +func FindAndFixGaps(rpc rpc.IRPCClient, s storage.IStorage, conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) error { + missingBlockNumbers, err := findMissingBlocksInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(missingBlockNumbers) == 0 { + log.Debug().Msg("No missing blocks found") + return nil + } + log.Debug().Msgf("Found %d missing blocks: %v", len(missingBlockNumbers), missingBlockNumbers) + + // query missing blocks + poller := orchestrator.NewBoundlessPoller(rpc, s) + poller.Poll(context.Background(), missingBlockNumbers) + log.Debug().Msg("Missing blocks polled") + + blocksData, err := s.StagingStorage.GetStagingData(storage.QueryFilter{BlockNumbers: missingBlockNumbers, ChainId: rpc.GetChainID()}) + if err != nil { + log.Fatal().Err(err).Msg("Failed to get staging data") + } + if len(blocksData) == 0 { + log.Fatal().Msg("Failed to get staging data") + } + + err = s.MainStorage.InsertBlockData(blocksData) + if err != nil { + log.Error().Err(err).Msgf("Failed to commit blocks: %v", blocksData) + } + + if err := s.StagingStorage.DeleteStagingData(blocksData); err != nil { + log.Error().Err(err).Msgf("Failed to delete staging data: %v", blocksData) + } + return nil +} + +func findMissingBlocksInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]*big.Int, error) { + query := ` + WITH sequence AS ( + SELECT + {startBlock:UInt256} + number AS expected_block_number + FROM + numbers(toUInt64({endBlock:UInt256} - {startBlock:UInt256} + 1)) + ), + existing_blocks AS ( + SELECT DISTINCT + block_number + FROM + blocks FINAL + WHERE + chain_id = {chainId:UInt256} + AND block_number >= {startBlock:UInt256} + AND block_number <= {endBlock:UInt256} + ) + SELECT + s.expected_block_number AS missing_block_number + FROM + sequence s + LEFT JOIN + existing_blocks e ON s.expected_block_number = e.block_number + WHERE + e.block_number = 0 + ORDER BY + missing_block_number + ` + rows, err := conn.Query(context.Background(), query, clickhouse.Named("chainId", chainId.String()), clickhouse.Named("startBlock", startBlock.String()), clickhouse.Named("endBlock", endBlock.String())) + if err != nil { + return nil, err + } + defer rows.Close() + + blockNumbers := make([]*big.Int, 0) + for rows.Next() { + var blockNumber *big.Int + err := rows.Scan(&blockNumber) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, blockNumber) + } + return blockNumbers, nil +} diff --git a/internal/validation/validation.go b/internal/validation/validation.go new file mode 100644 index 0000000..fe110b4 --- /dev/null +++ b/internal/validation/validation.go @@ -0,0 +1,389 @@ +package validation + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + "sort" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" + "github.com/rs/zerolog/log" + "github.com/thirdweb-dev/indexer/internal/orchestrator" + "github.com/thirdweb-dev/indexer/internal/rpc" + "github.com/thirdweb-dev/indexer/internal/storage" +) + +func ValidateAndFixBlocks(rpcClient rpc.IRPCClient, s storage.IStorage, conn clickhouse.Conn, startBlock *big.Int, endBlock *big.Int, fixBatchSize int) error { + dbData, err := FetchBlockDataFromDBForRange(conn, rpcClient.GetChainID(), startBlock, endBlock) + if err != nil { + return err + } + + invalidBlocks := make([]*big.Int, 0) + for _, blockData := range dbData { + err = VerifyBlock(blockData) + if err != nil { + log.Error().Err(err).Msgf("Block verification failed for block %s", blockData.Block.Number) + invalidBlocks = append(invalidBlocks, blockData.Block.Number) + } + } + if len(invalidBlocks) == 0 { + log.Debug().Msg("No invalid blocks found") + return nil + } + + if fixBatchSize == 0 { + fixBatchSize = len(invalidBlocks) + } + + // Process blocks in batches + for i := 0; i < len(invalidBlocks); i += fixBatchSize { + end := i + fixBatchSize + if end > len(invalidBlocks) { + end = len(invalidBlocks) + } + batch := invalidBlocks[i:end] + + log.Debug().Msgf("Processing batch of blocks %d to %d", invalidBlocks[i], invalidBlocks[end-1]) + + poller := orchestrator.NewBoundlessPoller(rpcClient, s) + poller.Poll(context.Background(), batch) + log.Debug().Msgf("Batch of invalid blocks polled: %d to %d", invalidBlocks[i], invalidBlocks[end-1]) + + blocksData, err := s.StagingStorage.GetStagingData(storage.QueryFilter{BlockNumbers: batch, ChainId: rpcClient.GetChainID()}) + if err != nil { + log.Fatal().Err(err).Msg("Failed to get staging data") + } + if len(blocksData) == 0 { + log.Fatal().Msg("Failed to get staging data") + } + + _, err = s.MainStorage.ReplaceBlockData(blocksData) + if err != nil { + log.Fatal().Err(err).Msgf("Failed to replace blocks: %v", blocksData) + } + + if err := s.StagingStorage.DeleteStagingData(blocksData); err != nil { + log.Error().Err(err).Msgf("Failed to delete staging data: %v", blocksData) + } + } + return nil +} + +func VerifyBlock(blockData BlockData) error { + if blockData.Block.TransactionCount != uint64(len(blockData.Transactions)) { + return fmt.Errorf("transaction count mismatch: expected=%d, fetched from DB=%d", blockData.Block.TransactionCount, len(blockData.Transactions)) + } + + // Calculate logsBloom from logs + calculatedLogsBloom, err := calculateLogsBloom(blockData.Logs) + if err != nil { + return fmt.Errorf("failed to calculate logsBloom: %v", err) + } + + // Compare calculated logsBloom with block's logsBloom + if calculatedLogsBloom != blockData.Block.LogsBloom { + return fmt.Errorf("logsBloom mismatch: calculated=%s, block=%s", calculatedLogsBloom, blockData.Block.LogsBloom) + } + + // TODO: remove this once we know how to validate all tx types + for _, tx := range blockData.Transactions { + if tx.TransactionType > 4 { // Currently supported types are 0-4 + log.Warn().Msgf("Skipping transaction root validation for block %s due to unsupported transaction type %d", blockData.Block.Number, tx.TransactionType) + return nil + } + } + + // Calculate transactionsRoot from transactions + calculatedTransactionsRoot, err := calculateTransactionsRoot(blockData.Transactions) + if err != nil { + return fmt.Errorf("failed to calculate transactionsRoot: %v", err) + } + + // Compare calculated transactionsRoot with block's transactionsRoot + if calculatedTransactionsRoot != blockData.Block.TransactionsRoot { + return fmt.Errorf("transactionsRoot mismatch: calculated=%s, block=%s", calculatedTransactionsRoot, blockData.Block.TransactionsRoot) + } + + return nil +} + +func calculateLogsBloom(logs []Log) (string, error) { + // Convert our logs to go-ethereum types.Log + ethLogs := make([]*types.Log, len(logs)) + for i, log := range logs { + // Convert address + addr := common.HexToAddress(log.Address) + + // Convert topics + topics := make([]common.Hash, 0, 4) + if log.Topic0 != "" { + topics = append(topics, common.HexToHash(log.Topic0)) + } + if log.Topic1 != "" { + topics = append(topics, common.HexToHash(log.Topic1)) + } + if log.Topic2 != "" { + topics = append(topics, common.HexToHash(log.Topic2)) + } + if log.Topic3 != "" { + topics = append(topics, common.HexToHash(log.Topic3)) + } + + ethLogs[i] = &types.Log{ + Address: addr, + Topics: topics, + } + } + + receipt := &types.Receipt{ + Logs: ethLogs, + } + // Create bloom filter using go-ethereum's implementation + bloom := types.CreateBloom(receipt) + return "0x" + hex.EncodeToString(bloom[:]), nil +} + +func calculateTransactionsRoot(transactions []Transaction) (string, error) { + // Sort transactions by transaction index + sort.Slice(transactions, func(i, j int) bool { + return transactions[i].TransactionIndex < transactions[j].TransactionIndex + }) + + // Convert our transactions to go-ethereum types.Transaction + ethTxs := make([]*types.Transaction, 0, len(transactions)) + for _, tx := range transactions { + // Convert to address + to := common.HexToAddress(tx.ToAddress) + isContractCreation := tx.ToAddress == "0x0000000000000000000000000000000000000000" && tx.Data != "0x" + + // Convert data - ensure we're using the full input data + data, err := hex.DecodeString(strings.TrimPrefix(tx.Data, "0x")) + if err != nil { + return "", fmt.Errorf("failed to decode transaction data: %v", err) + } + + // Create transaction based on type + var ethTx *types.Transaction + switch tx.TransactionType { + case 0: // Legacy transaction + // For legacy transactions, we need to set the signature values + if isContractCreation { + ethTx = types.NewContractCreation( + tx.Nonce, + tx.Value, + tx.Gas, + tx.GasPrice, + data, + ) + } else { + ethTx = types.NewTransaction( + tx.Nonce, + to, + tx.Value, + tx.Gas, + tx.GasPrice, + data, + ) + } + + // Set the signature values + if tx.V != nil && tx.R != nil && tx.S != nil { + v := tx.V.Uint64() + if v >= 35 { + // This is an EIP-155 transaction + signer := types.NewEIP155Signer(tx.ChainId) + sig := make([]byte, 65) + tx.R.FillBytes(sig[:32]) + tx.S.FillBytes(sig[32:64]) + // Calculate recovery_id from v + recovery_id := byte(v - (tx.ChainId.Uint64()*2 + 35)) + sig[64] = recovery_id + ethTx, err = ethTx.WithSignature(signer, sig) + if err != nil { + return "", fmt.Errorf("failed to set EIP-155 signature: %v", err) + } + } else { + // This is a legacy transaction + signer := types.HomesteadSigner{} + sig := make([]byte, 65) + tx.R.FillBytes(sig[:32]) + tx.S.FillBytes(sig[32:64]) + // Calculate recovery_id from v + recovery_id := byte(v - 27) + sig[64] = recovery_id + ethTx, err = ethTx.WithSignature(signer, sig) + if err != nil { + return "", fmt.Errorf("failed to set legacy signature: %v", err) + } + } + } + case 1: // Access list transaction + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + var toAddr *common.Address + if !isContractCreation { + toAddr = &to + } + + ethTx = types.NewTx(&types.AccessListTx{ + ChainID: tx.ChainId, + Nonce: tx.Nonce, + GasPrice: tx.GasPrice, + Gas: tx.Gas, + To: toAddr, + Value: tx.Value, + Data: data, + AccessList: accessList, + V: tx.V, + R: tx.R, + S: tx.S, + }) + case 2: // Dynamic fee transaction + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + var toAddr *common.Address + if !isContractCreation { + toAddr = &to + } + + ethTx = types.NewTx(&types.DynamicFeeTx{ + ChainID: tx.ChainId, + Nonce: tx.Nonce, + GasTipCap: tx.MaxPriorityFeePerGas, + GasFeeCap: tx.MaxFeePerGas, + Gas: tx.Gas, + To: toAddr, + Value: tx.Value, + Data: data, + AccessList: accessList, + V: tx.V, + R: tx.R, + S: tx.S, + }) + case 3: // Blob transaction + // Convert big.Int values to uint256.Int + chainID, _ := uint256.FromBig(tx.ChainId) + gasTipCap, _ := uint256.FromBig(tx.MaxPriorityFeePerGas) + gasFeeCap, _ := uint256.FromBig(tx.MaxFeePerGas) + value, _ := uint256.FromBig(tx.Value) + blobFeeCap, _ := uint256.FromBig(tx.MaxFeePerBlobGas) + v, _ := uint256.FromBig(tx.V) + r, _ := uint256.FromBig(tx.R) + s, _ := uint256.FromBig(tx.S) + + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + blobHashes := make([]common.Hash, len(tx.BlobVersionedHashes)) + for i, hash := range tx.BlobVersionedHashes { + blobHashes[i] = common.HexToHash(hash) + } + + var toAddr common.Address + if !isContractCreation { + toAddr = to + } + + ethTx = types.NewTx(&types.BlobTx{ + ChainID: chainID, + Nonce: tx.Nonce, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Gas: tx.Gas, + To: toAddr, + Value: value, + Data: data, + AccessList: accessList, + BlobFeeCap: blobFeeCap, + BlobHashes: blobHashes, + V: v, + R: r, + S: s, + }) + case 4: // EIP-7702 + // Convert big.Int values to uint256.Int + chainID, _ := uint256.FromBig(tx.ChainId) + gasTipCap, _ := uint256.FromBig(tx.MaxPriorityFeePerGas) + gasFeeCap, _ := uint256.FromBig(tx.MaxFeePerGas) + value, _ := uint256.FromBig(tx.Value) + v, _ := uint256.FromBig(tx.V) + r, _ := uint256.FromBig(tx.R) + s, _ := uint256.FromBig(tx.S) + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + var authList []types.SetCodeAuthorization + if tx.AuthorizationListJson != nil { + err = json.Unmarshal([]byte(*tx.AuthorizationListJson), &authList) + if err != nil { + return "", fmt.Errorf("failed to parse authorization list: %v", err) + } + } + + var toAddr common.Address + if !isContractCreation { + toAddr = to + } + + ethTx = types.NewTx(&types.SetCodeTx{ + ChainID: chainID, + Nonce: tx.Nonce, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Gas: tx.Gas, + To: toAddr, + Value: value, + Data: data, + AccessList: accessList, + AuthList: authList, + V: v, + R: r, + S: s, + }) + default: + return "", fmt.Errorf("unsupported transaction type: %d", tx.TransactionType) + } + + // Set the transaction hash + ethTxs = append(ethTxs, ethTx) + } + + // Create transactions root using go-ethereum's implementation + root := types.DeriveSha(types.Transactions(ethTxs), trie.NewStackTrie(nil)) + calculatedRoot := "0x" + hex.EncodeToString(root[:]) + return calculatedRoot, nil +}