diff --git a/client/blob_server.go b/client/blob_server.go index 1baf920..54170b5 100644 --- a/client/blob_server.go +++ b/client/blob_server.go @@ -5,12 +5,13 @@ import ( "encoding/hex" "fmt" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/bnb-chain/blob-hub/models" blobproto "github.com/bnb-chain/blob-hub/proto" "github.com/bnb-chain/blob-hub/service" "github.com/bnb-chain/blob-hub/types" "github.com/bnb-chain/blob-hub/util" - "github.com/ethereum/go-ethereum/common/hexutil" ) type BlobServer struct { @@ -53,11 +54,11 @@ func (s *BlobServer) GetBlobSidecars(ctx context.Context, req *blobproto.GetBlob return nil, err } } else { - slot, err := util.StringToUint64(blockID) + blockNumOrSlot, err := util.StringToUint64(blockID) if err != nil { return nil, err } - sidecars, err = service.BlobSvc.GetBlobSidecarsBySlot(slot, indicesInx) + sidecars, err = service.BlobSvc.GetBlobSidecarsByBlockNumOrSlot(blockNumOrSlot, indicesInx) if err != nil { return nil, err } diff --git a/cmd/blob-hub-syncer/main.go b/cmd/blob-hub-syncer/main.go index ce9696c..65f9f30 100644 --- a/cmd/blob-hub-syncer/main.go +++ b/cmd/blob-hub-syncer/main.go @@ -34,7 +34,6 @@ func main() { if configFilePath == "" { configFilePath = os.Getenv(config.EnvVarConfigFilePath) } - //configFilePath = "config/local/bsc-config-syncer.json" cfg = config.ParseSyncerConfigFromFile(configFilePath) if cfg == nil { panic("failed to get configuration") diff --git a/config/config.go b/config/config.go index a963a4d..612dfa6 100644 --- a/config/config.go +++ b/config/config.go @@ -62,8 +62,11 @@ func (s *SyncerConfig) Validate() { if len(s.PrivateKey) == 0 { panic("private key is not provided") } - if s.CreateBundleSlotOrBlockInterval > 100 { - panic("create_bundle_slot_interval is supposed less than 100") + if s.Chain == BSC && s.CreateBundleSlotOrBlockInterval > 200 { + panic("create_bundle_slot_interval is supposed to be less than 100") + } + if s.Chain == ETH && s.CreateBundleSlotOrBlockInterval > 30 { + panic("create_bundle_slot_interval is supposed to be less than 30") } if s.BundleNotSealedReuploadThreshold <= 60 { panic("Bundle_not_sealed_reupload_threshold is supposed larger than 60 (s)") @@ -87,6 +90,7 @@ func (s *SyncerConfig) GetReUploadBundleThresh() int64 { } type ServerConfig struct { + Chain string `json:"chain"` BucketName string `json:"bucket_name"` BundleServiceEndpoints []string `json:"bundle_service_endpoints"` // BundleServiceEndpoints is a list of bundle service address CacheConfig CacheConfig `json:"cache_config"` @@ -94,6 +98,9 @@ type ServerConfig struct { } func (s *ServerConfig) Validate() { + if !strings.EqualFold(s.Chain, ETH) && !strings.EqualFold(s.Chain, BSC) { + panic("chain not support") + } if len(s.BucketName) == 0 { panic("the Greenfield bucket name is not is not provided") } diff --git a/external/client.go b/external/client.go index dbef7a3..e71135c 100644 --- a/external/client.go +++ b/external/client.go @@ -5,13 +5,14 @@ import ( "math/big" "strconv" - "github.com/bnb-chain/blob-hub/config" - "github.com/bnb-chain/blob-hub/external/eth" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/prysmaticlabs/prysm/v5/api/server/structs" + + "github.com/bnb-chain/blob-hub/config" + "github.com/bnb-chain/blob-hub/external/eth" ) const BSCBlockConfirmNum = 3 diff --git a/models/sidecar.go b/models/sidecar.go index 788d69e..ed4698c 100644 --- a/models/sidecar.go +++ b/models/sidecar.go @@ -29,7 +29,7 @@ type Sidecar struct { KzgCommitment string `json:"kzg_commitment,omitempty"` // kzg commitment inclusion proof - KzgCommitmentInclusionProof []string `json:"kzg_commitment_inclusion_proof"` + KzgCommitmentInclusionProof []string `json:"kzg_commitment_inclusion_proof,omitempty"` // kzg proof KzgProof string `json:"kzg_proof,omitempty"` diff --git a/restapi/configure_blob_hub.go b/restapi/configure_blob_hub.go index 0e78cf1..20502fa 100644 --- a/restapi/configure_blob_hub.go +++ b/restapi/configure_blob_hub.go @@ -11,24 +11,22 @@ import ( "net/http" "os" - "google.golang.org/grpc" - - "github.com/bnb-chain/blob-hub/client" - "github.com/bnb-chain/blob-hub/external/cmn" "github.com/go-openapi/errors" "github.com/go-openapi/runtime" "github.com/go-openapi/swag" + grpcruntime "github.com/grpc-ecosystem/grpc-gateway/runtime" + "google.golang.org/grpc" "github.com/bnb-chain/blob-hub/cache" + "github.com/bnb-chain/blob-hub/client" "github.com/bnb-chain/blob-hub/config" syncerdb "github.com/bnb-chain/blob-hub/db" + "github.com/bnb-chain/blob-hub/external/cmn" blobproto "github.com/bnb-chain/blob-hub/proto" "github.com/bnb-chain/blob-hub/restapi/handlers" "github.com/bnb-chain/blob-hub/restapi/operations" "github.com/bnb-chain/blob-hub/restapi/operations/blob" "github.com/bnb-chain/blob-hub/service" - - grpcruntime "github.com/grpc-ecosystem/grpc-gateway/runtime" ) //go:generate swagger generate server --target ../../blob-syncer --name BlobHub --spec ../swagger.yaml --principal interface{} diff --git a/restapi/embedded_spec.go b/restapi/embedded_spec.go index f12e85d..c8d0570 100644 --- a/restapi/embedded_spec.go +++ b/restapi/embedded_spec.go @@ -146,7 +146,8 @@ func init() { "type": "array", "items": { "type": "string" - } + }, + "x-omitempty": true }, "kzg_proof": { "type": "string" @@ -312,7 +313,8 @@ func init() { "type": "array", "items": { "type": "string" - } + }, + "x-omitempty": true }, "kzg_proof": { "type": "string" diff --git a/restapi/handlers/blob.go b/restapi/handlers/blob.go index 52c80a2..7bccf13 100644 --- a/restapi/handlers/blob.go +++ b/restapi/handlers/blob.go @@ -4,13 +4,13 @@ import ( "encoding/hex" "fmt" - "github.com/bnb-chain/blob-hub/types" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/go-openapi/runtime/middleware" "github.com/bnb-chain/blob-hub/models" "github.com/bnb-chain/blob-hub/restapi/operations/blob" "github.com/bnb-chain/blob-hub/service" + "github.com/bnb-chain/blob-hub/types" "github.com/bnb-chain/blob-hub/util" ) @@ -52,7 +52,7 @@ func HandleGetBlobSidecars() func(params blob.GetBlobSidecarsByBlockNumParams) m if err != nil { return blob.NewGetBlobSidecarsByBlockNumBadRequest().WithPayload(service.BadRequestWithError(err)) } - sidecars, err = service.BlobSvc.GetBlobSidecarsBySlot(slot, indicesInx) + sidecars, err = service.BlobSvc.GetBlobSidecarsByBlockNumOrSlot(slot, indicesInx) if err != nil { return blob.NewGetBlobSidecarsByBlockNumInternalServerError().WithPayload(service.InternalErrorWithError(err)) } diff --git a/scripts/.env b/scripts/.env index b1728d6..2c00dd4 100644 --- a/scripts/.env +++ b/scripts/.env @@ -1,7 +1,7 @@ PRIVATE_KEY=... -BUCKET_NAME=bsc-testnet-blobs -GRANTEE_BUNDLE_ACCOUNT=0x4605BFc98E0a5EA63D9D5a4a1Df549732a6963f3 +BUCKET_NAME=... +GRANTEE_BUNDLE_ACCOUNT=... GREENFIELD_RPC=https://gnfd-testnet-fullnode-tendermint-us.bnbchain.org:443 GREENFIELD_CHAIN_ID=greenfield_5600-1 ALLOWANCE=10000000000000000000 -SP_ADDRESS=0x5fff5a6c94b182fb965b40c7b9f30199b969ed2f \ No newline at end of file +SP_ADDRESS=... \ No newline at end of file diff --git a/service/blob.go b/service/blob.go index 6b0b178..94453dc 100644 --- a/service/blob.go +++ b/service/blob.go @@ -15,14 +15,14 @@ const prefixHex = "0x" type Blob interface { GetBlobSidecarsByRoot(root string, indices []int64) ([]*models.Sidecar, error) - GetBlobSidecarsBySlot(slot uint64, indices []int64) ([]*models.Sidecar, error) + GetBlobSidecarsByBlockNumOrSlot(slot uint64, indices []int64) ([]*models.Sidecar, error) } type BlobService struct { blobDB db.BlobDao bundleClient *cmn.BundleClient cacheService cache.Cache - config *config.ServerConfig + cfg *config.ServerConfig } func NewBlobService(blobDB db.BlobDao, bundleClient *cmn.BundleClient, cache cache.Cache, config *config.ServerConfig) Blob { @@ -30,20 +30,20 @@ func NewBlobService(blobDB db.BlobDao, bundleClient *cmn.BundleClient, cache cac blobDB: blobDB, bundleClient: bundleClient, cacheService: cache, - config: config, + cfg: config, } } -func (b BlobService) GetBlobSidecarsBySlot(slot uint64, indices []int64) ([]*models.Sidecar, error) { +func (b BlobService) GetBlobSidecarsByBlockNumOrSlot(blockNumOrSlot uint64, indices []int64) ([]*models.Sidecar, error) { var err error - blobs, found := b.cacheService.Get(util.Uint64ToString(slot)) + blobs, found := b.cacheService.Get(util.Uint64ToString(blockNumOrSlot)) if found { blobsFound := blobs.([]*models.Sidecar) if len(indices) != 0 { blobReturn := make([]*models.Sidecar, 0) for _, idx := range indices { if int(idx) >= len(blobsFound) { - return nil, fmt.Errorf("index %d out of bound, only %d blob at slot %d", idx, len(blobsFound), slot) + return nil, fmt.Errorf("index %d out of bound, only %d blob at block %d", idx, len(blobsFound), blockNumOrSlot) } blobReturn = append(blobReturn, blobsFound[idx]) } @@ -52,19 +52,19 @@ func (b BlobService) GetBlobSidecarsBySlot(slot uint64, indices []int64) ([]*mod return blobsFound, nil } - block, err := b.blobDB.GetBlock(slot) + block, err := b.blobDB.GetBlock(blockNumOrSlot) if err != nil { return nil, err } var blobMetas []*db.Blob if len(indices) == 0 { - blobMetas, err = b.blobDB.GetBlobByBlockID(slot) + blobMetas, err = b.blobDB.GetBlobByBlockID(blockNumOrSlot) if err != nil { return nil, err } } else { - blobMetas, err = b.blobDB.GetBlobByBlockIDAndIndices(slot, indices) + blobMetas, err = b.blobDB.GetBlobByBlockIDAndIndices(blockNumOrSlot, indices) if err != nil { return nil, err } @@ -72,19 +72,22 @@ func (b BlobService) GetBlobSidecarsBySlot(slot uint64, indices []int64) ([]*mod sideCars := make([]*models.Sidecar, 0) for _, meta := range blobMetas { - bundleObject, err := b.bundleClient.GetObject(b.config.BucketName, block.BundleName, meta.Name) + bundleObject, err := b.bundleClient.GetObject(b.cfg.BucketName, block.BundleName, meta.Name) if err != nil { return nil, err } - header := &models.SidecarSignedBlockHeader{ - Message: &models.SidecarSignedBlockHeaderMessage{ - BodyRoot: fmt.Sprintf("%s%s", prefixHex, block.BodyRoot), - ParentRoot: fmt.Sprintf("%s%s", prefixHex, block.ParentRoot), - StateRoot: fmt.Sprintf("%s%s", prefixHex, block.StateRoot), - ProposerIndex: util.Uint64ToString(block.ProposerIndex), - Slot: util.Uint64ToString(block.Slot), - }, - Signature: fmt.Sprintf("%s%s", prefixHex, block.Signature), + var header *models.SidecarSignedBlockHeader + if b.cfg.Chain == config.ETH { + header = &models.SidecarSignedBlockHeader{ + Message: &models.SidecarSignedBlockHeaderMessage{ + BodyRoot: fmt.Sprintf("%s%s", prefixHex, block.BodyRoot), + ParentRoot: fmt.Sprintf("%s%s", prefixHex, block.ParentRoot), + StateRoot: fmt.Sprintf("%s%s", prefixHex, block.StateRoot), + ProposerIndex: util.Uint64ToString(block.ProposerIndex), + Slot: util.Uint64ToString(block.Slot), + }, + Signature: fmt.Sprintf("%s%s", prefixHex, block.Signature), + } } sideCars = append(sideCars, &models.Sidecar{ @@ -99,7 +102,7 @@ func (b BlobService) GetBlobSidecarsBySlot(slot uint64, indices []int64) ([]*mod // cache all blobs at a specified slot if len(indices) == 0 { - b.cacheService.Set(util.Uint64ToString(slot), sideCars) + b.cacheService.Set(util.Uint64ToString(blockNumOrSlot), sideCars) } return sideCars, nil } @@ -109,5 +112,5 @@ func (b BlobService) GetBlobSidecarsByRoot(root string, indices []int64) ([]*mod if err != nil { return nil, err } - return b.GetBlobSidecarsBySlot(block.Slot, indices) + return b.GetBlobSidecarsByBlockNumOrSlot(block.Slot, indices) } diff --git a/swagger.yaml b/swagger.yaml index 8dbd557..38d4c5c 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -79,6 +79,7 @@ definitions: type: string kzg_commitment_inclusion_proof: type: array + x-omitempty: true items: type: string signed_block_header: diff --git a/syncer/syncer.go b/syncer/syncer.go index a301f22..348cad4 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -11,24 +11,23 @@ import ( "strings" "time" - "gorm.io/gorm" - - "github.com/bnb-chain/blob-hub/external" - "github.com/bnb-chain/blob-hub/external/cmn" - "github.com/bnb-chain/blob-hub/external/eth" "github.com/ethereum/go-ethereum/common/hexutil" + ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/prysmaticlabs/prysm/v5/api/server/structs" v1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1" ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1" "github.com/prysmaticlabs/prysm/v5/runtime/version" + "gorm.io/gorm" "github.com/bnb-chain/blob-hub/config" "github.com/bnb-chain/blob-hub/db" + "github.com/bnb-chain/blob-hub/external" + "github.com/bnb-chain/blob-hub/external/cmn" + "github.com/bnb-chain/blob-hub/external/eth" "github.com/bnb-chain/blob-hub/logging" "github.com/bnb-chain/blob-hub/metrics" "github.com/bnb-chain/blob-hub/types" "github.com/bnb-chain/blob-hub/util" - ethtypes "github.com/ethereum/go-ethereum/core/types" ) const ( @@ -463,6 +462,7 @@ func (s *BlobSyncer) toBlockAndBlobs(blockResp *structs.GetBlockV2Response, side logging.Logger.Errorf("failed to get header, slot=%d, err=%s", blockNumOrSlot, err.Error()) return nil, nil, err } + rootBz, err := hexutil.Decode(header.Data.Root) if err != nil { logging.Logger.Errorf("failed to decode header.Data.Root=%s, err=%s", header.Data.Root, err.Error()) diff --git a/syncer/verifier.go b/syncer/verifier.go index 000deb4..a796515 100644 --- a/syncer/verifier.go +++ b/syncer/verifier.go @@ -9,19 +9,20 @@ import ( "path/filepath" "time" + "github.com/prysmaticlabs/prysm/v5/api/server/structs" "gorm.io/gorm" + "github.com/bnb-chain/blob-hub/db" "github.com/bnb-chain/blob-hub/external/cmn" "github.com/bnb-chain/blob-hub/external/eth" - "github.com/prysmaticlabs/prysm/v5/api/server/structs" - - "github.com/bnb-chain/blob-hub/db" "github.com/bnb-chain/blob-hub/logging" "github.com/bnb-chain/blob-hub/metrics" "github.com/bnb-chain/blob-hub/types" "github.com/bnb-chain/blob-hub/util" ) +const VerifyPauseTime = 90 * time.Second + var ( ErrVerificationFailed = errors.New("verification failed") ) @@ -34,21 +35,11 @@ var ( // // a new bundle should be re-uploaded. func (s *BlobSyncer) verify() error { - var ( - err error - sleepTime time.Duration - ) - if s.BSCChain() { - sleepTime = BSCPauseTime - } else { - sleepTime = ETHPauseTime - } - verifyBlock, err := s.blobDao.GetEarliestUnverifiedBlock() if err != nil { if err == gorm.ErrRecordNotFound { logging.Logger.Debugf("found no unverified block in DB") - time.Sleep(sleepTime) + time.Sleep(VerifyPauseTime) return nil } return err @@ -67,7 +58,7 @@ func (s *BlobSyncer) verify() error { } if bundle.Status == db.Finalizing { logging.Logger.Debugf("the bundle has not been submitted to bundle service yet, bundleName=%s", bundleName) - time.Sleep(sleepTime) + time.Sleep(VerifyPauseTime) return nil } @@ -100,6 +91,7 @@ func (s *BlobSyncer) verify() error { // the bundle is not sealed yet if bundleInfo.Status == BundleStatusFinalized || bundleInfo.Status == BundleStatusCreatedOnChain { if bundle.CreatedTime > 0 && time.Now().Unix()-bundle.CreatedTime > s.config.GetReUploadBundleThresh() { + logging.Logger.Infof("the bundle %s is not sealed and exceed the re-upload threshold %d ", bundleName, s.config.GetReUploadBundleThresh()) return s.reUploadBundle(bundleName) } return nil