feat: Blob Syncer

Makefile

@@ -0,0 +1,63 @@
+VERSION=$(shell git describe --tags)
+GIT_COMMIT=$(shell git rev-parse HEAD)
+GIT_COMMIT_DATE=$(shell git log -n1 --pretty='format:%cd' --date=format:'%Y%m%d')
+REPO=github.com/bnb-chain/blob-syncer
+IMAGE_NAME=ghcr.io/bnb-chain/blob-syncer
+
+ldflags = -X $(REPO)/version.AppVersion=$(VERSION) \
+          -X $(REPO)/version.GitCommit=$(GIT_COMMIT) \
+          -X $(REPO)/version.GitCommitDate=$(GIT_COMMIT_DATE)
+
+build_syncer:
+ifeq ($(OS),Windows_NT)
+	go build -o build/blob-syncer.exe -ldflags="$(ldflags)" main.go
+else
+	go build -o build/blob-syncer -ldflags="$(ldflags)" main.go
+endif
+
+build_server:
+ifeq ($(OS),Windows_NT)
+	go build $(BUILD_FLAGS) -o build/server.exe cmd/blob-syncer-server/main.go
+else
+	go build $(BUILD_FLAGS) -o build/server cmd/blob-syncer-server/main.go
+endif
+
+install:
+ifeq ($(OS),Windows_NT)
+	go install main.go
+else
+	go install main.go
+endif
+
+build_docker:
+	docker build . -t ${IMAGE_NAME}
+
+.PHONY: build install build_docker
+
+
+###############################################################################
+###                                Linting                                  ###
+###############################################################################
+
+golangci_lint_cmd=golangci-lint
+golangci_version=v1.51.2
+
+lint:
+	@echo "--> Running linter"
+	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)
+	@$(golangci_lint_cmd) run --timeout=10m
+
+lint-fix:
+	@echo "--> Running linter"
+	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(golangci_version)
+	@$(golangci_lint_cmd) run --fix --out-format=tab --issues-exit-code=0
+
+format:
+	bash scripts/format.sh
+
+.PHONY: lint lint-fix format
+
+swagger-gen:
+	swagger generate server -f ./swagger.yaml -A blob-syncer --default-scheme=http
+
+

README.md

@@ -1 +1,7 @@
-# blob-syncer
+./build/blob-syncer --config-type local --config-path config/config.json
+
+./build/server --config-type local --config-path config/config.json
+
+
+
+

config/config.go

@@ -0,0 +1,95 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+)
+
+type Config struct {
+	LogConfig    LogConfig    `json:"log_config"`
+	DBConfig     DBConfig     `json:"db_config"`
+	SyncerConfig SyncerConfig `json:"syncer_config"`
+}
+
+type SyncerConfig struct {
+	BucketName         string   `json:"bucket_name"`
+	StartHeight        uint64   `json:"start_height"`
+	BundleServiceAddrs []string `json:"bundle_service_addrs"`
+	BeaconAddrs        []string `json:"beacon_addrs"`
+	ETHRPCAddrs        []string `json:"eth_rpc_addrs"`
+	TempFilePath       string   `json:"temp_file_path"` // used to create file for every blob, bundle service might support sending stream.
+	PrivateKey         string   `json:"private_key"`
+}
+
+type DBConfig struct {
+	Dialect       string `json:"dialect"`
+	KeyType       string `json:"key_type"`
+	AWSRegion     string `json:"aws_region"`
+	AWSSecretName string `json:"aws_secret_name"`
+	Password      string `json:"password"`
+	Username      string `json:"username"`
+	Url           string `json:"url"`
+	MaxIdleConns  int    `json:"max_idle_conns"`
+	MaxOpenConns  int    `json:"max_open_conns"`
+}
+
+func (cfg *DBConfig) Validate() {
+	if cfg.Dialect != DBDialectMysql && cfg.Dialect != DBDialectSqlite3 {
+		panic(fmt.Sprintf("only %s and %s supported", DBDialectMysql, DBDialectSqlite3))
+	}
+	if cfg.Dialect == DBDialectMysql && (cfg.Username == "" || cfg.Url == "") {
+		panic("db config is not correct, missing username and/or url")
+	}
+	if cfg.MaxIdleConns == 0 || cfg.MaxOpenConns == 0 {
+		panic("db connections is not correct")
+	}
+}
+
+type LogConfig struct {
+	Level                        string `json:"level"`
+	Filename                     string `json:"filename"`
+	MaxFileSizeInMB              int    `json:"max_file_size_in_mb"`
+	MaxBackupsOfLogFiles         int    `json:"max_backups_of_log_files"`
+	MaxAgeToRetainLogFilesInDays int    `json:"max_age_to_retain_log_files_in_days"`
+	UseConsoleLogger             bool   `json:"use_console_logger"`
+	UseFileLogger                bool   `json:"use_file_logger"`
+	Compress                     bool   `json:"compress"`
+}
+
+func (cfg *LogConfig) Validate() {
+	if cfg.UseFileLogger {
+		if cfg.Filename == "" {
+			panic("filename should not be empty if use file logger")
+		}
+		if cfg.MaxFileSizeInMB <= 0 {
+			panic("max_file_size_in_mb should be larger than 0 if use file logger")
+		}
+		if cfg.MaxBackupsOfLogFiles <= 0 {
+			panic("max_backups_off_log_files should be larger than 0 if use file logger")
+		}
+	}
+}
+
+func ParseConfigFromJson(content string) *Config {
+	var config Config
+	if err := json.Unmarshal([]byte(content), &config); err != nil {
+		panic(err)
+	}
+	//config.Validate()
+	return &config
+}
+
+func ParseConfigFromFile(filePath string) *Config {
+	bz, err := os.ReadFile(filePath)
+	if err != nil {
+		panic(err)
+	}
+
+	var config Config
+	if err := json.Unmarshal(bz, &config); err != nil {
+		panic(err)
+	}
+	//config.Validate()
+	return &config
+}

config/config.json

@@ -0,0 +1,46 @@
+{
+  "syncer_config": {
+    "bucket_name": "bsc-blobs",
+    "start_height": 8775600,
+    "beacon_addrs": [
+      "https://eth2-beacon-mainnet.nodereal.io/v1/a24f28a0f2484effa9fea36b8e281272"
+    ],
+    "bundle_service_addrs": [
+      "https://gnfd-testnet-bundle.nodereal.io"
+    ],
+    "eth_rpc_addrs": [
+      "https://eth-mainnet.nodereal.io/v1/a24f28a0f2484effa9fea36b8e281272"
+    ],
+    "temp_file_path": "blob-store",
+    "private_key": "d65f7cf21fe3eff9feef1dd86cea6bae8a30f6c26830e734d628c78e80debfd5"
+  },
+  "log_config": {
+    "level": "DEBUG",
+    "filename": "",
+    "max_file_size_in_mb": 0,
+    "max_backups_of_log_files": 0,
+    "max_age_to_retain_log_files_in_days": 0,
+    "use_console_logger": true,
+    "use_file_logger": false,
+    "compress": false
+  },
+  "admin_config": {
+    "port": 8080
+  },
+  "db_config": {
+    "dialect": "mysql",
+    "key_type": "local_private_key",
+    "aws_region": "",
+    "aws_secret_name": "",
+    "password": "pass",
+    "username": "root",
+    "url": "/local-blob-syncer?charset=utf8&parseTime=True&loc=Local",
+    "max_idle_conns": 10,
+    "max_open_conns": 100
+  },
+  "alert_config": {
+    "identity": "your_service_name",
+    "telegram_bot_id": "your_bot_id",
+    "telegram_chat_id": "your_chat_id"
+  }
+}

config/const.go

@@ -0,0 +1,23 @@
+package config
+
+const (
+	FlagConfigPath          = "config-path"
+	FlagConfigType          = "config-type"
+	FlagConfigAwsRegion     = "aws-region"
+	FlagConfigAwsSecretKey  = "aws-secret-key"
+	FlagConfigPrivateKey    = "private-key"
+	FlagConfigBlsPrivateKey = "bls-private-key"
+	FlagConfigDbPass        = "db-pass"
+
+	DBDialectMysql   = "mysql"
+	DBDialectSqlite3 = "sqlite3"
+
+	LocalConfig            = "local"
+	AWSConfig              = "aws"
+	KeyTypeLocalPrivateKey = "local_private_key"
+	KeyTypeAWSPrivateKey   = "aws_private_key"
+
+	ConfigType     = "CONFIG_TYPE"
+	ConfigFilePath = "CONFIG_FILE_PATH"
+	ConfigDBPass   = "DB_PASS"
+)

config/utils.go

@@ -0,0 +1,45 @@
+package config
+
+import (
+	"encoding/base64"
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/secretsmanager"
+)
+
+func GetSecret(secretName, region string) (string, error) {
+	// CreateBlock a Secrets Manager client
+	sess, err := session.NewSession(&aws.Config{
+		Region: &region,
+	})
+	if err != nil {
+		return "", err
+	}
+
+	svc := secretsmanager.New(sess)
+	input := &secretsmanager.GetSecretValueInput{
+		SecretId:     aws.String(secretName),
+		VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified
+	}
+
+	result, err := svc.GetSecretValue(input)
+	if err != nil {
+		return "", err
+	}
+
+	var secretString, decodedBinarySecret string
+	if result.SecretString != nil {
+		secretString = *result.SecretString
+		return secretString, nil
+	} else {
+		decodedBinarySecretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(result.SecretBinary)))
+		length, err := base64.StdEncoding.Decode(decodedBinarySecretBytes, result.SecretBinary)
+		if err != nil {
+			fmt.Println("Base64 Decode Error:", err)
+			return "", err
+		}
+		decodedBinarySecret = string(decodedBinarySecretBytes[:length])
+		return decodedBinarySecret, nil
+	}
+}