Bring back ETL and explorer

.gitignore

@@ -10,3 +10,9 @@ tmp
 .release_notes.md
 
 contracts/build
+
+# Node.js dependencies
+node_modules/
+**/node_modules/
+package-lock.json
+**/package-lock.json

Makefile

@@ -17,7 +17,10 @@ CORE_SQL_ARTIFACTS := $(wildcard pkg/core/db/*.sql.go)
 ETH_SQL_SRCS := $(shell find pkg/eth/db/sql -type f -name '*.sql') pkg/eth/db/sqlc.yaml
 ETH_SQL_ARTIFACTS := $(wildcard pkg/eth/db/*.sql.go)
 
-SQL_ARTIFACTS := $(CORE_SQL_ARTIFACTS) $(ETH_SQL_ARTIFACTS)
+ETL_SQL_SRCS := $(shell find pkg/etl/db/sql -type f -name '*.sql') pkg/etl/db/sqlc.yaml
+ETL_SQL_ARTIFACTS := $(wildcard pkg/etl/db/*.sql.go)
+
+SQL_ARTIFACTS := $(CORE_SQL_ARTIFACTS) $(ETH_SQL_ARTIFACTS) $(ETL_SQL_ARTIFACTS)
 
 ###### PROTO
 PROTO_SRCS := $(shell find proto -type f -name '*.proto')
@@ -27,6 +30,12 @@ PROTO_ARTIFACTS := $(shell find pkg/api -type f -name '*.pb.go')
 TEMPL_SRCS := $(shell find pkg/core/console -type f -name "*.templ")
 TEMPL_ARTIFACTS := $(shell find pkg/core/console -type f -name "*_templ.go")
 
+EXPLORER_TEMPL_SRCS := $(shell find pkg/console/templates -type f -name "*.templ")
+EXPLORER_TEMPL_ARTIFACTS := $(shell find pkg/console/templates -type f -name "*_templ.go")
+
+###### CSS
+EXPLORER_CSS_INPUT := pkg/console/assets/input.css
+EXPLORER_CSS_OUTPUT := pkg/console/assets/css/output.css
 
 ###### CODE
 JSON_SRCS := $(wildcard pkg/core/config/genesis/*.json)
@@ -92,12 +101,29 @@ go.mod: $(GO_SRCS)
 gen: regen-templ regen-proto regen-sql
 
 .PHONY: regen-templ
-regen-templ: $(TEMPL_ARTIFACTS)
+regen-templ: $(TEMPL_ARTIFACTS) $(EXPLORER_TEMPL_ARTIFACTS) regen-css
 
 $(TEMPL_ARTIFACTS): $(TEMPL_SRCS)
-	@echo Regenerating templ code
+	@echo Regenerating console templ code
 	cd pkg/core/console && templ generate -log-level error
 
+$(EXPLORER_TEMPL_ARTIFACTS): $(EXPLORER_TEMPL_SRCS)
+	@echo Regenerating explorer templ code
+	cd pkg/console/templates && templ generate -log-level error
+	@touch pkg/console/templates/layouts/frame.templ 2>/dev/null || touch pkg/console/console.go 2>/dev/null || true
+
+.PHONY: regen-css
+regen-css: $(EXPLORER_CSS_OUTPUT)
+
+$(EXPLORER_CSS_OUTPUT): $(EXPLORER_CSS_INPUT) $(EXPLORER_TEMPL_SRCS)
+	@echo Regenerating explorer CSS
+	@cd pkg/console/assets && \
+		(npm list @tailwindcss/postcss > /dev/null 2>&1 || npm install --no-save @tailwindcss/postcss postcss-cli > /dev/null 2>&1) && \
+		npx postcss input.css -o css/output.css --minify || \
+		(echo "Error: Failed to regenerate CSS. You may need to run manually:" && \
+		 echo "  cd pkg/console/assets && npm install && npx postcss input.css -o css/output.css" && exit 1)
+	@touch pkg/console/templates/layouts/frame.templ 2>/dev/null || touch pkg/console/console.go 2>/dev/null || true
+
 .PHONY: regen-proto
 regen-proto: $(PROTO_ARTIFACTS)
 
@@ -107,7 +133,7 @@ $(PROTO_ARTIFACTS): $(PROTO_SRCS)
 	buf generate
 
 .PHONY: regen-sql
-regen-sql: regen-core-sql regen-eth-sql
+regen-sql: regen-core-sql regen-eth-sql regen-etl-sql
 
 .PHONY: regen-core-sql
 regen-core-sql: $(CORE_SQL_ARTIFACTS)
@@ -123,6 +149,13 @@ $(ETH_SQL_ARTIFACTS): $(ETH_SQL_SRCS)
 	@echo Regenerating eth sql code
 	cd pkg/eth/db && sqlc generate
 
+.PHONY: regen-etl-sql
+regen-etl-sql: $(ETL_SQL_ARTIFACTS)
+
+$(ETL_SQL_ARTIFACTS): $(ETL_SQL_SRCS)
+	@echo Regenerating etl sql code
+	cd pkg/etl/db && sqlc generate
+
 .PHONY: regen-contracts
 regen-contracts:
 	@echo Regenerating contracts

cmd/openaudio/main.go

@@ -28,10 +28,12 @@ import (
 
 	"connectrpc.com/connect"
 	"github.com/OpenAudio/go-openaudio/pkg/common"
+	"github.com/OpenAudio/go-openaudio/pkg/console"
 	"github.com/OpenAudio/go-openaudio/pkg/core"
 	"github.com/OpenAudio/go-openaudio/pkg/core/config"
 	coreServer "github.com/OpenAudio/go-openaudio/pkg/core/server"
 	"github.com/OpenAudio/go-openaudio/pkg/eth"
+	"github.com/OpenAudio/go-openaudio/pkg/etl"
 	"github.com/OpenAudio/go-openaudio/pkg/lifecycle"
 	aLogger "github.com/OpenAudio/go-openaudio/pkg/logger"
 	"github.com/OpenAudio/go-openaudio/pkg/mediorum"
@@ -51,6 +53,7 @@ import (
 	v1 "github.com/OpenAudio/go-openaudio/pkg/api/core/v1"
 	corev1connect "github.com/OpenAudio/go-openaudio/pkg/api/core/v1/v1connect"
 	ethv1connect "github.com/OpenAudio/go-openaudio/pkg/api/eth/v1/v1connect"
+	etlv1connect "github.com/OpenAudio/go-openaudio/pkg/api/etl/v1/v1connect"
 	storagev1connect "github.com/OpenAudio/go-openaudio/pkg/api/storage/v1/v1connect"
 	systemv1connect "github.com/OpenAudio/go-openaudio/pkg/api/system/v1/v1connect"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -110,13 +113,42 @@ func main() {
 
 	setupDelegateKeyPair(rootLogger)
 
+	// Read config to check feature flags
+	cfg, err := config.ReadConfig()
+	if err != nil {
+		panic(fmt.Sprintf("failed to read config: %v", err))
+	}
+
 	ethService := eth.NewEthService(dbUrl, config.GetEthRPC(), config.GetRegistryAddress(), rootLogger, config.GetRuntimeEnvironment())
 	coreService := coreServer.NewCoreService()
 	storageService := server.NewStorageService()
 	// Only set storage service on core if storage is enabled
 	if isStorageEnabled() {
 		coreService.SetStorageService(storageService)
 	}
+
+	// Initialize ETL service if enabled
+	var etlService *etl.ETLService
+	if cfg.EnableETL {
+		// Create an HTTP client that will connect to the local service
+		// The ETL service needs a CoreServiceClient, so we create one that connects via HTTP
+		httpClient := &http.Client{
+			Timeout: 30 * time.Second,
+		}
+		// Determine the base URL - we'll use localhost with the port from hostUrl
+		baseURL := fmt.Sprintf("http://localhost")
+		if hostUrl.Port() != "" {
+			baseURL = fmt.Sprintf("http://localhost:%s", hostUrl.Port())
+		} else {
+			// Default to port 80 if no port specified
+			baseURL = "http://localhost:80"
+		}
+		coreClient := corev1connect.NewCoreServiceClient(httpClient, baseURL, connectJSONOpt)
+		etlService = etl.NewETLService(coreClient, rootLogger)
+		etlService.SetDBURL(dbUrl)
+		etlService.SetCheckReadiness(false) // Don't wait for core to be ready when console is enabled
+	}
+
 	systemService := system.NewSystemService(coreService, storageService)
 
 	services := []struct {
@@ -127,7 +159,7 @@ func main() {
 		{
 			"audiusd-echo-server",
 			func() error {
-				return startEchoProxy(hostUrl, rootLogger, coreService, storageService, systemService, ethService)
+				return startEchoProxy(hostUrl, rootLogger, coreService, storageService, systemService, ethService, etlService, cfg)
 			},
 			true,
 		},
@@ -153,6 +185,17 @@ func main() {
 			func() error { return ethService.Run(ctx) },
 			true,
 		},
+		{
+			"etl",
+			func() error {
+				if etlService == nil {
+					return nil
+				}
+				etlService.SetRunDownMigrations(os.Getenv("OPENAUDIO_ETL_RUN_DOWN_MIGRATIONS") == "true")
+				return etlService.Run()
+			},
+			cfg.EnableETL,
+		},
 	}
 
 	for _, svc := range services {
@@ -489,10 +532,24 @@ func setProtobufField(msgReflect protoreflect.Message, field protoreflect.FieldD
 	return nil
 }
 
-func startEchoProxy(hostUrl *url.URL, logger *zap.Logger, coreService *coreServer.CoreService, storageService *server.StorageService, systemService *system.SystemService, ethService *eth.EthService) error {
+func startEchoProxy(hostUrl *url.URL, logger *zap.Logger, coreService *coreServer.CoreService, storageService *server.StorageService, systemService *system.SystemService, ethService *eth.EthService, etlService *etl.ETLService, cfg *config.Config) error {
+	// Explorer console requires ETL to be enabled
+	consoleEnabled := cfg.EnableExplorer && cfg.EnableETL && etlService != nil
 	e := echo.New()
 	e.HideBanner = true
-	e.Use(middleware.Logger(), middleware.Recover(), common.InjectRealIP())
+
+	// Configure logger to skip internal ETL requests
+	loggerConfig := middleware.LoggerConfig{
+		Skipper: func(c echo.Context) bool {
+			// Skip logging for internal ETL requests to Core service
+			path := c.Request().URL.Path
+			if strings.Contains(path, "/core.v1.CoreService/GetBlock") {
+				return true
+			}
+			return false
+		},
+	}
+	e.Use(middleware.LoggerWithConfig(loggerConfig), middleware.Recover(), common.InjectRealIP())
 
 	rpcGroup := e.Group("")
 	rpcGroup.Use(common.CORS())
@@ -508,6 +565,18 @@ func startEchoProxy(hostUrl *url.URL, logger *zap.Logger, coreService *coreServe
 	ethPath, ethHandler := ethv1connect.NewEthServiceHandler(ethService, connectJSONOpt)
 	rpcGroup.POST(ethPath+"*", echo.WrapHandler(ethHandler))
 
+	// Register ETL routes if enabled
+	if cfg.EnableETL && etlService != nil {
+		etlPath, etlHandler := etlv1connect.NewETLServiceHandler(etlService, connectJSONOpt)
+		rpcGroup.POST(etlPath+"*", echo.WrapHandler(etlHandler))
+	}
+
+	// Initialize explorer console if enabled
+	if consoleEnabled {
+		c := console.NewConsole(etlService, e, cfg.Environment)
+		c.Initialize()
+	}
+
 	// register GET routes
 
 	// core GET routes
@@ -544,6 +613,10 @@ func startEchoProxy(hostUrl *url.URL, logger *zap.Logger, coreService *coreServe
 		grpcServerGroup.Any(storagePath+"*", echo.WrapHandler(storageHandler))
 		grpcServerGroup.Any(systemPath+"*", echo.WrapHandler(systemHandler))
 		grpcServerGroup.Any(ethPath+"*", echo.WrapHandler(ethHandler))
+		if cfg.EnableETL && etlService != nil {
+			etlPath, etlHandler := etlv1connect.NewETLServiceHandler(etlService, connectJSONOpt)
+			grpcServerGroup.Any(etlPath+"*", echo.WrapHandler(etlHandler))
+		}
 
 		// Create h2c-compatible server
 		h2cServer := &http.Server{
@@ -559,9 +632,12 @@ func startEchoProxy(hostUrl *url.URL, logger *zap.Logger, coreService *coreServe
 
 	baseRoutes := e.Group("")
 	baseRoutes.Use(common.CORS())
-	baseRoutes.GET("/", func(c echo.Context) error {
-		return c.JSON(http.StatusOK, map[string]int{"a": 440})
-	})
+	// Base route collides with console dashboard when explorer is enabled
+	if !consoleEnabled {
+		baseRoutes.GET("/", func(c echo.Context) error {
+			return c.JSON(http.StatusOK, map[string]int{"a": 440})
+		})
+	}
 	baseRoutes.GET("/console", func(c echo.Context) error {
 		return c.Redirect(http.StatusMovedPermanently, "/console/overview")
 	})

dev/env/openaudio-1.env

@@ -8,3 +8,7 @@ externalAddress="openaudio-1:26656"
 archive=true
 stateSyncServeSnapshots=true
 stateSyncEnable=false
+
+# Enable ETL and Explorer functionality
+# OPENAUDIO_ETL_ENABLED=true
+# OPENAUDIO_EXPLORER_ENABLED=true

docs/developers.md

@@ -84,12 +84,6 @@ open https://node1.oap.devnet/console/uptime
 > By default, hot reloading is only enabled on node1.oap.devnet to conserve system resources.
 > To enable on other nodes, update the corresponding env file in [dev/env](../dev/env).
 
-**Cleanup**
-
-```bash
-make down
-```
-
 ## Develop against stage or prod
 
 Build a local docker image
@@ -127,3 +121,23 @@ Run only integration tests
 ```bash
 make test-integration
 ```
+
+### ETL
+
+The ETL service indexes blockchain data into the postgres database, enabling faster queries for certain views.
+
+```bash
+OPENAUDIO_ETL_ENABLED=true
+```
+
+### Explorer
+
+The Explorer provides a web-based interface to browse blocks, transactions, validators, and other  data. If enabled, the explorer runs at the site root, e.g. https://node1.oap.devnet/. Explorer requires ETL.
+
+```bash
+OPENAUDIO_ETL_ENABLED=true
+OPENAUDIO_EXPLORER_ENABLED=true
+
+# View explorer in browser
+open https://node1.oap.devnet/
+```

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/tus/tusd/v2 v2.8.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.38.0
+	modernc.org/sqlite v1.38.0
 )
 
 require (
@@ -182,6 +183,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
@@ -199,6 +201,7 @@ require (
 	github.com/quic-go/quic-go v0.44.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/refraction-networking/utls v1.5.3 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/sasha-s/go-deadlock v0.3.5 // indirect
@@ -235,6 +238,9 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.65.10 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 

go.sum

@@ -485,6 +485,8 @@ github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2
 github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@@ -555,6 +557,8 @@ github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0
 github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/refraction-networking/utls v1.5.3 h1:Ds5Ocg1+MC1ahNx5iBEcHe0jHeLaA/fLey61EENm7ro=
 github.com/refraction-networking/utls v1.5.3/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -862,5 +866,29 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
 lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
+modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s=
+modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
+modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
+modernc.org/fileutil v1.3.3 h1:3qaU+7f7xxTUmvU1pJTZiDLAIoJVdUSSauJNHg9yXoA=
+modernc.org/fileutil v1.3.3/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/libc v1.65.10 h1:ZwEk8+jhW7qBjHIT+wd0d9VjitRyQef9BnzlzGwMODc=
+modernc.org/libc v1.65.10/go.mod h1:StFvYpx7i/mXtBAfVOjaU0PWZOvIRoZSgXhrwXzr8Po=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.38.0 h1:+4OrfPQ8pxHKuWG4md1JpR/EYAh3Md7TdejuuzE7EUI=
+modernc.org/sqlite v1.38.0/go.mod h1:1Bj+yES4SVvBZ4cBOpVZ6QgesMCKpJZDq0nxYzOpmNE=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
 rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=