Refactor API handlers and implement storage queries (#43)

# API Enhancements and Storage Integration

## Changes
- Renamed `Aggregate` to `Aggregates` in `QueryParams` struct
- Updated route handler names for better clarity:
  - `GetTransactionsWithContract` to `GetTransactionsByContract`
  - `GetEventsWithContract` to `GetLogsByContract`
  - `GetTransactionsWithContractAndSignature` to `GetTransactionsByContractAndSignature`
  - `GetEventsWithContractAndSignature` to `GetLogsByContractAndSignature`
- Implemented `handleLogsRequest` and `handleTransactionsRequest` functions to centralize logic
- Integrated storage layer with API handlers:
  - Added `getMainStorage` function to create storage connector
  - Updated `GetLogs` and `GetTransactions` methods to use storage queries
- Enhanced `QueryFilter` struct with additional fields for filtering and aggregation
- Modified `ClickHouseConnector` methods to remove block number filtering and prepare for more flexible querying
- Renamed `newConnector` to `NewConnector` and made it public

## TODO

- Implement total items count and total pages count in API responses
- Add aggregations, filters, and grouping to log queries in ClickHouse connector

Author: AmineAfia

api/api.go

@@ -25,7 +25,7 @@ type QueryParams struct {
 	SortOrder    string            `schema:"sort_order"`
 	Page         int               `schema:"page"`
 	Limit        int               `schema:"limit"`
-	Aggregate    []string          `schema:"aggregate"`
+	Aggregates    []string          `schema:"aggregate"`
 }
 
 type Meta struct {

internal/common/transaction.go

@@ -16,7 +16,7 @@ type Transaction struct {
 	FromAddress          string
 	ToAddress            string
 	Value                *big.Int
-	Gas                  *big.Int
+	Gas                  uint64
 	GasPrice             *big.Int
 	Data                 string
 	MaxFeePerGas         *big.Int

internal/handlers/api.go

@@ -18,12 +18,12 @@ func Handler(r *chi.Mux) {
 		router.Get("/{chainId}/events", GetLogs)
 
 		// contract scoped queries
-		router.Get("/{chainId}/transactions/{contractAddress}", GetTransactionsWithContract)
-		router.Get("/{chainId}/events/{contractAddress}", GetEventsWithContract)
+		router.Get("/{chainId}/transactions/{contractAddress}", GetTransactionsByContract)
+		router.Get("/{chainId}/events/{contractAddress}", GetLogsByContract)
 
 		// signature scoped queries
-		router.Get("/{chainId}/transactions/{contractAddress}/{functionSig}", GetTransactionsWithContractAndSignature)
-		router.Get("/{chainId}/events/{contractAddress}/{functionSig}", GetEventsWithContractAndSignature)
+		router.Get("/{chainId}/transactions/{contractAddress}/{functionSig}", GetTransactionsByContractAndSignature)
+		router.Get("/{chainId}/events/{contractAddress}/{functionSig}", GetLogsByContractAndSignature)
 	})
 
 	r.Get("/health", func(w http.ResponseWriter, r *http.Request) {

internal/handlers/logs_handlers.go

@@ -3,119 +3,109 @@ package handlers
 import (
 	"encoding/json"
 	"net/http"
+	"sync"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/rs/zerolog/log"
 	"github.com/thirdweb-dev/indexer/api"
+	config "github.com/thirdweb-dev/indexer/configs"
+	"github.com/thirdweb-dev/indexer/internal/storage"
+)
+
+// package-level variables
+var (
+	mainStorage storage.IMainStorage
+	storageOnce sync.Once
+	storageErr  error
 )
 
 func GetLogs(w http.ResponseWriter, r *http.Request) {
-	chainId, err := api.GetChainId(r)
-	if err != nil {
-		api.BadRequestErrorHandler(w, err)
-		return
-	}
-	queryParams, err := api.ParseQueryParams(r)
-	if err != nil {
-		api.BadRequestErrorHandler(w, err)
-		return
-	}
+	handleLogsRequest(w, r, "", "")
+}
 
-	var response = api.QueryResponse{
-		Meta: api.Meta{
-			ChainIdentifier: chainId,
-			ContractAddress: "todo",
-			FunctionSig:     "todo",
-			Page:            1,
-			Limit:           100,
-			TotalItems:      0,
-			TotalPages:      0,
-		},
-		Data: []interface{}{queryParams},
-	}
+func GetLogsByContract(w http.ResponseWriter, r *http.Request) {
+	contractAddress := chi.URLParam(r, "contractAddress")
+	handleLogsRequest(w, r, contractAddress, "")
+}
 
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(response)
-	if err != nil {
-		log.Error().Err(err).Msg("Error encoding response")
-		api.InternalErrorHandler(w)
-		return
-	}
+func GetLogsByContractAndSignature(w http.ResponseWriter, r *http.Request) {
+	contractAddress := chi.URLParam(r, "contractAddress")
+	functionSig := chi.URLParam(r, "functionSig")
+	handleLogsRequest(w, r, contractAddress, functionSig)
 }
 
-func GetEventsWithContract(w http.ResponseWriter, r *http.Request) {
+func handleLogsRequest(w http.ResponseWriter, r *http.Request, contractAddress, functionSig string) {
 	chainId, err := api.GetChainId(r)
 	if err != nil {
 		api.BadRequestErrorHandler(w, err)
 		return
 	}
-	contractAddress := chi.URLParam(r, "contractAddress")
+
 	queryParams, err := api.ParseQueryParams(r)
 	if err != nil {
 		api.BadRequestErrorHandler(w, err)
 		return
 	}
 
-	var response = api.QueryResponse{
-		Meta: api.Meta{
-			ChainIdentifier: chainId,
-			ContractAddress: contractAddress,
-			FunctionSig:     "todo",
-			Page:            1,
-			Limit:           100,
-			TotalItems:      0,
-			TotalPages:      0,
-		},
-		Data: []interface{}{queryParams},
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(response)
+	mainStorage, err := getMainStorage()
 	if err != nil {
-		log.Error().Err(err).Msg("Error encoding response")
+		log.Error().Err(err).Msg("Error getting main storage")
 		api.InternalErrorHandler(w)
 		return
 	}
 
-}
-
-func GetEventsWithContractAndSignature(w http.ResponseWriter, r *http.Request) {
-	chainId, err := api.GetChainId(r)
+	logs, err := mainStorage.GetLogs(storage.QueryFilter{
+		FilterParams:    queryParams.FilterParams,
+		GroupBy:         queryParams.GroupBy,
+		SortBy:          queryParams.SortBy,
+		SortOrder:       queryParams.SortOrder,
+		Page:            queryParams.Page,
+		Limit:           queryParams.Limit,
+		Aggregates:      queryParams.Aggregates,
+		ContractAddress: contractAddress,
+		FunctionSig:     functionSig,
+	})
 	if err != nil {
-		api.BadRequestErrorHandler(w, err)
-		return
-	}
-	contractAddress := chi.URLParam(r, "contractAddress")
-	functionSig := chi.URLParam(r, "functionSig")
-	queryParams, err := api.ParseQueryParams(r)
-	if err != nil {
-		api.BadRequestErrorHandler(w, err)
+		log.Error().Err(err).Msg("Error querying logs")
+		api.InternalErrorHandler(w)
 		return
 	}
 
-	var response = api.QueryResponse{
+	response := api.QueryResponse{
 		Meta: api.Meta{
 			ChainIdentifier: chainId,
 			ContractAddress: contractAddress,
 			FunctionSig:     functionSig,
-			Page:            1,
-			Limit:           100,
-			TotalItems:      0,
-			TotalPages:      0,
+			Page:            queryParams.Page,
+			Limit:           queryParams.Limit,
+			TotalItems:      0, // TODO: Implement total items count
+			TotalPages:      0, // TODO: Implement total pages count
 		},
-		Data: []interface{}{queryParams},
+		Data: []interface{}{logs},
 		Aggregations: map[string]interface{}{
-			"count":         100,
-			"sum_value":     "1000000000000000000000",
-			"avg_gas_price": "20000000000",
+			"aggregates": queryParams.Aggregates,
 		},
 	}
 
+	sendJSONResponse(w, response)
+}
+
+func getMainStorage() (storage.IMainStorage, error) {
+	storageOnce.Do(func() {
+		var err error
+		mainStorage, err = storage.NewConnector[storage.IMainStorage](&config.Cfg.Storage.Main)
+		if err != nil {
+			storageErr = err
+			log.Error().Err(err).Msg("Error creating storage connector")
+		}
+	})
+	return mainStorage, storageErr
+}
+
+func sendJSONResponse(w http.ResponseWriter, response interface{}) {
 	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(response)
-	if err != nil {
+	if err := json.NewEncoder(w).Encode(response); err != nil {
 		log.Error().Err(err).Msg("Error encoding response")
 		api.InternalErrorHandler(w)
-		return
 	}
 }

internal/handlers/transactions_handlers.go

@@ -1,117 +1,81 @@
 package handlers
 
 import (
-	"encoding/json"
 	"net/http"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/rs/zerolog/log"
 	"github.com/thirdweb-dev/indexer/api"
+	"github.com/thirdweb-dev/indexer/internal/storage"
 )
 
 func GetTransactions(w http.ResponseWriter, r *http.Request) {
-	chainId, err := api.GetChainId(r)
-	if err != nil {
-		api.BadRequestErrorHandler(w, err)
-		return
-	}
-
-	queryParams, err := api.ParseQueryParams(r)
-	if err != nil {
-		api.BadRequestErrorHandler(w, err)
-		return
-	}
+	handleTransactionsRequest(w, r, "", "")
+}
 
-	var response = api.QueryResponse{
-		Meta: api.Meta{
-			ChainIdentifier: chainId,
-			ContractAddress: "todo",
-			FunctionSig:     "todo",
-			Page:            1,
-			Limit:           100,
-			TotalItems:      0,
-			TotalPages:      0,
-		},
-		Data: []interface{}{queryParams},
-	}
+func GetTransactionsByContract(w http.ResponseWriter, r *http.Request) {
+	contractAddress := chi.URLParam(r, "contractAddress")
+	handleTransactionsRequest(w, r, contractAddress, "")
+}
 
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(response)
-	if err != nil {
-		log.Error().Err(err).Msg("Error encoding response")
-		api.InternalErrorHandler(w)
-		return
-	}
+func GetTransactionsByContractAndSignature(w http.ResponseWriter, r *http.Request) {
+	contractAddress := chi.URLParam(r, "contractAddress")
+	functionSig := chi.URLParam(r, "functionSig")
+	handleTransactionsRequest(w, r, contractAddress, functionSig)
 }
 
-func GetTransactionsWithContract(w http.ResponseWriter, r *http.Request) {
+func handleTransactionsRequest(w http.ResponseWriter, r *http.Request, contractAddress, functionSig string) {
 	chainId, err := api.GetChainId(r)
 	if err != nil {
 		api.BadRequestErrorHandler(w, err)
 		return
 	}
-	contractAddress := chi.URLParam(r, "contractAddress")
+
 	queryParams, err := api.ParseQueryParams(r)
 	if err != nil {
 		api.BadRequestErrorHandler(w, err)
 		return
 	}
 
-	var response = api.QueryResponse{
-		Meta: api.Meta{
-			ChainIdentifier: chainId,
-			ContractAddress: contractAddress,
-			FunctionSig:     "todo",
-			Page:            1,
-			Limit:           100,
-			TotalItems:      0,
-			TotalPages:      0,
-		},
-		Data: []interface{}{queryParams},
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(response)
+	mainStorage, err := getMainStorage()
 	if err != nil {
-		log.Error().Err(err).Msg("Error encoding response")
+		log.Error().Err(err).Msg("Error creating storage connector")
 		api.InternalErrorHandler(w)
 		return
 	}
 
-}
-
-func GetTransactionsWithContractAndSignature(w http.ResponseWriter, r *http.Request) {
-	chainId, err := api.GetChainId(r)
-	if err != nil {
-		api.BadRequestErrorHandler(w, err)
-		return
-	}
-	contractAddress := chi.URLParam(r, "contractAddress")
-	functionSig := chi.URLParam(r, "functionSig")
-	queryParams, err := api.ParseQueryParams(r)
+	transactions, err := mainStorage.GetTransactions(storage.QueryFilter{
+		FilterParams:    queryParams.FilterParams,
+		GroupBy:         queryParams.GroupBy,
+		SortBy:          queryParams.SortBy,
+		SortOrder:       queryParams.SortOrder,
+		Page:            queryParams.Page,
+		Limit:           queryParams.Limit,
+		Aggregates:      queryParams.Aggregates,
+		ContractAddress: contractAddress,
+		FunctionSig:     functionSig,
+	})
 	if err != nil {
-		api.BadRequestErrorHandler(w, err)
+		log.Error().Err(err).Msg("Error querying transactions")
+		api.InternalErrorHandler(w)
 		return
 	}
 
-	var response = api.QueryResponse{
+	response := api.QueryResponse{
 		Meta: api.Meta{
 			ChainIdentifier: chainId,
 			ContractAddress: contractAddress,
 			FunctionSig:     functionSig,
-			Page:            1,
-			Limit:           100,
-			TotalItems:      0,
-			TotalPages:      0,
+			Page:            queryParams.Page,
+			Limit:           queryParams.Limit,
+			TotalItems:      0, // TODO: Implement total items count
+			TotalPages:      0, // TODO: Implement total pages count
+		},
+		Data: []interface{}{transactions},
+		Aggregations: map[string]interface{}{
+			"aggregates": queryParams.Aggregates,
 		},
-		Data: []interface{}{queryParams},
 	}
 
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(response)
-	if err != nil {
-		log.Error().Err(err).Msg("Error encoding response")
-		api.InternalErrorHandler(w)
-		return
-	}
+	sendJSONResponse(w, response)
 }