WIP: new transaction index

core/blockchain.go

@@ -146,7 +146,11 @@ const (
 	//  The following incompatible database changes were added:
 	//  * Total difficulty has been removed from both the key-value store and the ancient store.
 	//  * The metadata structure of freezer is changed by adding 'flushOffset'
-	BlockChainVersion uint64 = 9
+	//
+	// - Version 10
+	//  The following incompatible database changes were added:
+	//    * todo(omer)
+	BlockChainVersion uint64 = 10
 )
 
 // BlockChainConfig contains the configuration of the BlockChain object.
@@ -1127,7 +1131,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	if err := batch.Write(); err != nil {
 		log.Crit("Failed to write genesis block", "err", err)
 	}
-	bc.writeHeadBlock(genesis)
+	bc.writeHeadBlock(genesis, types.Receipts{})
 
 	// Last update all in-memory chain markers
 	bc.genesisBlock = genesis
@@ -1185,13 +1189,15 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
 // or if they are on a different side chain.
 //
 // Note, this function assumes that the `mu` mutex is held!
-func (bc *BlockChain) writeHeadBlock(block *types.Block) {
+func (bc *BlockChain) writeHeadBlock(block *types.Block, receipts types.Receipts) {
 	// Add the block to the canonical chain number scheme and mark as the head
 	batch := bc.db.NewBatch()
 	rawdb.WriteHeadHeaderHash(batch, block.Hash())
 	rawdb.WriteHeadFastBlockHash(batch, block.Hash())
 	rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
-	rawdb.WriteTxLookupEntriesByBlock(batch, block)
+	if bc.txIndexer != nil {
+		bc.txIndexer.indexHead(batch, block, receipts)
+	}
 	rawdb.WriteHeadBlockHash(batch, block.Hash())
 
 	// Flush the whole batch into the disk, exit the node if failed
@@ -1536,7 +1542,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error {
 			return err
 		}
 	}
-	bc.writeHeadBlock(block)
+	bc.writeHeadBlock(block, bc.GetReceiptsByHash(block.Hash()))
 	return nil
 }
 
@@ -1638,7 +1644,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
 	}
 
 	// Set new head.
-	bc.writeHeadBlock(block)
+	bc.writeHeadBlock(block, receipts)
 
 	bc.chainFeed.Send(ChainEvent{Header: block.Header()})
 	if len(logs) > 0 {
@@ -2267,7 +2273,7 @@ func (bc *BlockChain) collectLogs(b *types.Block, removed bool) []*types.Log {
 		blobGasPrice = eip4844.CalcBlobFee(bc.chainConfig, b.Header())
 	}
 	receipts := rawdb.ReadRawReceipts(bc.db, b.Hash(), b.NumberU64())
-	if err := receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), blobGasPrice, b.Transactions()); err != nil {
+	if err := receipts.DeriveFields(bc.chainConfig, b.Header(), blobGasPrice, b.Transactions()); err != nil {
 		log.Error("Failed to derive block receipts fields", "hash", b.Hash(), "number", b.NumberU64(), "err", err)
 	}
 	var logs []*types.Log
@@ -2430,7 +2436,7 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error
 			rebirthLogs = nil
 		}
 		// Update the head block
-		bc.writeHeadBlock(block)
+		bc.writeHeadBlock(block, bc.GetReceiptsByHash(block.Hash()))
 	}
 	if len(rebirthLogs) > 0 {
 		bc.logsFeed.Send(rebirthLogs)
@@ -2505,7 +2511,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
 			return common.Hash{}, err
 		}
 	}
-	bc.writeHeadBlock(head)
+	bc.writeHeadBlock(head, bc.GetReceiptsByHash(head.Hash()))
 
 	// Emit events
 	logs := bc.collectLogs(head, false)

core/blockchain_reader.go

@@ -18,9 +18,12 @@ package core
 
 import (
 	"errors"
+	"fmt"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
@@ -213,6 +216,37 @@ func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*type
 	return
 }
 
+// GetReceiptByIndex allows fetching a receipt for a transaction that was already
+// looked up on the index.
+func (bc *BlockChain) GetReceiptByIndex(tx *types.Transaction, blockHash common.Hash, blockNumber, txIndex uint64) (*types.Receipt, error) {
+	if receipts, ok := bc.receiptsCache.Get(blockHash); ok {
+		if int(txIndex) >= len(receipts) {
+			return nil, fmt.Errorf("receipt out of index, length: %d, index: %d", len(receipts), txIndex)
+		}
+		return receipts[int(txIndex)], nil
+	}
+	header := bc.GetHeader(blockHash, blockNumber)
+	if header == nil {
+		return nil, fmt.Errorf("block header is not found, %d, %x", blockNumber, blockHash)
+	}
+	var blobGasPrice *big.Int
+	if header.ExcessBlobGas != nil {
+		blobGasPrice = eip4844.CalcBlobFee(bc.chainConfig, header)
+	}
+	receipt, ctx, err := rawdb.ReadRawReceiptWithContext(bc.db, blockHash, blockNumber, txIndex)
+	if err != nil {
+		return nil, err
+	}
+	signer := types.MakeSigner(bc.chainConfig, new(big.Int).SetUint64(blockNumber), header.Time)
+	receipt.DeriveFields(types.MakeDeriveReceiptContext(signer, header, blobGasPrice, tx, ctx.GasUsed, uint(txIndex), ctx.LogIndex))
+	return receipt, nil
+}
+
+// GetRawReceipt
+func (bc *BlockChain) GetRawReceipt(blockHash common.Hash, blockNumber, txIndex uint64) (*types.Receipt, error) {
+	return rawdb.ReadRawReceipt(bc.db, blockHash, blockNumber, txIndex)
+}
+
 // GetReceiptsByHash retrieves the receipts for all transactions in a given block.
 func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
@@ -307,6 +341,36 @@ func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLoo
 	return lookup, tx
 }
 
+// GetTxIndex
+func (bc *BlockChain) GetTxIndex(txHash common.Hash) *rawdb.TxIndex {
+	return rawdb.ReadTxIndex(bc.db, txHash)
+}
+
+// MakeDeriveReceiptContextFromIndex
+func MakeDeriveReceiptContextFromIndex(txHash common.Hash, txIndex *rawdb.TxIndex) types.DeriveReceiptContext {
+	return types.DeriveReceiptContext{
+		BlockHash:    txIndex.BlockHash,
+		BlockNumber:  txIndex.BlockNumber,
+		BlockTime:    txIndex.BlockTime,
+		BaseFee:      txIndex.BaseFee,
+		BlobGasPrice: txIndex.BlobGasPrice, // todo
+
+		// Receipt fields
+		GasUsed:  txIndex.GasUsed,
+		LogIndex: uint(txIndex.LogIndex), // Number of logs in the block until this receipt
+
+		// Tx fields
+		Hash:              txHash,
+		Nonce:             txIndex.Nonce,
+		Index:             uint(txIndex.TxIndex),
+		Type:              txIndex.Type,
+		From:              txIndex.Sender,
+		To:                txIndex.To,
+		EffectiveGasPrice: txIndex.EffectiveGasPrice,
+		BlobGas:           txIndex.BlobGas,
+	}
+}
+
 // TxIndexDone returns true if the transaction indexer has finished indexing.
 func (bc *BlockChain) TxIndexDone() bool {
 	progress, err := bc.TxIndexProgress()

core/blockchain_test.go

@@ -25,10 +25,12 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"reflect"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/beacon"
@@ -46,6 +48,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/holiman/uint256"
+	"github.com/stretchr/testify/assert"
 )
 
 // So we can deterministically seed different blockchains
@@ -1016,17 +1019,35 @@ func testChainTxReorgs(t *testing.T, scheme string) {
 		if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn == nil {
 			t.Errorf("add %d: expected tx to be found", i)
 		}
-		if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash(), blockchain.Config()); rcpt == nil {
+		if rcpt, _, _, index := rawdb.ReadReceipt(db, tx.Hash(), blockchain.Config()); rcpt == nil {
 			t.Errorf("add %d: expected receipt to be found", i)
+		} else if rawRcpt, ctx, _ := rawdb.ReadRawReceiptWithContext(db, rcpt.BlockHash, rcpt.BlockNumber.Uint64(), index); rawRcpt == nil {
+			t.Errorf("add %d: expected raw receipt to be found", i)
+		} else {
+			if rcpt.GasUsed != ctx.GasUsed {
+				t.Errorf("add %d, raw gasUsedSoFar doesn't make sense", i)
+			}
+			if len(rcpt.Logs) > 0 && rcpt.Logs[0].Index != ctx.LogIndex {
+				t.Errorf("add %d, raw startingLogIndex doesn't make sense", i)
+			}
 		}
 	}
 	// shared tx
 	for i, tx := range (types.Transactions{postponed, swapped}) {
 		if txn, _, _, _ := rawdb.ReadTransaction(db, tx.Hash()); txn == nil {
 			t.Errorf("share %d: expected tx to be found", i)
 		}
-		if rcpt, _, _, _ := rawdb.ReadReceipt(db, tx.Hash(), blockchain.Config()); rcpt == nil {
+		if rcpt, _, _, index := rawdb.ReadReceipt(db, tx.Hash(), blockchain.Config()); rcpt == nil {
 			t.Errorf("share %d: expected receipt to be found", i)
+		} else if rawRcpt, ctx, _ := rawdb.ReadRawReceiptWithContext(db, rcpt.BlockHash, rcpt.BlockNumber.Uint64(), index); rawRcpt == nil {
+			t.Errorf("add %d: expected raw receipt to be found", i)
+		} else {
+			if rcpt.GasUsed != ctx.GasUsed {
+				t.Errorf("add %d, raw gasUsedSoFar doesn't make sense", i)
+			}
+			if len(rcpt.Logs) > 0 && rcpt.Logs[0].Index != ctx.LogIndex {
+				t.Errorf("add %d, raw startingLogIndex doesn't make sense", i)
+			}
 		}
 	}
 }
@@ -4404,6 +4425,93 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
 			if receipts == nil || len(receipts) != 1 {
 				t.Fatalf("Missed block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
 			}
+			for indx, receipt := range receipts {
+				receiptByLookup, err := chain.GetReceiptByIndex(body.Transactions[indx], receipt.BlockHash,
+					receipt.BlockNumber.Uint64(), uint64(indx))
+				assert.NoError(t, err)
+				assert.Equal(t, receipt, receiptByLookup)
+			}
+		}
+	}
+}
+
+func TestGetReceiptByIndex(t *testing.T) {
+	const chainLength = 64
+
+	// Configure and generate a sample block chain
+	var (
+		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		address = crypto.PubkeyToAddress(key.PublicKey)
+		funds   = big.NewInt(1000000000000000000)
+		gspec   = &Genesis{
+			Config:  params.MergedTestChainConfig,
+			Alloc:   types.GenesisAlloc{address: {Balance: funds}},
+			BaseFee: big.NewInt(params.InitialBaseFee),
+		}
+		signer  = types.LatestSigner(gspec.Config)
+		engine  = beacon.New(ethash.NewFaker())
+		codeBin = common.FromHex("0x608060405234801561000f575f5ffd5b507f8ae1c8c6e5f91159d0bc1c4b9a47ce45301753843012cbe641e4456bfc73538b33426040516100419291906100ff565b60405180910390a1610139565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f6100778261004e565b9050919050565b6100878161006d565b82525050565b5f819050919050565b61009f8161008d565b82525050565b5f82825260208201905092915050565b7f436f6e7374727563746f72207761732063616c6c6564000000000000000000005f82015250565b5f6100e96016836100a5565b91506100f4826100b5565b602082019050919050565b5f6060820190506101125f83018561007e565b61011f6020830184610096565b8181036040830152610130816100dd565b90509392505050565b603e806101455f395ff3fe60806040525f5ffdfea2646970667358221220e8bc3c31e3ac337eab702e8fdfc1c71894f4df1af4221bcde4a2823360f403fb64736f6c634300081e0033")
+	)
+	_, blocks, receipts := GenerateChainWithGenesis(gspec, engine, chainLength, func(i int, block *BlockGen) {
+		// SPDX-License-Identifier: MIT
+		// pragma solidity ^0.8.0;
+		//
+		// contract ConstructorLogger {
+		//    event ConstructorLog(address sender, uint256 timestamp, string message);
+		//
+		//    constructor() {
+		//        emit ConstructorLog(msg.sender, block.timestamp, "Constructor was called");
+		//    }
+		// }
+		//
+		// 608060405234801561000f575f5ffd5b507f8ae1c8c6e5f91159d0bc1c4b9a47ce45301753843012cbe641e4456bfc73538b33426040516100419291906100ff565b60405180910390a1610139565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f6100778261004e565b9050919050565b6100878161006d565b82525050565b5f819050919050565b61009f8161008d565b82525050565b5f82825260208201905092915050565b7f436f6e7374727563746f72207761732063616c6c6564000000000000000000005f82015250565b5f6100e96016836100a5565b91506100f4826100b5565b602082019050919050565b5f6060820190506101125f83018561007e565b61011f6020830184610096565b8181036040830152610130816100dd565b90509392505050565b603e806101455f395ff3fe60806040525f5ffdfea2646970667358221220e8bc3c31e3ac337eab702e8fdfc1c71894f4df1af4221bcde4a2823360f403fb64736f6c634300081e0033
+		nonce := block.TxNonce(address)
+		tx, err := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 100_000, block.header.BaseFee, codeBin), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		block.AddTx(tx)
+
+		tx2, err := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 100_000, block.header.BaseFee, codeBin), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		block.AddTx(tx2)
+
+		tx3, err := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 100_000, block.header.BaseFee, codeBin), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		block.AddTx(tx3)
+	})
+
+	db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
+	defer db.Close()
+	options := DefaultConfig().WithStateScheme(rawdb.PathScheme)
+	chain, _ := NewBlockChain(db, gspec, beacon.New(ethash.NewFaker()), options)
+	defer chain.Stop()
+
+	chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
+
+	for i := 0; i < chainLength; i++ {
+		block := blocks[i]
+		blockReceipts := chain.GetReceiptsByHash(block.Hash())
+		chain.receiptsCache.Purge() // ugly hack
+		for txIndex, tx := range block.Body().Transactions {
+			receipt, err := chain.GetReceiptByIndex(tx, block.Hash(), block.NumberU64(), uint64(txIndex))
+			if err != nil {
+				t.Fatalf("Unexpected error %v", err)
+			}
+			if !reflect.DeepEqual(receipts[i][txIndex], receipt) {
+				want := spew.Sdump(receipts[i][txIndex])
+				got := spew.Sdump(receipt)
+				t.Fatalf("Receipt is not matched, want %s, got: %s", want, got)
+			}
+			if !reflect.DeepEqual(blockReceipts[txIndex], receipt) {
+				want := spew.Sdump(blockReceipts[txIndex])
+				got := spew.Sdump(receipt)
+				t.Fatalf("Receipt is not matched, want %s, got: %s", want, got)
+			}
 		}
 	}
 }

core/chain_makers.go

@@ -453,7 +453,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if block.ExcessBlobGas() != nil {
 			blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header())
 		}
-		if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil {
+		if err := receipts.DeriveFields(config, block.Header(), blobGasPrice, txs); err != nil {
 			panic(err)
 		}
 
@@ -563,7 +563,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 		if block.ExcessBlobGas() != nil {
 			blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header())
 		}
-		if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil {
+		if err := receipts.DeriveFields(config, block.Header(), blobGasPrice, txs); err != nil {
 			panic(err)
 		}
 

core/rawdb/accessors_chain.go

@@ -585,19 +585,15 @@ func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64, time uint64,
 		return nil
 	}
 	header := ReadHeader(db, hash, number)
-
-	var baseFee *big.Int
 	if header == nil {
-		baseFee = big.NewInt(0)
-	} else {
-		baseFee = header.BaseFee
+		return nil
 	}
 	// Compute effective blob gas price.
 	var blobGasPrice *big.Int
-	if header != nil && header.ExcessBlobGas != nil {
+	if header.ExcessBlobGas != nil {
 		blobGasPrice = eip4844.CalcBlobFee(config, header)
 	}
-	if err := receipts.DeriveFields(config, hash, number, time, baseFee, blobGasPrice, body.Transactions); err != nil {
+	if err := receipts.DeriveFields(config, header, blobGasPrice, body.Transactions); err != nil {
 		log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err)
 		return nil
 	}