From 9082f09cca02dcc2591ff8a695c925c515560de0 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 9 Jul 2026 22:57:28 -0400 Subject: [PATCH 1/2] fullhistory: add per-CF tuning knobs to rocksdb.CFOptions Grow CFOptions with the memtable, compaction, and bloom knobs that were DB-wide fields on Tuning, and route the bloom filter through the per-CF override. Tuning now carries only the genuinely DB-wide resources shared across every CF of one store: block cache, WAL cap, background jobs, and open-file cap. This lets a facade scope a workload's tuning to the CF that has that workload instead of leaking it onto every CF in the shared per-chunk DB. --- .../fullhistory/storage/rocksdb/rocksdb.go | 99 ++++++++++--------- .../storage/rocksdb/rocksdb_test.go | 35 ++++--- .../fullhistory/storage/rocksdb/tuning.go | 77 ++++++++------- 3 files changed, 112 insertions(+), 99 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go index 54545e6a5..bb1b6e3c8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go @@ -586,31 +586,55 @@ func (s *Store) constructAndOpen() error { return nil } -// applyTuning splits configuration into wrapper-pinned values -// (every CF, unconditional), per-facade Tuning fields (applied -// only when non-zero), and per-CF overrides (applied last so they -// win over the pinned defaults). BloomFilterBitsPerKey == 0 is -// the documented "no bloom filter" sentinel. +// applyTuning splits configuration into wrapper-pinned values (every CF, +// unconditional), per-CF overrides (applied after the pinned defaults so +// they win), and DB-wide Tuning fields (applied to the shared Options). +// BloomFilterBitsPerKey == 0 is the documented "no bloom filter" sentinel. func (s *Store) applyTuning(opts *grocksdb.Options, cfNames []string, cfOpts []*grocksdb.Options) { t := s.cfg.Tuning for i, o := range cfOpts { applyPinnedCFOptions(o) - applyCFTuning(o, t) applyCFOverride(o, s.cfg.PerCFOptions[cfNames[i]]) } applyDBTuning(opts, t) s.applySharedTableOptions(cfNames, cfOpts, t) } -// applyCFOverride applies the per-CF Compression override. BlockSize -// is applied inside applySharedTableOptions when the BBTO is built. -func applyCFOverride(o *grocksdb.Options, override CFOptions) { +// applyCFOverride applies the per-CF memtable/compaction/compression knobs. +// Each is applied only when non-zero, leaving the pinned default otherwise. +// BlockSize and BloomFilterBitsPerKey are applied inside +// applySharedTableOptions when the BBTO is built. +func applyCFOverride(o *grocksdb.Options, c CFOptions) { // CompressionType is an int alias; NoCompression (the pinned // default) is 0. A zero-value override is therefore a no-op and // leaves the pinned NoCompression in place. Non-zero values // (Snappy, ZSTD, ...) replace it. - if override.Compression != grocksdb.NoCompression { - o.SetCompression(override.Compression) + if c.Compression != grocksdb.NoCompression { + o.SetCompression(c.Compression) + } + if c.WriteBufferMB > 0 { + o.SetWriteBufferSize(uint64(c.WriteBufferMB) << 20) + } + if c.MaxWriteBufferNumber > 0 { + o.SetMaxWriteBufferNumber(c.MaxWriteBufferNumber) + } + if c.Level0FileNumCompactionTrigger > 0 { + o.SetLevel0FileNumCompactionTrigger(c.Level0FileNumCompactionTrigger) + } + if c.Level0SlowdownWritesTrigger > 0 { + o.SetLevel0SlowdownWritesTrigger(c.Level0SlowdownWritesTrigger) + } + if c.Level0StopWritesTrigger > 0 { + o.SetLevel0StopWritesTrigger(c.Level0StopWritesTrigger) + } + if c.DisableAutoCompactions { + o.SetDisableAutoCompactions(true) + } + if c.TargetFileSizeMB > 0 { + o.SetTargetFileSizeBase(uint64(c.TargetFileSizeMB) << 20) + } + if c.MaxBytesForLevelBaseMB > 0 { + o.SetMaxBytesForLevelBase(uint64(c.MaxBytesForLevelBaseMB) << 20) } } @@ -625,33 +649,6 @@ func applyPinnedCFOptions(o *grocksdb.Options) { o.SetCompression(grocksdb.NoCompression) } -func applyCFTuning(o *grocksdb.Options, t Tuning) { - if t.WriteBufferMB > 0 { - o.SetWriteBufferSize(uint64(t.WriteBufferMB) << 20) - } - if t.MaxWriteBufferNumber > 0 { - o.SetMaxWriteBufferNumber(t.MaxWriteBufferNumber) - } - if t.Level0FileNumCompactionTrigger > 0 { - o.SetLevel0FileNumCompactionTrigger(t.Level0FileNumCompactionTrigger) - } - if t.Level0SlowdownWritesTrigger > 0 { - o.SetLevel0SlowdownWritesTrigger(t.Level0SlowdownWritesTrigger) - } - if t.Level0StopWritesTrigger > 0 { - o.SetLevel0StopWritesTrigger(t.Level0StopWritesTrigger) - } - if t.DisableAutoCompactions { - o.SetDisableAutoCompactions(true) - } - if t.TargetFileSizeMB > 0 { - o.SetTargetFileSizeBase(uint64(t.TargetFileSizeMB) << 20) - } - if t.MaxBytesForLevelBaseMB > 0 { - o.SetMaxBytesForLevelBase(uint64(t.MaxBytesForLevelBaseMB) << 20) - } -} - func applyDBTuning(opts *grocksdb.Options, t Tuning) { if t.MaxBackgroundJobs > 0 { opts.SetMaxBackgroundJobs(t.MaxBackgroundJobs) @@ -665,15 +662,19 @@ func applyDBTuning(opts *grocksdb.Options, t Tuning) { } // applySharedTableOptions builds one BBTO per CF, referencing the -// shared block cache (when set), a fresh per-CF bloom filter (when -// set), and the per-CF BlockSize override (when set). The Store -// retains every BBTO and the cache; Close destroys them after -// opts/cfOpts. +// DB-wide block cache (when set), a fresh per-CF bloom filter (when +// that CF sets BloomFilterBitsPerKey), and the per-CF BlockSize +// override (when set). The Store retains every BBTO and the cache; +// Close destroys them after opts/cfOpts. +// +// The block cache is DB-wide (one LRU shared across every CF from +// Tuning.BlockCacheMB); the bloom filter and block size are per-CF +// (from PerCFOptions), so only a CF that opts in pays for them. // -// A BBTO is installed on a CF iff any of the cache, a bloom filter, -// or that CF's BlockSize override is configured — preserving the -// previous behavior of leaving RocksDB's default BBTO untouched when -// no table-level knob is set. +// A BBTO is installed on a CF iff the shared cache, that CF's bloom +// filter, or that CF's BlockSize override is configured — preserving +// the previous behavior of leaving RocksDB's default BBTO untouched +// when no table-level knob is set. // // The bloom filter is built per CF because SetFilterPolicy MOVES the // policy into the BBTO (it nils the source pointer), so a single @@ -685,15 +686,15 @@ func (s *Store) applySharedTableOptions(cfNames []string, cfOpts []*grocksdb.Opt } for i, o := range cfOpts { override := s.cfg.PerCFOptions[cfNames[i]] - if s.cache == nil && t.BloomFilterBitsPerKey == 0 && override.BlockSize == 0 { + if s.cache == nil && override.BloomFilterBitsPerKey == 0 && override.BlockSize == 0 { continue } bbto := grocksdb.NewDefaultBlockBasedTableOptions() if s.cache != nil { bbto.SetBlockCache(s.cache) } - if t.BloomFilterBitsPerKey > 0 { - bbto.SetFilterPolicy(grocksdb.NewBloomFilter(float64(t.BloomFilterBitsPerKey))) + if override.BloomFilterBitsPerKey > 0 { + bbto.SetFilterPolicy(grocksdb.NewBloomFilter(float64(override.BloomFilterBitsPerKey))) } if override.BlockSize > 0 { bbto.SetBlockSize(override.BlockSize) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go index 8af932b10..c80564983 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go @@ -133,13 +133,14 @@ func TestNew_TwoStoresSamePathCollide(t *testing.T) { func TestStore_ConstructAndOpenFailureFreesCacheAndBBTOs(t *testing.T) { dir := t.TempDir() - tuning := Tuning{BlockCacheMB: 4, BloomFilterBitsPerKey: 10} + tuning := Tuning{BlockCacheMB: 4} + perCF := map[string]CFOptions{defaultCFName: {BloomFilterBitsPerKey: 10}} - holder, err := New(Config{Path: dir, Logger: silentLogger(), Tuning: tuning}) + holder, err := New(Config{Path: dir, Logger: silentLogger(), Tuning: tuning, PerCFOptions: perCF}) require.NoError(t, err) t.Cleanup(func() { _ = holder.Close() }) - collider := &Store{cfg: Config{Path: dir, Logger: silentLogger(), Tuning: tuning}} + collider := &Store{cfg: Config{Path: dir, Logger: silentLogger(), Tuning: tuning, PerCFOptions: perCF}} require.Error(t, collider.constructAndOpen()) assert.Nil(t, collider.cache) @@ -665,18 +666,22 @@ func TestStore_TuningRoundTrip(t *testing.T) { Path: t.TempDir(), Logger: newTestLogger(&buf), Tuning: Tuning{ - WriteBufferMB: 8, - MaxWriteBufferNumber: 2, - Level0FileNumCompactionTrigger: 4, - Level0SlowdownWritesTrigger: 20, - Level0StopWritesTrigger: 36, - TargetFileSizeMB: 16, - MaxBytesForLevelBaseMB: 64, - MaxBackgroundJobs: 2, - MaxOpenFiles: 500, - BlockCacheMB: 4, - BloomFilterBitsPerKey: 10, - MaxTotalWalSizeMB: 16, + MaxBackgroundJobs: 2, + MaxOpenFiles: 500, + BlockCacheMB: 4, + MaxTotalWalSizeMB: 16, + }, + PerCFOptions: map[string]CFOptions{ + defaultCFName: { + WriteBufferMB: 8, + MaxWriteBufferNumber: 2, + Level0FileNumCompactionTrigger: 4, + Level0SlowdownWritesTrigger: 20, + Level0StopWritesTrigger: 36, + TargetFileSizeMB: 16, + MaxBytesForLevelBaseMB: 64, + BloomFilterBitsPerKey: 10, + }, }, }) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go index 12f5df1d8..f41a2e305 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go @@ -2,12 +2,15 @@ package rocksdb import "github.com/linxGnu/grocksdb" -// CFOptions — per-CF overrides applied after the shared pinned -// defaults and global Tuning. Zero means "inherit the pinned -// default" (NoCompression for Compression, RocksDB's BBTO default -// block size for BlockSize). Use to opt specific CFs into -// compression or non-default block sizes while leaving the rest at -// the wrapper-pinned defaults. +// CFOptions — per-CF overrides applied after the shared pinned defaults. +// Zero means "inherit the pinned default" (NoCompression for Compression, +// RocksDB's BBTO default block size for BlockSize, no bloom filter, and +// grocksdb's memtable/compaction defaults for the rest). Each field a CF +// leaves zero rides on the wrapper-pinned defaults; a facade opts a specific +// CF into a knob by setting it here. The knobs are per-CF because each hot +// data type's workload differs (txhash is write-once/point-lookup; ledgers is +// never probed for missing keys; events holds compressible XDR) and one CF's +// tuning must not leak onto the others. type CFOptions struct { // Compression overrides the CF's compression type. The wrapper's // pinned default is NoCompression; CFs that hold compressible @@ -25,65 +28,69 @@ type CFOptions struct { // block holding the target key; larger blocks waste I/O per // cache miss. BlockSize int -} -// Tuning — per-store RocksDB knobs. Zero means "leave grocksdb's -// default alone" (wrapper skips the setter). BloomFilterBitsPerKey == 0 -// is the documented "install no bloom filter" sentinel. -// -// Wrapper-pinned values not exposed here (applied to every facade): -// MinWriteBufferNumberToMerge=1, CompactionStyle=Level, -// TargetFileSizeMultiplier=1, MaxBytesForLevelMultiplier=10, -// Compression=None, WAL=on, per-write Sync=on. -// Per-CF overrides for Compression and BlockSize live in -// Config.PerCFOptions. -type Tuning struct { - // WriteBufferMB sizes the active memtable per CF, in MB. + // WriteBufferMB sizes the active memtable for this CF, in MB. Zero + // leaves grocksdb's default. WriteBufferMB int // MaxWriteBufferNumber caps the active + immutable memtable count - // per CF before writes back-pressure. + // for this CF before writes back-pressure. Zero leaves the default. MaxWriteBufferNumber int // Level0FileNumCompactionTrigger — L0 file count that starts - // an L0→L1 compaction. + // an L0→L1 compaction. Zero leaves the default. Level0FileNumCompactionTrigger int // Level0SlowdownWritesTrigger — L0 file count that slows writes. + // Zero leaves the default. Level0SlowdownWritesTrigger int // Level0StopWritesTrigger — L0 file count that stalls writes - // entirely. + // entirely. Zero leaves the default. Level0StopWritesTrigger int - // DisableAutoCompactions turns automatic compaction off per CF — - // for write-once, point-lookup stores where compaction would - // rewrite the same data with no reordering benefit. + // DisableAutoCompactions turns automatic compaction off for this CF — + // for write-once, point-lookup CFs where compaction would rewrite the + // same data with no reordering benefit. DisableAutoCompactions bool // TargetFileSizeMB — size at which compaction produces new SSTs. + // Zero leaves the default. TargetFileSizeMB int // MaxBytesForLevelBaseMB — byte budget for level 1; each later - // level is this × MaxBytesForLevelMultiplier (10, pinned). + // level is this × MaxBytesForLevelMultiplier (10, pinned). Zero + // leaves the default. MaxBytesForLevelBaseMB int + // BloomFilterBitsPerKey installs this CF's bloom filter. 0 = no + // filter (the default) — right for a CF that is never probed for + // keys it may not hold. Positive values install one with that many + // bits; typical: 10 (~1% false positive), 12 (~0.4%). + BloomFilterBitsPerKey int +} + +// Tuning — DB-wide RocksDB knobs shared across every CF of one store. Zero +// means "leave grocksdb's default alone" (wrapper skips the setter). Per-CF +// knobs (memtables, compaction, bloom, block size, compression) live in +// Config.PerCFOptions, not here. +// +// Wrapper-pinned per-CF values not exposed anywhere (applied to every CF): +// MinWriteBufferNumberToMerge=1, CompactionStyle=Level, +// TargetFileSizeMultiplier=1, MaxBytesForLevelMultiplier=10, +// Compression=None, WAL=on, per-write Sync=on. +type Tuning struct { // MaxBackgroundJobs caps background threads for compactions - // and flushes combined. Orthogonal to DisableAutoCompactions: - // this rate-limits work, that turns compaction off entirely. + // and flushes combined, across the whole DB. MaxBackgroundJobs int - // MaxOpenFiles caps concurrent open SST files. + // MaxOpenFiles caps concurrent open SST files, across the whole DB. MaxOpenFiles int - // BlockCacheMB sizes the shared LRU block cache, per store. + // BlockCacheMB sizes the LRU block cache shared across every CF in + // the store. BlockCacheMB int - // BloomFilterBitsPerKey — per-CF bloom filter. 0 = no filter - // installed; positive values install one with that many bits. - // Typical: 10 (~1% false positive), 12 (~0.4%). - BloomFilterBitsPerKey int - // MaxTotalWalSizeMB caps total live WAL size. Crash-recovery // replay scales with this cap; graceful Close drains the // memtable so this only bounds ungraceful shutdowns. From df545509e025bc05d06ce6c816b034d3de47ceca Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 9 Jul 2026 22:57:36 -0400 Subject: [PATCH 2/2] fullhistory: scope txhash tuning to the txhash CF Split txhash's config into the DB-wide Tuning it always set (block cache, WAL cap, background jobs, open files) and a new CFOptions carrying its per-CF workload knobs (bloom, write buffers, compaction triggers). The hotchunk opener merges the per-CF options from every facade so those knobs land on the txhash CF alone. Restores each store's pre-unification standalone tuning: the ledgers CF runs with no bloom filter (it is never probed for missing keys) and default memtables/compaction; the events CFs keep their ZSTD and block-size overrides; the txhash CF keeps its full calibration. The DB-wide resources stay shared, sized to what the three standalone instances used together (only txhash set them). --- .../storage/stores/hotchunk/hotchunk.go | 17 ++-- .../storage/stores/hotchunk/hotchunk_test.go | 33 +++++++ .../storage/stores/txhash/hot_store.go | 94 +++++++++++-------- 3 files changed, 99 insertions(+), 45 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go index 63f1a2ca6..1e9ae1346 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "iter" + "maps" "slices" "time" @@ -54,18 +55,22 @@ func ColumnFamilies() []string { return slices.Concat(ledger.CFNames(), eventstore.CFNames(), txhash.CFNames()) } -// config builds the shared store's rocksdb.Config: events' per-CF options (ZSTD -// on DataCF, tuned block sizes) plus the txhash workload's Tuning. Tuning's -// per-CF fields apply to every CF — a benign over-application (ledger/events CFs -// just gain a bloom + larger write buffer); the per-CF overrides keep events -// distinct. +// config builds the shared store's rocksdb.Config: the DB-wide Tuning +// (block cache, WAL cap, background jobs) from txhash — the only facade +// that historically set them — plus the per-CF options merged from every +// facade. txhash's per-CF knobs (bloom, write buffers, compaction) land on +// the txhash CF alone; events' overrides (ZSTD, block sizes) on its three +// CFs; the ledgers CF rides on RocksDB defaults with no bloom, matching each +// store's pre-unification standalone tuning. func config(path string, logger *supportlog.Entry, readOnly, mustExist bool) rocksdb.Config { + perCF := eventstore.CFOptions() + maps.Copy(perCF, txhash.CFOptions()) return rocksdb.Config{ Path: path, ColumnFamilies: ColumnFamilies(), Logger: logger, Tuning: txhash.Tuning(), - PerCFOptions: eventstore.CFOptions(), + PerCFOptions: perCF, ReadOnly: readOnly, MustExist: mustExist, } diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go index cffa7b4c6..539d1a4e1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/linxGnu/grocksdb" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -51,6 +52,38 @@ func assertWriteItems(t *testing.T, rep LedgerReport, txhash int) { assert.Equal(t, 1, rep.Phases[PhaseEvents].Items, "events items") } +// TestConfig_PerCFOptionRouting asserts the txhash Tuning knobs reach ONLY the +// txhash CF: the ledger CF gets no bloom (it is never probed for missing keys) +// and the events CFs keep their own compression/block-size overrides. Asserted +// at the config-construction seam because grocksdb options can't be read back +// off a live DB. +func TestConfig_PerCFOptionRouting(t *testing.T) { + perCF := config(t.TempDir(), silentLogger(), false, false).PerCFOptions + + txCF := txhash.CFNames()[0] + assert.Equal(t, 12, perCF[txCF].BloomFilterBitsPerKey, "txhash CF keeps its bloom") + assert.Equal(t, 64, perCF[txCF].WriteBufferMB, "txhash CF keeps its write buffer") + assert.True(t, perCF[txCF].DisableAutoCompactions, "txhash CF keeps compaction off") + + ledgerCF := ledger.CFNames()[0] + assert.Zero(t, perCF[ledgerCF].BloomFilterBitsPerKey, "ledger CF gets no bloom") + assert.Zero(t, perCF[ledgerCF].WriteBufferMB, "ledger CF rides on RocksDB defaults") + assert.False(t, perCF[ledgerCF].DisableAutoCompactions, "ledger CF keeps auto-compaction") + + assert.Equal(t, grocksdb.ZSTDCompression, perCF[eventstore.DataCF].Compression, + "events data CF keeps its ZSTD override") + assert.NotZero(t, perCF[eventstore.DataCF].BlockSize, "events data CF keeps its block-size override") + assert.Zero(t, perCF[eventstore.DataCF].BloomFilterBitsPerKey, "events data CF gets no bloom") +} + +func TestConfig_DBWideTuningStaysShared(t *testing.T) { + tuning := config(t.TempDir(), silentLogger(), false, false).Tuning + assert.Equal(t, 512, tuning.BlockCacheMB) + assert.Equal(t, 1024, tuning.MaxTotalWalSizeMB) + assert.Equal(t, 8, tuning.MaxBackgroundJobs) + assert.Equal(t, 10_000, tuning.MaxOpenFiles) +} + func TestOpen_ValidatesInputs(t *testing.T) { _, err := Open("", chunk.ID(0), silentLogger()) require.ErrorIs(t, err, stores.ErrInvalidConfig) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go index 5d1312862..ffa849280 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go @@ -44,42 +44,14 @@ func NewWithStore(store *rocksdb.Store) *HotStore { // the hotchunk shared-DB opener can register it alongside the other CFs. func CFNames() []string { return []string{txhashCF} } -// Tuning returns this facade's RocksDB tuning, applied to the shared per-chunk -// DB by the hotchunk opener. The hot txhash workload is write-once / -// point-lookup; the cross-knob interactions below are non-obvious enough that -// they get an explicit per-stanza rationale. The other facades ride on RocksDB -// defaults by contrast — only this workload earned the calibration. +// Tuning returns the DB-wide RocksDB knobs the shared per-chunk DB adopts — +// resources that were per-instance before the hot stores collapsed into one +// multi-CF DB and are now shared across every CF. The txhash instance was the +// only one that set them; the ledger and events instances rode on RocksDB +// defaults, so keeping txhash's values is the faithful consolidation of the +// three instances' combined footprint. func Tuning() rocksdb.Tuning { return rocksdb.Tuning{ - // 64 MB memtable so one flush produces one ~64 MB SST under - // uniform writes. - WriteBufferMB: 64, - MaxWriteBufferNumber: 2, - - // L0 triggers pinned high + DisableAutoCompactions=true: - // compaction would re-write the same data with no reordering - // benefit (txhash is write-once, random-key, point-lookup). - // The L0 999s match DisableAutoCompactions so even if a future - // flush somehow exceeded the trigger, the engine still - // wouldn't try to compact. NOTE: DisableAutoCompactions and - // MaxBackgroundJobs are orthogonal — the former turns - // compaction off entirely, the latter only caps the thread - // budget for background work. - Level0FileNumCompactionTrigger: 999, - Level0SlowdownWritesTrigger: 999, - Level0StopWritesTrigger: 999, - DisableAutoCompactions: true, - - // 64 MB target file matches WriteBufferMB so one memtable - // flush produces one ~64 MB SST — fewer bloom checks per - // query at no-compaction scale. - // MaxBytesForLevelBaseMB is set explicitly even though it's - // irrelevant under DisableAutoCompactions (compaction never - // promotes past L0); explicit > implicit so a future reader - // doesn't have to derive that it's a no-op. - TargetFileSizeMB: 64, - MaxBytesForLevelBaseMB: 256, - // Background-job budget for the periodic memtable flushes. MaxBackgroundJobs: 8, MaxOpenFiles: 10_000, @@ -87,11 +59,7 @@ func Tuning() rocksdb.Tuning { // 512 MB block cache — bloom-filter blocks are the hot // working set; the cache needs to hold recently-touched // bloom blocks at scale. - // 12 bits/key bloom (~0.4% false-positive) is tighter than - // the standard 10 bits/key because every false positive at - // no-compaction SST count costs a disk seek across many SSTs. - BlockCacheMB: 512, - BloomFilterBitsPerKey: 12, + BlockCacheMB: 512, // 1 GB WAL cap. Graceful Close auto-Flushes (see // rocksdb.Store.Close), so this cap only bounds ungraceful-shutdown @@ -100,6 +68,54 @@ func Tuning() rocksdb.Tuning { } } +// CFOptions returns the per-CF knobs the hotchunk opener installs on the +// txhash CF alone (merged into the shared DB's PerCFOptions). The hot txhash +// workload is write-once / point-lookup; the cross-knob interactions below are +// non-obvious enough that they get an explicit per-stanza rationale. The other +// CFs ride on RocksDB defaults by contrast — only this workload earned the +// calibration, and scoping it here keeps it off the ledger and events CFs. +func CFOptions() map[string]rocksdb.CFOptions { + return map[string]rocksdb.CFOptions{ + txhashCF: { + // 64 MB memtable so one flush produces one ~64 MB SST under + // uniform writes. + WriteBufferMB: 64, + MaxWriteBufferNumber: 2, + + // L0 triggers pinned high + DisableAutoCompactions=true: + // compaction would re-write the same data with no reordering + // benefit (txhash is write-once, random-key, point-lookup). + // The L0 999s match DisableAutoCompactions so even if a future + // flush somehow exceeded the trigger, the engine still + // wouldn't try to compact. NOTE: DisableAutoCompactions and + // Tuning.MaxBackgroundJobs are orthogonal — the former turns + // compaction off entirely, the latter only caps the thread + // budget for background work. + Level0FileNumCompactionTrigger: 999, + Level0SlowdownWritesTrigger: 999, + Level0StopWritesTrigger: 999, + DisableAutoCompactions: true, + + // 64 MB target file matches WriteBufferMB so one memtable + // flush produces one ~64 MB SST — fewer bloom checks per + // query at no-compaction scale. + // MaxBytesForLevelBaseMB is set explicitly even though it's + // irrelevant under DisableAutoCompactions (compaction never + // promotes past L0); explicit > implicit so a future reader + // doesn't have to derive that it's a no-op. + TargetFileSizeMB: 64, + MaxBytesForLevelBaseMB: 256, + + // 12 bits/key bloom (~0.4% false-positive) is tighter than + // the standard 10 bits/key because every false positive at + // no-compaction SST count costs a disk seek across many SSTs. + // Scoped to this CF: it is the only one probed for keys it may + // not hold, so the ledger and events CFs install no bloom. + BloomFilterBitsPerKey: 12, + }, + } +} + // AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on the txhash // CF — the building block hotchunk uses to fold the tx-hash writes into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the