diff --git a/README.md b/README.md
index 795aea628..dff6c849f 100644
--- a/README.md
+++ b/README.md
@@ -356,6 +356,12 @@ The *disk cache policy* determines if lookup operations use the OS page
cache. Caching may improve the performance of lookups and updates if
database access follows certain patterns.
+`confMergeBatchSize`
+The merge batch size balances the maximum latency of individual update
+operations, versus the latency of a sequence of update operations.
+Bigger batches improves overall performance but some updates will take a
+lot longer than others. The default is to use a large batch size.
+
##### Fine-tuning: Merge Policy, Size Ratio, and Write Buffer Size
The configuration parameters `confMergePolicy`, `confSizeRatio`, and
@@ -647,6 +653,33 @@ locality if it is likely to access entries that have nearby keys.
does not have good spatial or temporal locality. For instance, if the
access pattern is uniformly random.
+##### Fine-tuning: Merge Batch Size
+
+The *merge batch size* is a micro-tuning parameter, and in most cases
+you do need to think about it and can leave it at its default.
+
+When using the `Incremental` merge schedule, merging is done in batches.
+This is a trade-off: larger batches tends to mean better overall
+performance but the downside is that while most updates (inserts,
+deletes, upserts) are fast, some are slower (when a batch of merging
+work has to be done).
+
+If you care most about the maximum latency of updates, then use a small
+batch size. If you don't care about latency of individual operations,
+just the latency of the overall sequence of operations then use a large
+batch size. The default is to use a large batch size, the same size as
+the write buffer itself. The minimum batch size is 1. The maximum batch
+size is the size of the write buffer `confWriteBufferAlloc`.
+
+Note that the actual batch size is the minimum of this configuration
+parameter and the size of the batch of operations performed (e.g.
+`inserts`). So if you consistently use large batches, you can use a
+batch size of 1 and the merge batch size will always be determined by
+the operation batch size.
+
+A further reason why it may be preferable to use minimal batch sizes is
+to get good parallel work balance, when using parallelism.
+
### References
The implementation of LSM-trees in this package draws inspiration from:
diff --git a/bench/macro/lsm-tree-bench-wp8.hs b/bench/macro/lsm-tree-bench-wp8.hs
index 5870c933d..fd071c980 100644
--- a/bench/macro/lsm-tree-bench-wp8.hs
+++ b/bench/macro/lsm-tree-bench-wp8.hs
@@ -180,13 +180,23 @@ mkTableConfigSetup GlobalOpts{diskCachePolicy} SetupOpts{bloomFilterAlloc} conf
, LSM.confBloomFilterAlloc = bloomFilterAlloc
}
-mkTableConfigRun :: GlobalOpts -> LSM.TableConfig -> LSM.TableConfig
-mkTableConfigRun GlobalOpts{diskCachePolicy} conf = conf {
- LSM.confDiskCachePolicy = diskCachePolicy
+mkTableConfigRun :: GlobalOpts -> RunOpts -> LSM.TableConfig -> LSM.TableConfig
+mkTableConfigRun GlobalOpts{diskCachePolicy} RunOpts {pipelined} conf =
+ conf {
+ LSM.confDiskCachePolicy = diskCachePolicy,
+ LSM.confMergeBatchSize = if pipelined
+ then LSM.MergeBatchSize 1
+ else LSM.confMergeBatchSize conf
}
-mkOverrideDiskCachePolicy :: GlobalOpts -> LSM.OverrideDiskCachePolicy
-mkOverrideDiskCachePolicy GlobalOpts{diskCachePolicy} = LSM.OverrideDiskCachePolicy diskCachePolicy
+mkTableConfigOverride :: GlobalOpts -> RunOpts -> LSM.TableConfigOverride
+mkTableConfigOverride GlobalOpts{diskCachePolicy} RunOpts {pipelined} =
+ LSM.noTableConfigOverride {
+ LSM.overrideDiskCachePolicy = Just diskCachePolicy,
+ LSM.overrideMergeBatchSize = if pipelined
+ then Just (LSM.MergeBatchSize 1)
+ else Nothing
+ }
mkTracer :: GlobalOpts -> Tracer IO LSM.LSMTreeTrace
mkTracer gopts
@@ -582,8 +592,10 @@ doRun gopts opts = do
-- reference version starts with empty (as it's not practical or
-- necessary for testing to load the whole snapshot).
tbl <- if check opts
- then LSM.newTableWith @IO @K @V @B (mkTableConfigRun gopts benchTableConfig) session
- else LSM.openTableFromSnapshotWith @IO @K @V @B (mkOverrideDiskCachePolicy gopts) session name label
+ then let conf = mkTableConfigRun gopts opts benchTableConfig
+ in LSM.newTableWith @IO @K @V @B conf session
+ else let conf = mkTableConfigOverride gopts opts
+ in LSM.openTableFromSnapshotWith @IO @K @V @B conf session name label
-- In checking mode, compare each output against a pure reference.
checkvar <- newIORef $ pureReference
diff --git a/lsm-tree.cabal b/lsm-tree.cabal
index f05d07216..bbcd15ce9 100644
--- a/lsm-tree.cabal
+++ b/lsm-tree.cabal
@@ -183,6 +183,12 @@ description:
The /disk cache policy/ determines if lookup operations use the OS page cache.
Caching may improve the performance of lookups and updates if database access follows certain patterns.
+ [@confMergeBatchSize@]
+ The merge batch size balances the maximum latency of individual update
+ operations, versus the latency of a sequence of update operations. Bigger
+ batches improves overall performance but some updates will take a lot
+ longer than others. The default is to use a large batch size.
+
==== Fine-tuning: Merge Policy, Size Ratio, and Write Buffer Size #fine_tuning_data_layout#
The configuration parameters @confMergePolicy@, @confSizeRatio@, and @confWriteBufferAlloc@ affect how the table organises its data.
@@ -429,6 +435,31 @@ description:
* Use the @DiskCacheNone@ policy if the database's access pattern has does not have good spatial or temporal locality.
For instance, if the access pattern is uniformly random.
+ ==== Fine-tuning: Merge Batch Size #fine_tuning_merge_batch_size#
+
+ The /merge batch size/ is a micro-tuning parameter, and in most cases you do
+ need to think about it and can leave it at its default.
+
+ When using the 'Incremental' merge schedule, merging is done in batches. This
+ is a trade-off: larger batches tends to mean better overall performance but the
+ downside is that while most updates (inserts, deletes, upserts) are fast, some
+ are slower (when a batch of merging work has to be done).
+
+ If you care most about the maximum latency of updates, then use a small batch
+ size. If you don't care about latency of individual operations, just the
+ latency of the overall sequence of operations then use a large batch size. The
+ default is to use a large batch size, the same size as the write buffer itself.
+ The minimum batch size is 1. The maximum batch size is the size of the write
+ buffer 'confWriteBufferAlloc'.
+
+ Note that the actual batch size is the minimum of this configuration
+ parameter and the size of the batch of operations performed (e.g. 'inserts').
+ So if you consistently use large batches, you can use a batch size of 1 and
+ the merge batch size will always be determined by the operation batch size.
+
+ A further reason why it may be preferable to use minimal batch sizes is to get
+ good parallel work balance, when using parallelism.
+
== References
The implementation of LSM-trees in this package draws inspiration from:
diff --git a/src-extras/Database/LSMTree/Extras/NoThunks.hs b/src-extras/Database/LSMTree/Extras/NoThunks.hs
index f9d2f7b87..8e9d36857 100644
--- a/src-extras/Database/LSMTree/Extras/NoThunks.hs
+++ b/src-extras/Database/LSMTree/Extras/NoThunks.hs
@@ -659,6 +659,9 @@ deriving anyclass instance NoThunks DiskCachePolicy
deriving stock instance Generic MergeSchedule
deriving anyclass instance NoThunks MergeSchedule
+deriving stock instance Generic MergeBatchSize
+deriving anyclass instance NoThunks MergeBatchSize
+
{-------------------------------------------------------------------------------
RWVar
-------------------------------------------------------------------------------}
diff --git a/src/Database/LSMTree.hs b/src/Database/LSMTree.hs
index 04cd1c412..e53fac48f 100644
--- a/src/Database/LSMTree.hs
+++ b/src/Database/LSMTree.hs
@@ -109,7 +109,8 @@ module Database.LSMTree (
confBloomFilterAlloc,
confFencePointerIndex,
confDiskCachePolicy,
- confMergeSchedule
+ confMergeSchedule,
+ confMergeBatchSize
),
defaultTableConfig,
MergePolicy (LazyLevelling),
@@ -119,9 +120,11 @@ module Database.LSMTree (
BloomFilterAlloc (AllocFixed, AllocRequestFPR),
FencePointerIndexType (OrdinaryIndex, CompactIndex),
DiskCachePolicy (..),
+ MergeBatchSize (..),
-- ** Table Configuration Overrides #table_configuration_overrides#
- OverrideDiskCachePolicy (..),
+ TableConfigOverride (..),
+ noTableConfigOverride,
-- * Ranges #ranges#
Range (..),
@@ -214,11 +217,12 @@ import qualified Database.LSMTree.Internal.BlobRef as Internal
import Database.LSMTree.Internal.Config
(BloomFilterAlloc (AllocFixed, AllocRequestFPR),
DiskCachePolicy (..), FencePointerIndexType (..),
- LevelNo (..), MergePolicy (..), MergeSchedule (..),
- SizeRatio (..), TableConfig (..), WriteBufferAlloc (..),
- defaultTableConfig, serialiseKeyMinimalSize)
+ LevelNo (..), MergeBatchSize (..), MergePolicy (..),
+ MergeSchedule (..), SizeRatio (..), TableConfig (..),
+ WriteBufferAlloc (..), defaultTableConfig,
+ serialiseKeyMinimalSize)
import Database.LSMTree.Internal.Config.Override
- (OverrideDiskCachePolicy (..))
+ (TableConfigOverride (..), noTableConfigOverride)
import Database.LSMTree.Internal.Entry (NumEntries (..))
import qualified Database.LSMTree.Internal.Entry as Entry
import Database.LSMTree.Internal.Merge (LevelMergeType (..))
@@ -2400,7 +2404,7 @@ Variant of 'withTableFromSnapshot' that accepts [table configuration overrides](
withTableFromSnapshotWith ::
forall k v b a.
(ResolveValue v) =>
- OverrideDiskCachePolicy ->
+ TableConfigOverride ->
Session IO ->
SnapshotName ->
SnapshotLabel ->
@@ -2411,7 +2415,7 @@ withTableFromSnapshotWith ::
forall m k v b a.
(IOLike m) =>
(ResolveValue v) =>
- OverrideDiskCachePolicy ->
+ TableConfigOverride ->
Session m ->
SnapshotName ->
SnapshotLabel ->
@@ -2475,7 +2479,7 @@ openTableFromSnapshot ::
SnapshotLabel ->
m (Table m k v b)
openTableFromSnapshot session snapName snapLabel =
- openTableFromSnapshotWith NoOverrideDiskCachePolicy session snapName snapLabel
+ openTableFromSnapshotWith noTableConfigOverride session snapName snapLabel
{- |
Variant of 'openTableFromSnapshot' that accepts [table configuration overrides](#g:table_configuration_overrides).
@@ -2484,7 +2488,7 @@ Variant of 'openTableFromSnapshot' that accepts [table configuration overrides](
openTableFromSnapshotWith ::
forall k v b.
(ResolveValue v) =>
- OverrideDiskCachePolicy ->
+ TableConfigOverride ->
Session IO ->
SnapshotName ->
SnapshotLabel ->
@@ -2494,7 +2498,7 @@ openTableFromSnapshotWith ::
forall m k v b.
(IOLike m) =>
(ResolveValue v) =>
- OverrideDiskCachePolicy ->
+ TableConfigOverride ->
Session m ->
SnapshotName ->
SnapshotLabel ->
diff --git a/src/Database/LSMTree/Internal/Config.hs b/src/Database/LSMTree/Internal/Config.hs
index f0aa6b83f..15d405fdd 100644
--- a/src/Database/LSMTree/Internal/Config.hs
+++ b/src/Database/LSMTree/Internal/Config.hs
@@ -26,12 +26,16 @@ module Database.LSMTree.Internal.Config (
, diskCachePolicyForLevel
-- * Merge schedule
, MergeSchedule (..)
+ -- * Merge batch size
+ , MergeBatchSize (..)
+ , creditThresholdForLevel
) where
import Control.DeepSeq (NFData (..))
import Database.LSMTree.Internal.Index (IndexType)
import qualified Database.LSMTree.Internal.Index as Index
(IndexType (Compact, Ordinary))
+import qualified Database.LSMTree.Internal.MergingRun as MR
import qualified Database.LSMTree.Internal.RawBytes as RB
import Database.LSMTree.Internal.Run (RunDataCaching (..))
import Database.LSMTree.Internal.RunAcc (RunBloomFilterAlloc (..))
@@ -90,6 +94,12 @@ For a detailed discussion of fine-tuning the table configuration, see [Fine-tuni
[@confDiskCachePolicy :: t'DiskCachePolicy'@]
The /disk cache policy/ supports caching lookup operations using the OS page cache.
Caching may improve the performance of lookups and updates if database access follows certain patterns.
+
+[@confMergeBatchSize :: t'MergeBatchSize'@]
+ The merge batch size balances the maximum latency of individual update
+ operations, versus the latency of a sequence of update operations. Bigger
+ batches improves overall performance but some updates will take a lot
+ longer than others. The default is to use a large batch size.
-}
data TableConfig = TableConfig {
confMergePolicy :: !MergePolicy
@@ -99,12 +109,14 @@ data TableConfig = TableConfig {
, confBloomFilterAlloc :: !BloomFilterAlloc
, confFencePointerIndex :: !FencePointerIndexType
, confDiskCachePolicy :: !DiskCachePolicy
+ , confMergeBatchSize :: !MergeBatchSize
}
deriving stock (Show, Eq)
instance NFData TableConfig where
- rnf (TableConfig a b c d e f g) =
- rnf a `seq` rnf b `seq` rnf c `seq` rnf d `seq` rnf e `seq` rnf f `seq` rnf g
+ rnf (TableConfig a b c d e f g h) =
+ rnf a `seq` rnf b `seq` rnf c `seq` rnf d `seq`
+ rnf e `seq` rnf f `seq` rnf g `seq` rnf h
-- | The 'defaultTableConfig' defines reasonable defaults for all 'TableConfig' parameters.
--
@@ -122,6 +134,8 @@ instance NFData TableConfig where
-- OrdinaryIndex
-- >>> confDiskCachePolicy defaultTableConfig
-- DiskCacheAll
+-- >>> confMergeBatchSize defaultTableConfig
+-- MergeBatchSize 20000
--
defaultTableConfig :: TableConfig
defaultTableConfig =
@@ -133,6 +147,7 @@ defaultTableConfig =
, confBloomFilterAlloc = AllocRequestFPR 1.0e-3
, confFencePointerIndex = OrdinaryIndex
, confDiskCachePolicy = DiskCacheAll
+ , confMergeBatchSize = MergeBatchSize 20_000 -- same as write buffer
}
data RunLevelNo = RegularLevel LevelNo | UnionLevel
@@ -238,6 +253,8 @@ data MergeSchedule =
The 'Incremental' merge schedule spreads out the merging work over time.
This is less efficient than the 'OneShot' merge schedule, but has a consistent workload.
Using the 'Incremental' merge schedule, the worst-case disk I\/O complexity of the update operations is /logarithmic/ in the size of the table.
+ This 'Incremental' merge schedule still uses batching to improve performance.
+ The batch size can be controlled using the 'MergeBatchSize'.
-}
| Incremental
deriving stock (Eq, Show)
@@ -385,3 +402,50 @@ diskCachePolicyForLevel policy levelNo =
RegularLevel l | l <= LevelNo n -> CacheRunData
| otherwise -> NoCacheRunData
UnionLevel -> NoCacheRunData
+
+{-------------------------------------------------------------------------------
+ Merge batch size
+-------------------------------------------------------------------------------}
+
+{- |
+The /merge batch size/ is a micro-tuning parameter, and in most cases you do
+need to think about it and can leave it at its default.
+
+When using the 'Incremental' merge schedule, merging is done in batches. This
+is a trade-off: larger batches tends to mean better overall performance but the
+downside is that while most updates (inserts, deletes, upserts) are fast, some
+are slower (when a batch of merging work has to be done).
+
+If you care most about the maximum latency of updates, then use a small batch
+size. If you don't care about latency of individual operations, just the
+latency of the overall sequence of operations then use a large batch size. The
+default is to use a large batch size, the same size as the write buffer itself.
+The minimum batch size is 1. The maximum batch size is the size of the write
+buffer 'confWriteBufferAlloc'.
+
+Note that the actual batch size is the minimum of this configuration
+parameter and the size of the batch of operations performed (e.g. 'inserts').
+So if you consistently use large batches, you can use a batch size of 1 and
+the merge batch size will always be determined by the operation batch size.
+
+A further reason why it may be preferable to use minimal batch sizes is to get
+good parallel work balance, when using parallelism.
+-}
+newtype MergeBatchSize = MergeBatchSize Int
+ deriving stock (Show, Eq, Ord)
+ deriving newtype (NFData)
+
+-- TODO: the thresholds for doing merge work should be different for each level,
+-- and ideally all-pairs co-prime.
+creditThresholdForLevel :: TableConfig -> LevelNo -> MR.CreditThreshold
+creditThresholdForLevel TableConfig {
+ confMergeBatchSize = MergeBatchSize mergeBatchSz,
+ confWriteBufferAlloc = AllocNumEntries writeBufferSz
+ }
+ (LevelNo _i) =
+ MR.CreditThreshold
+ . MR.UnspentCredits
+ . MR.MergeCredits
+ . max 1
+ . min writeBufferSz
+ $ mergeBatchSz
diff --git a/src/Database/LSMTree/Internal/Config/Override.hs b/src/Database/LSMTree/Internal/Config/Override.hs
index a2e7d5877..9000d73b6 100644
--- a/src/Database/LSMTree/Internal/Config/Override.hs
+++ b/src/Database/LSMTree/Internal/Config/Override.hs
@@ -5,9 +5,10 @@
module Database.LSMTree.Internal.Config.Override (
-- $override-policy
- -- * Override disk cache policy
- OverrideDiskCachePolicy (..)
- , overrideDiskCachePolicy
+ -- * Override table config
+ TableConfigOverride (..)
+ , noTableConfigOverride
+ , overrideTableConfig
) where
import qualified Data.Vector as V
@@ -42,35 +43,75 @@ import Database.LSMTree.Internal.Snapshot
-- Another complicating factor is that we have thought about the possibility of
-- restoring sharing of ongoing merges between live tables and newly opened
-- snapshots. At that point, we run into the same challenges again... But for
--- now, changing only the disk cache policy offline should work fine.
+-- now, changing only the disk cache policy and merge batch size offline should
+-- work fine.
{-------------------------------------------------------------------------------
- Override disk cache policy
+ Helper class
+-------------------------------------------------------------------------------}
+
+-- | This class is only here so that we can recursively call 'override' on all
+-- fields of a datatype, instead of having to invent a new name for each type
+-- that the function is called on such as @overrideTableConfig@,
+-- @overrideSnapshotRun@, etc.
+class Override o a where
+ override :: o -> a -> a
+
+instance Override a c => Override (Maybe a) c where
+ override = maybe id override
+
+{-------------------------------------------------------------------------------
+ Override table config
-------------------------------------------------------------------------------}
{- |
-The 'OverrideDiskCachePolicy' can be used to override the 'DiskCachePolicy'
+The 'TableConfigOverride' can be used to override the 'TableConfig'
when opening a table from a snapshot.
-}
-data OverrideDiskCachePolicy =
- NoOverrideDiskCachePolicy
- | OverrideDiskCachePolicy DiskCachePolicy
+data TableConfigOverride = TableConfigOverride {
+ overrideDiskCachePolicy :: Maybe DiskCachePolicy,
+ overrideMergeBatchSize :: Maybe MergeBatchSize
+ }
deriving stock (Show, Eq)
--- | Override the disk cache policy that is stored in snapshot metadata.
+-- | No override of the 'TableConfig'. You can use this as a default value and
+-- record update to override some parameters, while being future-proof to new
+-- parameters, e.g.
+--
+-- > noTableConfigOverride { overrideDiskCachePolicy = DiskCacheNone }
+--
+noTableConfigOverride :: TableConfigOverride
+noTableConfigOverride = TableConfigOverride Nothing Nothing
+
+-- | Override the a subset of the table configuration parameters that are
+-- stored in snapshot metadata.
--
-- Tables opened from the new 'SnapshotMetaData' will use the new value for the
--- disk cache policy.
-overrideDiskCachePolicy :: OverrideDiskCachePolicy -> SnapshotMetaData -> SnapshotMetaData
-overrideDiskCachePolicy (OverrideDiskCachePolicy dcp) = override dcp
-overrideDiskCachePolicy NoOverrideDiskCachePolicy = id
+-- table configuration.
+overrideTableConfig :: TableConfigOverride
+ -> SnapshotMetaData -> SnapshotMetaData
+overrideTableConfig = override
--- | This class is only here so that we can recursively call 'override' on all
--- fields of a datatype, instead of having to invent a new name for each type
--- that the function is called on such as 'overrideTableConfig',
--- 'overrideSnapshotRun', etc.
-class Override o a where
- override :: o -> a -> a
+instance Override TableConfigOverride SnapshotMetaData where
+ override TableConfigOverride {..} =
+ override overrideMergeBatchSize
+ . override overrideDiskCachePolicy
+
+{-------------------------------------------------------------------------------
+ Override merge batch size
+-------------------------------------------------------------------------------}
+
+instance Override MergeBatchSize SnapshotMetaData where
+ override mbs smd =
+ smd { snapMetaConfig = override mbs (snapMetaConfig smd) }
+
+instance Override MergeBatchSize TableConfig where
+ override confMergeBatchSize' tc =
+ tc { confMergeBatchSize = confMergeBatchSize' }
+
+{-------------------------------------------------------------------------------
+ Override disk cache policy
+-------------------------------------------------------------------------------}
-- NOTE: the instances below explicitly pattern match on the types of
-- constructor fields. This makes the code more verbose, but it also makes the
@@ -91,16 +132,8 @@ instance Override DiskCachePolicy SnapshotMetaData where
in fmap (override rdc) smt
instance Override DiskCachePolicy TableConfig where
- override confDiskCachePolicy' TableConfig {..}
- = TableConfig
- { confMergePolicy,
- confMergeSchedule,
- confSizeRatio,
- confWriteBufferAlloc,
- confBloomFilterAlloc,
- confFencePointerIndex,
- confDiskCachePolicy = confDiskCachePolicy'
- }
+ override confDiskCachePolicy' tc =
+ tc { confDiskCachePolicy = confDiskCachePolicy' }
instance Override DiskCachePolicy (SnapLevels SnapshotRun) where
override dcp (SnapLevels (vec :: V.Vector (SnapLevel SnapshotRun))) =
diff --git a/src/Database/LSMTree/Internal/IncomingRun.hs b/src/Database/LSMTree/Internal/IncomingRun.hs
index 70f9222a6..665a126f2 100644
--- a/src/Database/LSMTree/Internal/IncomingRun.hs
+++ b/src/Database/LSMTree/Internal/IncomingRun.hs
@@ -218,13 +218,6 @@ supplyCreditsIncomingRun conf ln (Merging _ nominalDebt nominalCreditsVar mr)
-- use atomic operations for its counters). We could potentially simplify
-- MergingRun by dispensing with batching for the MergeCredits counters.
--- TODO: the thresholds for doing merge work should be different for each level,
--- maybe co-prime?
-creditThresholdForLevel :: TableConfig -> LevelNo -> MR.CreditThreshold
-creditThresholdForLevel conf (LevelNo _i) =
- let AllocNumEntries x = confWriteBufferAlloc conf
- in MR.CreditThreshold (MR.UnspentCredits (MergeCredits x))
-
-- | Deposit nominal credits in the local credits var, ensuring the total
-- credits does not exceed the total debt.
--
diff --git a/src/Database/LSMTree/Internal/Snapshot/Codec.hs b/src/Database/LSMTree/Internal/Snapshot/Codec.hs
index 61c5d3c6a..5dbc4100b 100644
--- a/src/Database/LSMTree/Internal/Snapshot/Codec.hs
+++ b/src/Database/LSMTree/Internal/Snapshot/Codec.hs
@@ -57,23 +57,25 @@ import Text.Printf
-- for more. Forwards compatibility is not provided at all: snapshots with a
-- later version than the current version for the library release will always
-- fail.
-data SnapshotVersion = V0
- deriving stock (Show, Eq)
+data SnapshotVersion = V0 | V1
+ deriving stock (Show, Eq, Ord)
-- >>> prettySnapshotVersion currentSnapshotVersion
--- "v0"
+-- "v1"
prettySnapshotVersion :: SnapshotVersion -> String
prettySnapshotVersion V0 = "v0"
+prettySnapshotVersion V1 = "v1"
-- >>> currentSnapshotVersion
-- V0
currentSnapshotVersion :: SnapshotVersion
-currentSnapshotVersion = V0
+currentSnapshotVersion = V1
isCompatible :: SnapshotVersion -> Either String ()
-isCompatible otherVersion = do
- case ( currentSnapshotVersion, otherVersion ) of
- (V0, V0) -> Right ()
+isCompatible otherVersion
+ -- for the moment, all versions are backwards compatible:
+ | currentSnapshotVersion >= otherVersion = Right ()
+ | otherwise = Left "forward compatibility not supported"
{-------------------------------------------------------------------------------
Writing and reading files
@@ -197,6 +199,7 @@ instance Encode SnapshotVersion where
encodeListLen 1
<> case ver of
V0 -> encodeWord 0
+ V1 -> encodeWord 1
instance Decode SnapshotVersion where
decode = do
@@ -204,6 +207,7 @@ instance Decode SnapshotVersion where
ver <- decodeWord
case ver of
0 -> pure V0
+ 1 -> pure V1
_ -> fail ("Unknown snapshot format version number: " <> show ver)
{-------------------------------------------------------------------------------
@@ -222,7 +226,7 @@ instance Encode SnapshotMetaData where
<> encodeMaybe mergingTree
instance DecodeVersioned SnapshotMetaData where
- decodeVersioned ver@V0 = do
+ decodeVersioned ver = do
_ <- decodeListLenOf 5
SnapshotMetaData
<$> decodeVersioned ver
@@ -237,7 +241,7 @@ instance Encode SnapshotLabel where
encode (SnapshotLabel s) = encodeString s
instance DecodeVersioned SnapshotLabel where
- decodeVersioned V0 = SnapshotLabel <$> decodeString
+ decodeVersioned _v = SnapshotLabel <$> decodeString
instance Encode SnapshotRun where
encode SnapshotRun { snapRunNumber, snapRunCaching, snapRunIndex } =
@@ -248,7 +252,7 @@ instance Encode SnapshotRun where
<> encode snapRunIndex
instance DecodeVersioned SnapshotRun where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -274,9 +278,10 @@ instance Encode TableConfig where
, confBloomFilterAlloc = bloomFilterAlloc
, confFencePointerIndex = fencePointerIndex
, confDiskCachePolicy = diskCachePolicy
+ , confMergeBatchSize = mergeBatchSize
}
) =
- encodeListLen 7
+ encodeListLen 8
<> encode mergePolicy
<> encode mergeSchedule
<> encode sizeRatio
@@ -284,10 +289,25 @@ instance Encode TableConfig where
<> encode bloomFilterAlloc
<> encode fencePointerIndex
<> encode diskCachePolicy
+ <> encode mergeBatchSize
instance DecodeVersioned TableConfig where
decodeVersioned v@V0 = do
- _ <- decodeListLenOf 7
+ decodeListLenOf 7
+ confMergePolicy <- decodeVersioned v
+ confMergeSchedule <- decodeVersioned v
+ confSizeRatio <- decodeVersioned v
+ confWriteBufferAlloc <- decodeVersioned v
+ confBloomFilterAlloc <- decodeVersioned v
+ confFencePointerIndex <- decodeVersioned v
+ confDiskCachePolicy <- decodeVersioned v
+ let confMergeBatchSize = case confWriteBufferAlloc of
+ AllocNumEntries n -> MergeBatchSize n
+ pure TableConfig {..}
+
+ -- We introduced the confMergeBatchSize in V1
+ decodeVersioned v@V1 = do
+ decodeListLenOf 8
confMergePolicy <- decodeVersioned v
confMergeSchedule <- decodeVersioned v
confSizeRatio <- decodeVersioned v
@@ -295,6 +315,7 @@ instance DecodeVersioned TableConfig where
confBloomFilterAlloc <- decodeVersioned v
confFencePointerIndex <- decodeVersioned v
confDiskCachePolicy <- decodeVersioned v
+ confMergeBatchSize <- decodeVersioned v
pure TableConfig {..}
-- MergePolicy
@@ -303,7 +324,7 @@ instance Encode MergePolicy where
encode LazyLevelling = encodeWord 0
instance DecodeVersioned MergePolicy where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure LazyLevelling
@@ -315,7 +336,7 @@ instance Encode SizeRatio where
encode Four = encodeInt 4
instance DecodeVersioned SizeRatio where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
x <- decodeWord64
case x of
4 -> pure Four
@@ -330,7 +351,7 @@ instance Encode WriteBufferAlloc where
<> encodeInt numEntries
instance DecodeVersioned WriteBufferAlloc where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
_ <- decodeListLenOf 2
tag <- decodeWord
case tag of
@@ -348,7 +369,7 @@ instance Encode RunParams where
<> encode runParamIndex
instance DecodeVersioned RunParams where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -363,7 +384,7 @@ instance Encode RunDataCaching where
encode NoCacheRunData = encodeWord 1
instance DecodeVersioned RunDataCaching where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure CacheRunData
@@ -375,7 +396,7 @@ instance Encode IndexType where
encode Compact = encodeWord 1
instance DecodeVersioned IndexType where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure Ordinary
@@ -393,7 +414,7 @@ instance Encode RunBloomFilterAlloc where
<> encodeDouble fpr
instance DecodeVersioned RunBloomFilterAlloc where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -414,7 +435,7 @@ instance Encode BloomFilterAlloc where
<> encodeDouble x
instance DecodeVersioned BloomFilterAlloc where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -429,7 +450,7 @@ instance Encode FencePointerIndexType where
encode OrdinaryIndex = encodeWord 1
instance DecodeVersioned FencePointerIndexType where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure CompactIndex
@@ -451,7 +472,7 @@ instance Encode DiskCachePolicy where
<> encodeWord 2
instance DecodeVersioned DiskCachePolicy where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -467,13 +488,21 @@ instance Encode MergeSchedule where
encode Incremental = encodeWord 1
instance DecodeVersioned MergeSchedule where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure OneShot
1 -> pure Incremental
_ -> fail ("[MergeSchedule] Unexpected tag: " <> show tag)
+-- MergeBatchSize
+
+instance Encode MergeBatchSize where
+ encode (MergeBatchSize n) = encodeInt n
+
+instance DecodeVersioned MergeBatchSize where
+ decodeVersioned _v = MergeBatchSize <$> decodeInt
+
{-------------------------------------------------------------------------------
Encoding and decoding: SnapLevels
-------------------------------------------------------------------------------}
@@ -484,7 +513,7 @@ instance Encode r => Encode (SnapLevels r) where
encode (SnapLevels levels) = encode levels
instance DecodeVersioned r => DecodeVersioned (SnapLevels r) where
- decodeVersioned v@V0 = SnapLevels <$> decodeVersioned v
+ decodeVersioned v = SnapLevels <$> decodeVersioned v
-- SnapLevel
@@ -496,7 +525,7 @@ instance Encode r => Encode (SnapLevel r) where
instance DecodeVersioned r => DecodeVersioned (SnapLevel r) where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
_ <- decodeListLenOf 2
SnapLevel <$> decodeVersioned v <*> decodeVersioned v
@@ -514,7 +543,7 @@ instance Encode RunNumber where
encode (RunNumber x) = encodeInt x
instance DecodeVersioned RunNumber where
- decodeVersioned V0 = RunNumber <$> decodeInt
+ decodeVersioned _v = RunNumber <$> decodeInt
-- SnapIncomingRun
@@ -532,7 +561,7 @@ instance Encode r => Encode (SnapIncomingRun r) where
<> encode x
instance DecodeVersioned r => DecodeVersioned (SnapIncomingRun r) where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -549,7 +578,7 @@ instance Encode MergePolicyForLevel where
encode LevelLevelling = encodeWord 1
instance DecodeVersioned MergePolicyForLevel where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure LevelTiering
@@ -573,7 +602,7 @@ instance (Encode t, Encode r) => Encode (SnapMergingRun t r) where
<> encode mt
instance (DecodeVersioned t, DecodeVersioned r) => DecodeVersioned (SnapMergingRun t r) where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -589,25 +618,25 @@ instance Encode NominalDebt where
encode (NominalDebt x) = encodeInt x
instance DecodeVersioned NominalDebt where
- decodeVersioned V0 = NominalDebt <$> decodeInt
+ decodeVersioned _v = NominalDebt <$> decodeInt
instance Encode NominalCredits where
encode (NominalCredits x) = encodeInt x
instance DecodeVersioned NominalCredits where
- decodeVersioned V0 = NominalCredits <$> decodeInt
+ decodeVersioned _v = NominalCredits <$> decodeInt
instance Encode MergeDebt where
encode (MergeDebt (MergeCredits x)) = encodeInt x
instance DecodeVersioned MergeDebt where
- decodeVersioned V0 = (MergeDebt . MergeCredits) <$> decodeInt
+ decodeVersioned _v = (MergeDebt . MergeCredits) <$> decodeInt
instance Encode MergeCredits where
encode (MergeCredits x) = encodeInt x
instance DecodeVersioned MergeCredits where
- decodeVersioned V0 = MergeCredits <$> decodeInt
+ decodeVersioned _v = MergeCredits <$> decodeInt
-- MergeType
@@ -616,7 +645,7 @@ instance Encode MR.LevelMergeType where
encode MR.MergeLastLevel = encodeWord 1
instance DecodeVersioned MR.LevelMergeType where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
0 -> pure MR.MergeMidLevel
@@ -638,7 +667,7 @@ instance Encode MR.TreeMergeType where
encode MR.MergeUnion = encodeWord 2
instance DecodeVersioned MR.TreeMergeType where
- decodeVersioned V0 = do
+ decodeVersioned _v = do
tag <- decodeWord
case tag of
1 -> pure MR.MergeLevel
@@ -655,7 +684,7 @@ instance Encode r => Encode (SnapMergingTree r) where
encode (SnapMergingTree tState) = encode tState
instance DecodeVersioned r => DecodeVersioned (SnapMergingTree r) where
- decodeVersioned ver@V0 = SnapMergingTree <$> decodeVersioned ver
+ decodeVersioned ver = SnapMergingTree <$> decodeVersioned ver
-- SnapMergingTreeState
@@ -674,7 +703,7 @@ instance Encode r => Encode (SnapMergingTreeState r) where
<> encode smrs
instance DecodeVersioned r => DecodeVersioned (SnapMergingTreeState r) where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -697,7 +726,7 @@ instance Encode r => Encode (SnapPendingMerge r) where
<> encodeList mts
instance DecodeVersioned r => DecodeVersioned (SnapPendingMerge r) where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -718,7 +747,7 @@ instance Encode r => Encode (SnapPreExistingRun r) where
<> encode smrs
instance DecodeVersioned r => DecodeVersioned (SnapPreExistingRun r) where
- decodeVersioned v@V0 = do
+ decodeVersioned v = do
n <- decodeListLen
tag <- decodeWord
case (n, tag) of
@@ -736,7 +765,7 @@ encodeMaybe = \case
Just en -> encode en
decodeMaybe :: DecodeVersioned a => SnapshotVersion -> Decoder s (Maybe a)
-decodeMaybe v@V0 = do
+decodeMaybe v = do
tok <- peekTokenType
case tok of
TypeNull -> Nothing <$ decodeNull
diff --git a/src/Database/LSMTree/Internal/Unsafe.hs b/src/Database/LSMTree/Internal/Unsafe.hs
index 0ab05717d..523d540ed 100644
--- a/src/Database/LSMTree/Internal/Unsafe.hs
+++ b/src/Database/LSMTree/Internal/Unsafe.hs
@@ -109,8 +109,8 @@ import Database.LSMTree.Internal.Arena (ArenaManager, newArenaManager)
import Database.LSMTree.Internal.BlobRef (WeakBlobRef (..))
import qualified Database.LSMTree.Internal.BlobRef as BlobRef
import Database.LSMTree.Internal.Config
-import Database.LSMTree.Internal.Config.Override
- (OverrideDiskCachePolicy, overrideDiskCachePolicy)
+import Database.LSMTree.Internal.Config.Override (TableConfigOverride,
+ overrideTableConfig)
import Database.LSMTree.Internal.CRC32C (FileCorruptedError (..),
FileFormat (..))
import qualified Database.LSMTree.Internal.Cursor as Cursor
@@ -159,7 +159,7 @@ data LSMTreeTrace =
| TraceCloseSession
-- Table
| TraceNewTable
- | TraceOpenTableFromSnapshot SnapshotName OverrideDiskCachePolicy
+ | TraceOpenTableFromSnapshot SnapshotName TableConfigOverride
| TraceTable TableId TableTrace
| TraceDeleteSnapshot SnapshotName
| TraceListSnapshots
@@ -1287,7 +1287,7 @@ data SnapshotNotCompatibleError
deriving anyclass (Exception)
{-# SPECIALISE openTableFromSnapshot ::
- OverrideDiskCachePolicy
+ TableConfigOverride
-> Session IO h
-> SnapshotName
-> SnapshotLabel
@@ -1296,7 +1296,7 @@ data SnapshotNotCompatibleError
-- | See 'Database.LSMTree.openTableFromSnapshot'.
openTableFromSnapshot ::
(MonadMask m, MonadMVar m, MonadST m, MonadSTM m)
- => OverrideDiskCachePolicy
+ => TableConfigOverride
-> Session m h
-> SnapshotName
-> SnapshotLabel -- ^ Expected label
@@ -1322,7 +1322,7 @@ openTableFromSnapshot policyOveride sesh snap label resolve =
snapMetaData <- readFileSnapshotMetaData hfs contentPath checksumPath
let SnapshotMetaData label' conf snapWriteBuffer snapLevels mTreeOpt
- = overrideDiskCachePolicy policyOveride snapMetaData
+ = overrideTableConfig policyOveride snapMetaData
unless (label == label') $
throwIO (ErrSnapshotWrongLabel snap label label')
diff --git a/src/Database/LSMTree/Simple.hs b/src/Database/LSMTree/Simple.hs
index 67c4545a1..e7aaf15a3 100644
--- a/src/Database/LSMTree/Simple.hs
+++ b/src/Database/LSMTree/Simple.hs
@@ -111,9 +111,11 @@ module Database.LSMTree.Simple (
FencePointerIndexType (OrdinaryIndex, CompactIndex),
DiskCachePolicy (..),
MergeSchedule (..),
+ MergeBatchSize (..),
-- ** Table Configuration Overrides #table_configuration_overrides#
- OverrideDiskCachePolicy (..),
+ TableConfigOverride (..),
+ noTableConfigOverride,
-- * Ranges #ranges#
Range (..),
@@ -165,18 +167,18 @@ import Data.Vector (Vector)
import Data.Void (Void)
import Database.LSMTree (BloomFilterAlloc, CursorClosedError (..),
DiskCachePolicy, FencePointerIndexType,
- InvalidSnapshotNameError (..), MergePolicy, MergeSchedule,
- OverrideDiskCachePolicy (..), Range (..), RawBytes,
- ResolveAsFirst (..), SerialiseKey (..),
- SerialiseKeyOrderPreserving, SerialiseValue (..),
- SessionClosedError (..), SizeRatio,
+ InvalidSnapshotNameError (..), MergeBatchSize, MergePolicy,
+ MergeSchedule, Range (..), RawBytes, ResolveAsFirst (..),
+ SerialiseKey (..), SerialiseKeyOrderPreserving,
+ SerialiseValue (..), SessionClosedError (..), SizeRatio,
SnapshotCorruptedError (..),
SnapshotDoesNotExistError (..), SnapshotExistsError (..),
SnapshotLabel (..), SnapshotName,
SnapshotNotCompatibleError (..), TableClosedError (..),
- TableConfig (..), TableCorruptedError (..),
- TableTooLargeError (..), UnionCredits (..), UnionDebt (..),
- WriteBufferAlloc, isValidSnapshotName, packSlice,
+ TableConfig (..), TableConfigOverride (..),
+ TableCorruptedError (..), TableTooLargeError (..),
+ UnionCredits (..), UnionDebt (..), WriteBufferAlloc,
+ isValidSnapshotName, noTableConfigOverride, packSlice,
serialiseKeyIdentity, serialiseKeyIdentityUpToSlicing,
serialiseKeyMinimalSize, serialiseKeyPreservesOrdering,
serialiseValueIdentity, serialiseValueIdentityUpToSlicing,
@@ -1424,7 +1426,7 @@ Variant of 'withTableFromSnapshot' that accepts [table configuration overrides](
-}
withTableFromSnapshotWith ::
forall k v a.
- OverrideDiskCachePolicy ->
+ TableConfigOverride ->
Session ->
SnapshotName ->
SnapshotLabel ->
@@ -1467,7 +1469,7 @@ Variant of 'openTableFromSnapshot' that accepts [table configuration overrides](
-}
openTableFromSnapshotWith ::
forall k v.
- OverrideDiskCachePolicy ->
+ TableConfigOverride ->
Session ->
SnapshotName ->
SnapshotLabel ->
diff --git a/test/Test/Database/LSMTree/Internal/Snapshot/Codec.hs b/test/Test/Database/LSMTree/Internal/Snapshot/Codec.hs
index 39794d02d..a5a1731e2 100644
--- a/test/Test/Database/LSMTree/Internal/Snapshot/Codec.hs
+++ b/test/Test/Database/LSMTree/Internal/Snapshot/Codec.hs
@@ -191,8 +191,9 @@ testAll test = [
-------------------------------------------------------------------------------}
instance Arbitrary SnapshotVersion where
- arbitrary = elements [V0]
+ arbitrary = elements [V0, V1]
shrink V0 = []
+ shrink V1 = [V0]
deriving newtype instance Arbitrary a => Arbitrary (Versioned a)
@@ -230,11 +231,11 @@ instance Arbitrary SnapshotRun where
instance Arbitrary TableConfig where
arbitrary =
- TableConfig <$> arbitrary <*> arbitrary <*> arbitrary
- <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
- shrink (TableConfig a b c d e f g) =
- [ TableConfig a' b' c' d' e' f' g'
- | (a', b', c', d', e', f', g') <- shrink (a, b, c, d, e, f, g) ]
+ TableConfig <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
+ <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
+ shrink (TableConfig a b c d e f g h) =
+ [ TableConfig a' b' c' d' e' f' g' h'
+ | (a', b', c', d', e', f', g', h') <- shrink (a, b, c, d, e, f, g, h) ]
instance Arbitrary MergePolicy where
arbitrary = pure LazyLevelling
@@ -273,6 +274,10 @@ instance Arbitrary MergeSchedule where
arbitrary = elements [OneShot, Incremental]
shrink _ = []
+instance Arbitrary MergeBatchSize where
+ arbitrary = MergeBatchSize <$> arbitrary
+ shrink (MergeBatchSize n) = map MergeBatchSize (shrink n)
+
{-------------------------------------------------------------------------------
Arbitrary: SnapLevels
-------------------------------------------------------------------------------}
diff --git a/test/Test/Database/LSMTree/Internal/Snapshot/Codec/Golden.hs b/test/Test/Database/LSMTree/Internal/Snapshot/Codec/Golden.hs
index e327816af..ec629e8ec 100644
--- a/test/Test/Database/LSMTree/Internal/Snapshot/Codec/Golden.hs
+++ b/test/Test/Database/LSMTree/Internal/Snapshot/Codec/Golden.hs
@@ -15,8 +15,8 @@ import Data.Typeable
import qualified Data.Vector as V
import Database.LSMTree.Internal.Config (BloomFilterAlloc (..),
DiskCachePolicy (..), FencePointerIndexType (..),
- MergePolicy (..), MergeSchedule (..), SizeRatio (..),
- TableConfig (..), WriteBufferAlloc (..))
+ MergeBatchSize (..), MergePolicy (..), MergeSchedule (..),
+ SizeRatio (..), TableConfig (..), WriteBufferAlloc (..))
import Database.LSMTree.Internal.MergeSchedule
(MergePolicyForLevel (..), NominalCredits (..),
NominalDebt (..))
@@ -153,6 +153,7 @@ forallSnapshotTypes f = [
, f (Proxy @FencePointerIndexType)
, f (Proxy @DiskCachePolicy)
, f (Proxy @MergeSchedule)
+ , f (Proxy @MergeBatchSize)
-- SnapLevels
, f (Proxy @(SnapLevels SnapshotRun))
, f (Proxy @(SnapLevel SnapshotRun))
@@ -276,7 +277,8 @@ instance EnumGolden SnapshotLabel where
SnapshotLabel{} -> ()
instance EnumGolden TableConfig where
- singGolden = TableConfig singGolden singGolden singGolden singGolden singGolden singGolden singGolden
+ singGolden = TableConfig singGolden singGolden singGolden singGolden
+ singGolden singGolden singGolden singGolden
where
_coveredAllCases = \case
TableConfig{} -> ()
@@ -329,6 +331,9 @@ instance EnumGolden MergeSchedule where
OneShot{} -> ()
Incremental{} -> ()
+instance EnumGolden MergeBatchSize where
+ enumGolden = map MergeBatchSize [ 1, 1000 ]
+
instance EnumGolden (SnapLevels SnapshotRun) where
singGolden = SnapLevels singGolden
where
diff --git a/test/Test/Database/LSMTree/Internal/Snapshot/FS.hs b/test/Test/Database/LSMTree/Internal/Snapshot/FS.hs
index 7d0a3bee6..29fd05648 100644
--- a/test/Test/Database/LSMTree/Internal/Snapshot/FS.hs
+++ b/test/Test/Database/LSMTree/Internal/Snapshot/FS.hs
@@ -13,7 +13,7 @@ import Database.LSMTree.Extras (showPowersOf10)
import Database.LSMTree.Extras.Generators ()
import Database.LSMTree.Internal.Config
import Database.LSMTree.Internal.Config.Override
- (OverrideDiskCachePolicy (..))
+ (noTableConfigOverride)
import Database.LSMTree.Internal.Entry
import Database.LSMTree.Internal.Paths
import Database.LSMTree.Internal.Serialise
@@ -221,6 +221,6 @@ prop_flipSnapshotBit (Positive (Small bufferSize)) es pickFileBit =
saveSnapshot snapName snapLabel t
openSnap s =
- openTableFromSnapshot NoOverrideDiskCachePolicy s snapName snapLabel resolve
+ openTableFromSnapshot noTableConfigOverride s snapName snapLabel resolve
getConstructorName e = takeWhile (/= ' ') (show e)
diff --git a/test/Test/Database/LSMTree/StateMachine.hs b/test/Test/Database/LSMTree/StateMachine.hs
index 109d98434..ee93f0436 100644
--- a/test/Test/Database/LSMTree/StateMachine.hs
+++ b/test/Test/Database/LSMTree/StateMachine.hs
@@ -94,8 +94,7 @@ import qualified Database.LSMTree.Class as Class
import Database.LSMTree.Extras (showPowersOf)
import Database.LSMTree.Extras.Generators (KeyForIndexCompact)
import Database.LSMTree.Extras.NoThunks (propNoThunks)
-import qualified Database.LSMTree.Internal.Config as R
- (TableConfig (TableConfig))
+import qualified Database.LSMTree.Internal.Config as R (TableConfig (..))
import Database.LSMTree.Internal.Serialise (SerialisedBlob,
SerialisedValue)
import qualified Database.LSMTree.Internal.Types as R.Types
@@ -226,6 +225,8 @@ instance Arbitrary R.TableConfig where
]
confWriteBufferAlloc <- QC.arbitrary
confFencePointerIndex <- QC.arbitrary
+ confMergeBatchSize <- QC.sized $ \sz ->
+ R.MergeBatchSize <$> QC.chooseInt (1, sz)
pure $ R.TableConfig {
R.confMergePolicy = R.LazyLevelling
, R.confSizeRatio = R.Four
@@ -234,6 +235,7 @@ instance Arbitrary R.TableConfig where
, confFencePointerIndex
, R.confDiskCachePolicy = R.DiskCacheNone
, confMergeSchedule
+ , confMergeBatchSize
}
shrink R.TableConfig{..} =
diff --git a/test/Test/Database/LSMTree/StateMachine/DL.hs b/test/Test/Database/LSMTree/StateMachine/DL.hs
index 82750a560..0e452bff6 100644
--- a/test/Test/Database/LSMTree/StateMachine/DL.hs
+++ b/test/Test/Database/LSMTree/StateMachine/DL.hs
@@ -13,8 +13,7 @@ import Control.Tracer
import qualified Data.Map.Strict as Map
import qualified Data.Vector as V
import Database.LSMTree as R
-import qualified Database.LSMTree.Internal.Config as R
- (TableConfig (TableConfig))
+import qualified Database.LSMTree.Internal.Config as R (TableConfig (..))
import qualified Database.LSMTree.Model.Session as Model (fromSomeTable, tables)
import qualified Database.LSMTree.Model.Table as Model (values)
import Prelude
@@ -75,7 +74,9 @@ dl_example = do
, confBloomFilterAlloc = AllocFixed 10
, confFencePointerIndex = OrdinaryIndex
, confDiskCachePolicy = DiskCacheNone
- , confMergeSchedule = OneShot })
+ , confMergeSchedule = OneShot
+ , confMergeBatchSize = MergeBatchSize 4
+ })
let kvs :: Map.Map Key Value
kvs = Map.fromList $
QC.unGen (QC.vectorOf 37 $ (,) <$> QC.arbitrary <*> QC.arbitrary)
diff --git a/test/golden-file-data/snapshot-codec/SnapshotMetaData.A.snapshot.golden b/test/golden-file-data/snapshot-codec/SnapshotMetaData.A.snapshot.golden
index 3ab235bc5..e6cba0b49 100644
Binary files a/test/golden-file-data/snapshot-codec/SnapshotMetaData.A.snapshot.golden and b/test/golden-file-data/snapshot-codec/SnapshotMetaData.A.snapshot.golden differ
diff --git a/test/golden-file-data/snapshot-codec/TableConfig.A.snapshot.golden b/test/golden-file-data/snapshot-codec/TableConfig.A.snapshot.golden
index 0020ff3e5..77830a933 100644
Binary files a/test/golden-file-data/snapshot-codec/TableConfig.A.snapshot.golden and b/test/golden-file-data/snapshot-codec/TableConfig.A.snapshot.golden differ