Omit expired records from SCAN (#323)

microsoft · Apr 25, 2024 · e2f3613 · e2f3613
1 parent 47da479
commit e2f3613
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 4 deletions.
diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs
@@ -149,6 +149,8 @@ internal long SnapToLogicalAddressBoundary(ref long logicalAddress, long headAdd
             return logicalAddress += totalSizes - offset;
         }
 
+        private bool CheckExpiry(ref SpanByte value) => value.MetadataSize > 0 && value.ExtraMetadata < DateTimeOffset.Now.UtcTicks;
+
         /// <summary>
         /// Get next record in iterator
         /// </summary>
@@ -177,7 +179,8 @@ public unsafe bool GetNext(out RecordInfo recordInfo)
                 nextAddress = currentAddress + recordSize;
 
                 recordInfo = hlog.GetInfo(physicalAddress);
-                bool skipOnScan = includeSealedRecords ? recordInfo.Invalid : recordInfo.SkipOnScan;
+                bool skipOnScan = (includeSealedRecords ? recordInfo.Invalid : recordInfo.SkipOnScan)
+                                || CheckExpiry(ref hlog.GetValue(physicalAddress));
                 if (skipOnScan || recordInfo.IsNull())
                 {
                     epoch?.Suspend();
@@ -249,7 +252,8 @@ bool IPushScanIterator<SpanByte>.BeginGetPrevInMemory(ref SpanByte key, out Reco
 
                 recordInfo = hlog.GetInfo(physicalAddress);
                 nextAddress = recordInfo.PreviousAddress;
-                bool skipOnScan = includeSealedRecords ? recordInfo.Invalid : recordInfo.SkipOnScan;
+                bool skipOnScan = (includeSealedRecords ? recordInfo.Invalid : recordInfo.SkipOnScan)
+                                || CheckExpiry(ref hlog.GetValue(physicalAddress));
                 if (skipOnScan || recordInfo.IsNull() || !comparer.Equals(ref hlog.GetKey(physicalAddress), ref key))
                 {
                     epoch?.Suspend();

diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs
@@ -183,11 +183,19 @@ public void SetExpiry()
             db.StringSet("mykey", origValue, TimeSpan.FromSeconds(1));
 
             string retValue = db.StringGet("mykey");
-            Assert.AreEqual(origValue, retValue);
+            Assert.AreEqual(origValue, retValue, "Expected to Get() the value");
+
+            var actualDbSize = db.Execute("DBSIZE");
+            Assert.AreEqual(1, (ulong)actualDbSize, "Expected DBSIZE");
 
+            // Sleep to allow expiration
             Thread.Sleep(2000);
+
             retValue = db.StringGet("mykey");
-            Assert.AreEqual(null, retValue);
+            Assert.AreEqual(null, retValue, "Expected null value due to expiration");
+
+            actualDbSize = db.Execute("DBSIZE");
+            Assert.AreEqual(0, (ulong)actualDbSize, "Expected DBSIZE of zero due to expiration");
         }
 
         [Test]