HDDS-14225. Upgrade RocksDB from 7.7.3 to 10.10.1

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBProfile.java

@@ -87,6 +87,7 @@ public ManagedBlockBasedTableConfig getBlockBasedTableConfig() {
       ManagedBlockBasedTableConfig config = new ManagedBlockBasedTableConfig();
       config.setBlockCache(new ManagedLRUCache(blockCacheSize))
           .setBlockSize(blockSize)
+          .setFormatVersion(ManagedBlockBasedTableConfig.FORMAT_VERSION)
           .setPinL0FilterAndIndexBlocksInCache(true)
           .setFilterPolicy(new ManagedBloomFilter());
       return config;

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java

@@ -99,16 +99,16 @@ public boolean isEmpty() throws RocksDatabaseException {
   @Override
   public boolean isExist(byte[] key) throws RocksDatabaseException {
     rdbMetrics.incNumDBKeyMayExistChecks();
-    final Supplier<byte[]> holder = db.keyMayExist(family, key);
-    if (holder == null) {
+    final Supplier<byte[]> valueSupplier = db.keyMayExist(family, key);
+    if (valueSupplier == null) {
       return false;  // definitely not exists
     }
-    final byte[] value = holder.get();
+    final byte[] value = valueSupplier.get();
     if (value != null) {
       return true; // definitely exists
     }
 
-    // inconclusive: the key may or may not exist
+    // keyMayExist could not return the value; confirm via point-get.
     final boolean exists = get(key) != null;
     if (!exists) {
       rdbMetrics.incNumDBKeyMayExistMisses();
@@ -141,15 +141,16 @@ public byte[] getSkipCache(byte[] bytes) throws RocksDatabaseException {
   @Override
   public byte[] getIfExist(byte[] key) throws RocksDatabaseException {
     rdbMetrics.incNumDBKeyGetIfExistChecks();
-    final Supplier<byte[]> value = db.keyMayExist(family, key);
-    if (value == null) {
+    final Supplier<byte[]> valueSupplier = db.keyMayExist(family, key);
+    if (valueSupplier == null) {
       return null; // definitely not exists
     }
-    if (value.get() != null) {
-      return value.get(); // definitely exists
+    final byte[] value = valueSupplier.get();
+    if (value != null) {
+      return value; // definitely exists
     }
 
-    // inconclusive: the key may or may not exist
+    // keyMayExist could not return the value; confirm via point-get.
     rdbMetrics.incNumDBKeyGetIfExistGets();
     final byte[] val = get(key);
     if (val == null) {
@@ -160,19 +161,24 @@ public byte[] getIfExist(byte[] key) throws RocksDatabaseException {
 
   Integer getIfExist(ByteBuffer key, ByteBuffer outValue) throws RocksDatabaseException {
     rdbMetrics.incNumDBKeyGetIfExistChecks();
+    // Note: RocksDatabase.keyMayExist duplicates the key internally, so the caller's
+    // key buffer position is preserved for the fallback point-get below.
     final Supplier<Integer> value = db.keyMayExist(
         family, key, outValue.duplicate());
     if (value == null) {
       return null; // definitely not exists
     }
-    if (value.get() != null) {
+    final Integer length = value.get();
+    if (length != null) {
       // definitely exists, return value size.
-      return value.get();
+      return length;
     }
 
-    // inconclusive: the key may or may not exist
+    // keyMayExist could not return the value; confirm via point-get. get()
+    // advances the key position, so pass a duplicate to leave the caller's
+    // key buffer unchanged.
     rdbMetrics.incNumDBKeyGetIfExistGets();
-    final Integer val = get(key, outValue);
+    final Integer val = get(key.duplicate(), outValue);
     if (val == null) {
       rdbMetrics.incNumDBKeyGetIfExistMisses();
     }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDatabase.java

@@ -626,8 +626,11 @@ Supplier<byte[]> keyMayExist(ColumnFamily family, byte[] key)
   Supplier<Integer> keyMayExist(ColumnFamily family,
       ByteBuffer key, ByteBuffer out) throws RocksDatabaseException {
     try (UncheckedAutoCloseable ignored = acquire()) {
+      // keyMayExist may advance the input ByteBuffer position in native code.
+      // Always pass a duplicate so callers can safely reuse the original key
+      // buffer for a follow-up point-get.
       final KeyMayExist result = db.get().keyMayExist(
-          family.getHandle(), key, out);
+          family.getHandle(), key.duplicate(), out);
       switch (result.exists) {
       case kNotExist: return null;
       case kExistsWithValue: return () -> result.valueLength;
@@ -872,12 +875,12 @@ public void deleteFilesNotMatchingPrefix(TablePrefixInfo prefixInfo) throws Rock
         String sstFileColumnFamily = StringUtils.bytes2String(liveFileMetaData.columnFamilyName());
         int lastLevel = getLastLevel();
 
-        // RocksDB #deleteFile API allows only to delete the last level of
-        // SST Files. Any level < last level won't get deleted and
-        // only last file of level 0 can be deleted
-        // and will throw warning in the rocksdb manifest.
-        // Instead, perform the level check here
-        // itself to avoid failed delete attempts for lower level files.
+        // Restrict deletion to files at the last level (and skip entirely when
+        // the last level is 0). The old RocksDB #deleteFile API could only
+        // delete last-level SST files (and the last file of level 0);
+        // deleteSstFileRange, used below, no longer has that limitation, but
+        // this method keeps the last-level restriction to preserve its existing
+        // pruning behavior.
         if (liveFileMetaData.level() != lastLevel || lastLevel == 0) {
           continue;
         }
@@ -888,6 +891,12 @@ public void deleteFilesNotMatchingPrefix(TablePrefixInfo prefixInfo) throws Rock
         boolean isKeyWithPrefixPresent = RocksDiffUtils.isKeyWithPrefixPresent(
             prefixForColumnFamily, firstDbKey, lastDbKey);
         if (!isKeyWithPrefixPresent) {
+          ColumnFamilyHandle handle = getColumnFamilyHandle(sstFileColumnFamily);
+          if (handle == null) {
+            LOG.warn("Skipping sst file deletion for {}: no handle found for column family {}",
+                liveFileMetaData.fileName(), sstFileColumnFamily);
+            continue;
+          }
           LOG.info("Deleting sst file: {} with start key: {} and end key: {} "
                   + "corresponding to column family {} from db: {}. "
                   + "Prefix for the column family: {}.",
@@ -896,7 +905,15 @@ public void deleteFilesNotMatchingPrefix(TablePrefixInfo prefixInfo) throws Rock
               StringUtils.bytes2String(liveFileMetaData.columnFamilyName()),
               db.get().getName(),
               prefixForColumnFamily);
-          db.deleteFile(liveFileMetaData);
+          // deleteSstFileRange uses deleteFilesInRanges over this file's
+          // [smallestKey, largestKey]. It may also drop other files fully
+          // contained in that range, which is safe here: any such file's key
+          // range is a subset of this non-matching file's range. Because
+          // isKeyWithPrefixPresent is a monotone prefix-range test, a subset
+          // range cannot contain the prefix when the enclosing range does not,
+          // so every collaterally deleted file is likewise non-matching. This
+          // invariant holds only while isKeyWithPrefixPresent stays monotone.
+          db.deleteSstFileRange(handle, liveFileMetaData);
         }
       }
     }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java

@@ -64,6 +64,8 @@ public interface Table<KEY, VALUE> {
    * Check if a given key exists in Metadata store.
    * (Optimization to save on data deserialization)
    * A lock on the key / bucket needs to be acquired before invoking this API.
+   * Implementations may use fast existence checks internally, but the returned
+   * result must be definitive for the current table state.
    * @param key metadata key
    * @return true if the metadata store contains a key.
    */
@@ -107,12 +109,9 @@ default VALUE getReadCopy(KEY key) throws RocksDatabaseException, CodecException
    * Returns the value mapped to the given key in byte array or returns null
    * if the key is not found.
    *
-   * This method first checks using keyMayExist, if it returns false, we are
-   * 100% sure that key does not exist in DB, so it returns null with out
-   * calling db.get. If keyMayExist return true, then we use db.get and then
-   * return the value. This method will be useful in the cases where the
-   * caller is more sure that this key does not exist in DB and keyMayExist
-   * will help here.
+   * Implementations may use keyMayExist or similar fast-path checks
+   * internally, but the returned result must remain equivalent to a regular
+   * point lookup on the current table state.
    *
    * @param key metadata key
    * @return value in byte array or null if the key is not found.

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStoreCodecBufferIterator.java

@@ -100,13 +100,13 @@ Answer<Integer> newAnswer(String name, byte... b) {
   public void testForEachRemaining() throws Exception {
     when(rocksIteratorMock.isValid())
         .thenReturn(true, true, true, true, true, true, true, false);
-    when(rocksIteratorMock.key(any()))
+    when(rocksIteratorMock.key(any(ByteBuffer.class)))
         .then(newAnswerInt("key1", 0x00))
         .then(newAnswerInt("key2", 0x00))
         .then(newAnswerInt("key3", 0x01))
         .then(newAnswerInt("key4", 0x02))
         .thenThrow(new NoSuchElementException());
-    when(rocksIteratorMock.value(any()))
+    when(rocksIteratorMock.value(any(ByteBuffer.class)))
         .then(newAnswerInt("val1", 0x7f))
         .then(newAnswerInt("val2", 0x7f))
         .then(newAnswerInt("val3", 0x7e))
@@ -152,8 +152,8 @@ public void testNextCallsIsValidThenGetsTheValueAndStepsToNext()
     }
 
     verifier.verify(rocksIteratorMock).isValid();
-    verifier.verify(rocksIteratorMock).key(any());
-    verifier.verify(rocksIteratorMock).value(any());
+    verifier.verify(rocksIteratorMock).key(any(ByteBuffer.class));
+    verifier.verify(rocksIteratorMock).value(any(ByteBuffer.class));
     verifier.verify(rocksIteratorMock).next();
 
     CodecTestUtil.gc();
@@ -192,9 +192,9 @@ public void testSeekToLastSeeks() throws Exception {
   @Test
   public void testSeekReturnsTheActualKey() throws Exception {
     when(rocksIteratorMock.isValid()).thenReturn(true);
-    when(rocksIteratorMock.key(any()))
+    when(rocksIteratorMock.key(any(ByteBuffer.class)))
         .then(newAnswerInt("key1", 0x00));
-    when(rocksIteratorMock.value(any()))
+    when(rocksIteratorMock.value(any(ByteBuffer.class)))
         .then(newAnswerInt("val1", 0x7f));
 
     try (RDBStoreCodecBufferIterator i = newIterator();
@@ -208,8 +208,8 @@ public void testSeekReturnsTheActualKey() throws Exception {
       verifier.verify(rocksIteratorMock, times(1))
           .seek(any(ByteBuffer.class));
       verifier.verify(rocksIteratorMock, times(1)).isValid();
-      verifier.verify(rocksIteratorMock, times(1)).key(any());
-      verifier.verify(rocksIteratorMock, times(1)).value(any());
+      verifier.verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
+      verifier.verify(rocksIteratorMock, times(1)).value(any(ByteBuffer.class));
       assertArrayEquals(new byte[]{0x00}, val.getKey().getArray());
       assertArrayEquals(new byte[]{0x7f}, val.getValue().getArray());
     }
@@ -220,7 +220,7 @@ public void testSeekReturnsTheActualKey() throws Exception {
   @Test
   public void testGettingTheKeyIfIteratorIsValid() throws Exception {
     when(rocksIteratorMock.isValid()).thenReturn(true);
-    when(rocksIteratorMock.key(any()))
+    when(rocksIteratorMock.key(any(ByteBuffer.class)))
         .then(newAnswerInt("key1", 0x00));
 
     byte[] key = null;
@@ -233,7 +233,7 @@ public void testGettingTheKeyIfIteratorIsValid() throws Exception {
     InOrder verifier = inOrder(rocksIteratorMock);
 
     verifier.verify(rocksIteratorMock, times(1)).isValid();
-    verifier.verify(rocksIteratorMock, times(1)).key(any());
+    verifier.verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
     assertArrayEquals(new byte[]{0x00}, key);
 
     CodecTestUtil.gc();
@@ -242,9 +242,9 @@ public void testGettingTheKeyIfIteratorIsValid() throws Exception {
   @Test
   public void testGettingTheValueIfIteratorIsValid() throws Exception {
     when(rocksIteratorMock.isValid()).thenReturn(true);
-    when(rocksIteratorMock.key(any()))
+    when(rocksIteratorMock.key(any(ByteBuffer.class)))
         .then(newAnswerInt("key1", 0x00));
-    when(rocksIteratorMock.value(any()))
+    when(rocksIteratorMock.value(any(ByteBuffer.class)))
         .then(newAnswerInt("val1", 0x7f));
 
     byte[] key = null;
@@ -260,7 +260,7 @@ public void testGettingTheValueIfIteratorIsValid() throws Exception {
     InOrder verifier = inOrder(rocksIteratorMock);
 
     verifier.verify(rocksIteratorMock, times(1)).isValid();
-    verifier.verify(rocksIteratorMock, times(1)).key(any());
+    verifier.verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
     assertArrayEquals(new byte[]{0x00}, key);
     assertArrayEquals(new byte[]{0x7f}, value);
 
@@ -272,7 +272,7 @@ public void testRemovingFromDBActuallyDeletesFromTable() throws Exception {
     final byte[] testKey = new byte[10];
     ThreadLocalRandom.current().nextBytes(testKey);
     when(rocksIteratorMock.isValid()).thenReturn(true);
-    when(rocksIteratorMock.key(any()))
+    when(rocksIteratorMock.key(any(ByteBuffer.class)))
         .then(newAnswer("key1", testKey));
 
     try (RDBStoreCodecBufferIterator i = newIterator(null)) {
@@ -320,7 +320,7 @@ public void testNullPrefixedIterator() throws Exception {
       when(rocksIteratorMock.isValid()).thenReturn(true);
       assertTrue(i.hasNext());
       verify(rocksIteratorMock, times(1)).isValid();
-      verify(rocksIteratorMock, times(0)).key(any());
+      verify(rocksIteratorMock, times(0)).key(any(ByteBuffer.class));
 
       i.seekToLast();
       verify(rocksIteratorMock, times(1)).seekToLast();
@@ -343,11 +343,11 @@ public void testNormalPrefixedIterator() throws Exception {
       clearInvocations(rocksIteratorMock);
 
       when(rocksIteratorMock.isValid()).thenReturn(true);
-      when(rocksIteratorMock.key(any()))
+      when(rocksIteratorMock.key(any(ByteBuffer.class)))
           .then(newAnswer("key1", prefixBytes));
       assertTrue(i.hasNext());
       verify(rocksIteratorMock, times(1)).isValid();
-      verify(rocksIteratorMock, times(1)).key(any());
+      verify(rocksIteratorMock, times(1)).key(any(ByteBuffer.class));
 
       Exception e =
           assertThrows(Exception.class, () -> i.seekToLast(), "Prefixed iterator does not support seekToLast");

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTable.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils.db;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.nio.ByteBuffer;
+import java.util.function.Supplier;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase.ColumnFamily;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link RDBTable}.
+ */
+public class TestRDBTable {
+
+  @Test
+  public void testGetIfExistByteBufferFallbackUsesFreshKeyBuffer()
+      throws Exception {
+    RocksDatabase db = mock(RocksDatabase.class);
+    ColumnFamily columnFamily = mock(ColumnFamily.class);
+    RDBMetrics metrics = mock(RDBMetrics.class);
+    RDBTable table = new RDBTable(db, columnFamily, metrics);
+
+    byte[] keyBytes = "key-1".getBytes(UTF_8);
+    ByteBuffer key = ByteBuffer.wrap(keyBytes);
+    ByteBuffer outValue = ByteBuffer.allocate(64);
+
+    // RocksDatabase.keyMayExist duplicates the key internally, so it leaves the
+    // caller's key buffer untouched. Return an inconclusive result (value-less
+    // "may exist") to force the fallback point-get.
+    when(db.keyMayExist(eq(columnFamily), any(ByteBuffer.class),
+        any(ByteBuffer.class))).thenReturn((Supplier<Integer>) () -> null);
+
+    // get() advances the key buffer position as native RocksDB does. It must
+    // still see the full key, i.e. RDBTable must hand it a fresh duplicate.
+    when(db.get(eq(columnFamily), any(ByteBuffer.class), any(ByteBuffer.class)))
+        .thenAnswer(invocation -> {
+          ByteBuffer keyBuffer = invocation.getArgument(1);
+          if (keyBuffer.remaining() != keyBytes.length) {
+            return null;
+          }
+          keyBuffer.position(keyBuffer.limit());
+          return 0;
+        });
+
+    Integer result = table.getIfExist(key, outValue);
+    assertEquals(0, result);
+    assertEquals(0, key.position(), "caller key buffer position must be unchanged");
+  }
+
+  @Test
+  public void testGetIfExistByteBufferFastPathReturnsValue()
+      throws Exception {
+    RocksDatabase db = mock(RocksDatabase.class);
+    ColumnFamily columnFamily = mock(ColumnFamily.class);
+    RDBMetrics metrics = mock(RDBMetrics.class);
+    RDBTable table = new RDBTable(db, columnFamily, metrics);
+
+    byte[] keyBytes = "key-1".getBytes(UTF_8);
+    byte[] valueBytes = "value-1".getBytes(UTF_8);
+    ByteBuffer key = ByteBuffer.wrap(keyBytes);
+    ByteBuffer outValue = ByteBuffer.allocate(64);
+
+    // Simulate the RocksDB "exists with value" fast path: native code writes
+    // the value into the buffer handed to keyMayExist and reports its length.
+    // getIfExist passes outValue.duplicate(), so the write must land in the
+    // caller's outValue via the shared backing memory.
+    when(db.keyMayExist(eq(columnFamily), any(ByteBuffer.class),
+        any(ByteBuffer.class))).thenAnswer(invocation -> {
+          ByteBuffer valueBuffer = invocation.getArgument(2);
+          valueBuffer.put(valueBytes);
+          return (Supplier<Integer>) () -> valueBytes.length;
+        });
+
+    Integer result = table.getIfExist(key, outValue);
+    assertEquals(valueBytes.length, result);
+    // The fast path must not fall back to a point-get.
+    verify(db, never()).get(eq(columnFamily), any(ByteBuffer.class), any(ByteBuffer.class));
+    // Value bytes written through the duplicate are visible in the caller's buffer.
+    byte[] readBack = new byte[valueBytes.length];
+    outValue.duplicate().get(readBack);
+    assertArrayEquals(valueBytes, readBack);
+  }
+}
+