diff --git a/fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleHelpers.java b/fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleHelpers.java index 547231a462..934b675366 100644 --- a/fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleHelpers.java +++ b/fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleHelpers.java @@ -141,17 +141,17 @@ public static int packedSizeAsTupleItem(Object item) { /** * Get whether one tuple is a prefix of another. - * @param t1 the potential prefix - * @param t2 the whole tuple - * @return {@code true} if {@code t1} is a prefix of {@code t2} + * @param potentialPrefix the potential prefix + * @param wholeTuple the whole tuple + * @return {@code true} if {@code potentialPrefix} is a prefix of {@code wholeTuple} */ - public static boolean isPrefix(@Nonnull Tuple t1, @Nonnull Tuple t2) { - final int len = t1.size(); - if (t2.size() < len) { + public static boolean isPrefix(@Nonnull Tuple potentialPrefix, @Nonnull Tuple wholeTuple) { + final int len = potentialPrefix.size(); + if (wholeTuple.size() < len) { return false; } for (int i = 0; i < len; i++) { - int rc = TupleUtil.compareItems(t1.get(i), t2.get(i)); + int rc = TupleUtil.compareItems(potentialPrefix.get(i), wholeTuple.get(i)); if (rc != 0) { return false; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java new file mode 100644 index 0000000000..2e6dde4684 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java @@ -0,0 +1,58 @@ +/* + * DataInKeySpacePath.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.record.RecordCoreArgumentException; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.tuple.ByteArrayUtil2; + +import javax.annotation.Nonnull; +import java.util.concurrent.CompletableFuture; + +/** + * Class representing a {@link KeyValue} pair within in {@link KeySpacePath}. + */ +public class DataInKeySpacePath { + + @Nonnull + private final CompletableFuture resolvedPath; + @Nonnull + private final byte[] value; + + public DataInKeySpacePath(KeySpacePath path, KeyValue rawKeyValue, FDBRecordContext context) { + this.resolvedPath = path.toResolvedPathAsync(context, rawKeyValue.getKey()); + this.value = rawKeyValue.getValue(); + if (this.value == null) { + throw new RecordCoreArgumentException("Value cannot be null") + .addLogInfo(LogMessageKeys.KEY, ByteArrayUtil2.loggable(rawKeyValue.getKey())); + } + } + + public CompletableFuture getResolvedPath() { + return resolvedPath; + } + + public byte[] getValue() { + return this.value; + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpace.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpace.java index eaa7bbd1bd..d7d67936f9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpace.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpace.java @@ -227,9 +227,12 @@ public KeySpacePath pathFromKey(@Nonnull FDBRecordContext context, @Nonnull Tupl /** * Given a tuple from an FDB key, attempts to determine what path through this directory the tuple * represents, returning a ResolvedKeySpacePath representing the leaf-most directory in the path. - * If entries remained in the tuple beyond the leaf directory, then {@link KeySpacePath#getRemainder()} - * can be used to fetch the remaining portion. - * + *

+ * If entries remained in the tuple beyond the leaf directory, then {@link KeySpacePath#getRemainder()} can be + * used to fetch the remaining portion. + * See also {@link KeySpacePath#toResolvedPathAsync(FDBRecordContext, byte[])} if you need to resolve and you + * know that it is part of a given path. + *

* @param context context used, if needed, for any database operations * @param key the tuple to be decoded * @return a path entry representing the leaf directory entry that corresponds to a value in the diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index d52861ec69..8b0d9fe9f4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -198,6 +198,25 @@ default Tuple toTuple(@Nonnull FDBRecordContext context) { @Nonnull CompletableFuture toResolvedPathAsync(@Nonnull FDBRecordContext context); + /** + * Given a tuple from an FDB key, attempts to determine what sub-path through this directory the tuple + * represents, returning a ResolvedKeySpacePath representing the leaf-most directory in the path. + *

+ * If entries remained in the tuple beyond the leaf directory, then {@link KeySpacePath#getRemainder()} + * can be used to fetch the remaining portion. + * See also {@link KeySpace#resolveFromKeyAsync(FDBRecordContext, Tuple)} if you need to resolve from the root. + *

+ * @param context context used, if needed, for any database operations + * @param key a raw key from the database + * @return the {@link ResolvedKeySpacePath} corresponding to that key, with a potential remainder. + * @throws com.apple.foundationdb.record.RecordCoreArgumentException if the key provided is not part of this path + */ + @API(API.Status.EXPERIMENTAL) + @Nonnull + default CompletableFuture toResolvedPathAsync(@Nonnull FDBRecordContext context, byte[] key) { + throw new UnsupportedOperationException("toResolvedPathAsync is not supported"); + } + /** * Resolves the path into a {@link ResolvedKeySpacePath}, a form the retains all of the information about * the path itself along with the value to which each path entry is resolved. @@ -566,4 +585,39 @@ default List listSubdirectory(@Nonnull FDBRecordContext co */ @API(API.Status.UNSTABLE) String toString(@Nonnull Tuple tuple); + + /** + * Export all data stored under this KeySpacePath and return it in a RecordCursor. + * This method scans all keys that have this path as a prefix and returns the key-value pairs. + * Supports continuation to resume scanning from a previous position. + * + * @param context the transaction context in which to perform the data export + * @param continuation optional continuation from a previous export operation, or null to start from the beginning + * @param scanProperties properties controlling how the scan should be performed + * @return a RecordCursor that iterates over all KeyValue pairs under this path + */ + @API(API.Status.EXPERIMENTAL) + @Nonnull + default RecordCursor exportAllData(@Nonnull FDBRecordContext context, + @Nullable byte[] continuation, + @Nonnull ScanProperties scanProperties) { + throw new UnsupportedOperationException("exportAllData is not supported"); + } + + /** + * Imports the provided data exported via {@link #exportAllData} into this {@code KeySpacePath}. + * This will validate that any data provided in {@code dataToImport} has a path that should be in this path, + * or one of the sub-directories, if not the future will complete exceptionally with + * {@link RecordCoreIllegalImportDataException}. + * If there is any data already existing under this path, the new data will overwrite if the keys are the same. + * This will use the logical value in the {@link DataInKeySpacePath#getResolvedPath()} to determine the key, rather + * than the raw key, meaning that this will work even if the data was exported from a different cluster. + * @param context the transaction context in which to save the data + * @param dataToImport the data to be saved to the database + * @return a future to be completed once all data has been important. + */ + @API(API.Status.EXPERIMENTAL) + @Nonnull + CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index 73f8b86151..b8ef01a872 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -26,10 +26,16 @@ import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.ValueRange; +import com.apple.foundationdb.record.cursors.LazyCursor; +import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor; +import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil; import com.apple.foundationdb.tuple.Tuple; +import com.apple.foundationdb.tuple.TupleHelpers; import com.google.common.collect.Lists; + import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; @@ -242,6 +248,40 @@ public CompletableFuture toResolvedPathAsync(@Nonnull FDBR }); } + @Nonnull + @Override + public CompletableFuture toResolvedPathAsync(@Nonnull final FDBRecordContext context, final byte[] key) { + final Tuple keyTuple = Tuple.fromBytes(key); + return toResolvedPathAsync(context).thenCompose(resolvedPath -> { + // Now use the resolved path to find the child for the key + // We need to figure out how much of the key corresponds to the resolved path + Tuple pathTuple = resolvedPath.toTuple(); + int pathLength = pathTuple.size(); + + if (!TupleHelpers.isPrefix(pathTuple, keyTuple)) { + throw new RecordCoreArgumentException("Key is not under this path") + .addLogInfo(LogMessageKeys.EXPECTED, pathTuple, + LogMessageKeys.ACTUAL, keyTuple); + } + + // The remaining part of the key should be resolved from the resolved path's directory + if (keyTuple.size() > pathLength) { + // There's more in the key than just the path, so resolve the rest + if (resolvedPath.getDirectory().getSubdirectories().isEmpty()) { + return CompletableFuture.completedFuture( + new ResolvedKeySpacePath(resolvedPath.getParent(), resolvedPath.toPath(), + resolvedPath.getResolvedPathValue(), + TupleHelpers.subTuple(keyTuple, pathTuple.size(), keyTuple.size()))); + } else { + return resolvedPath.getDirectory().findChildForKey(context, resolvedPath, keyTuple, keyTuple.size(), pathLength); + } + } else { + // The key exactly matches the path + return CompletableFuture.completedFuture(resolvedPath); + } + }); + } + @Nonnull @Override public CompletableFuture hasDataAsync(@Nonnull FDBRecordContext context) { @@ -331,6 +371,58 @@ public String toString() { return toString(null); } + @Nonnull + @Override + public RecordCursor exportAllData(@Nonnull FDBRecordContext context, + @Nullable byte[] continuation, + @Nonnull ScanProperties scanProperties) { + return new LazyCursor<>(toTupleAsync(context) + .thenApply(tuple -> KeyValueCursor.Builder.withSubspace(new Subspace(tuple)) + .setContext(context) + .setContinuation(continuation) + .setScanProperties(scanProperties) + .build()), + context.getExecutor()) + .map(keyValue -> new DataInKeySpacePath(this, keyValue, context)); + } + + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return toTupleAsync(context).thenCompose(targetTuple -> { + List> importFutures = new ArrayList<>(); + + for (DataInKeySpacePath dataItem : dataToImport) { + CompletableFuture importFuture = dataItem.getResolvedPath().thenCompose(resolvedPath -> { + // Validate that this data belongs under this path + Tuple itemTuple = resolvedPath.toTuple(); + if (!TupleHelpers.isPrefix(targetTuple, itemTuple)) { + throw new RecordCoreIllegalImportDataException( + "Data item path does not belong under target path", + "target", targetTuple, "item", itemTuple); + } + + // Reconstruct the key using logical values from the resolved path + Tuple keyTuple = itemTuple; + if (resolvedPath.getRemainder() != null) { + keyTuple = keyTuple.addAll(resolvedPath.getRemainder()); + } + + // Store the data + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = dataItem.getValue(); + context.ensureActive().set(keyBytes, valueBytes); + + return AsyncUtil.DONE; + }); + importFutures.add(importFuture); + } + + return AsyncUtil.whenAll(importFutures); + }); + } + /** * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java index 769649b8da..25b47a2cc9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java @@ -206,6 +206,11 @@ public CompletableFuture toResolvedPathAsync(@Nonnull FDBR return inner.toResolvedPathAsync(context); } + @Nonnull + @Override + public CompletableFuture toResolvedPathAsync(@Nonnull final FDBRecordContext context, final byte[] key) { + return inner.toResolvedPathAsync(context, key); + } @Override public boolean equals(Object obj) { @@ -226,4 +231,19 @@ public String toString() { public String toString(@Nonnull Tuple t) { return inner.toString(t); } + + @Nonnull + @Override + public RecordCursor exportAllData(@Nonnull FDBRecordContext context, + @Nullable byte[] continuation, + @Nonnull ScanProperties scanProperties) { + return inner.exportAllData(context, continuation, scanProperties); + } + + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return inner.importData(context, dataToImport); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java new file mode 100644 index 0000000000..250e0bcbce --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java @@ -0,0 +1,33 @@ +/* + * RecordCoreIllegalImportDataException.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.record.RecordCoreArgumentException; + +import javax.annotation.Nonnull; + +public class RecordCoreIllegalImportDataException extends RecordCoreArgumentException { + private static final long serialVersionUID = 1L; + + public RecordCoreIllegalImportDataException(@Nonnull final String msg, @Nonnull final Object... keyValue) { + super(msg, keyValue); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePath.java index 0afd60db10..ddb72d041b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePath.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; +import com.google.common.annotations.VisibleForTesting; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -271,4 +272,16 @@ public static void appendValue(StringBuilder sb, Object value) { sb.append(value); } } + + /** + * Returns a new {@code ResolvedKeySpacePath} that is the same, except with the provided {@link #getRemainder()}. + * @param newRemainder a new remainder. This can be {@code null} to remove the remainder entirely. + * @return a new {@code ResolvedKeySpacePath} that is the same as this, except with a different {@link #getRemainder()}. + */ + @Nonnull + @VisibleForTesting + ResolvedKeySpacePath withRemainder(@Nullable final Tuple newRemainder) { + // this could probably copy the cachedTuple & cachedSubspace + return new ResolvedKeySpacePath(parent, inner, value, newRemainder); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java new file mode 100644 index 0000000000..64eccab827 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java @@ -0,0 +1,400 @@ +/* + * DataInKeySpacePathTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.RecordCoreArgumentException; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.foundationdb.tuple.TupleHelpers; +import com.apple.test.BooleanSource; +import com.apple.test.Tags; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link DataInKeySpacePath}. + */ +@Tag(Tags.RequiresFDB) +class DataInKeySpacePathTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 3, 4, 5}) + void resolution(int depth) { + // Include some extra children to make sure resolution doesn't get confused + final String companyUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("company", KeyType.STRING, companyUuid) + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("team_id", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("employee_uuid", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.NULL, null)) + .addSubdirectory(new KeySpaceDirectory("metaData", KeyType.LONG, 0)))) + .addSubdirectory(new KeySpaceDirectory("buildings", KeyType.STRING))))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + UUID employeeId = UUID.randomUUID(); + + KeySpacePath employeePath = root.path("company") + .add("department", "engineering") + .add("team_id", 42L) + .add("employee_uuid", employeeId) + .add("active", true) + .add("data"); + + // Add additional tuple elements after the KeySpacePath + final Tuple remainderTuple = Tuple.from("salary", 75000L, "start_date", "2023-01-15"); + byte[] keyBytes = employeePath.toSubspace(context).pack(remainderTuple); + byte[] valueBytes = Tuple.from("employee_record").pack(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the company-level path + List> pathNavigation = List.of( + path -> path.add("department", "engineering"), + path -> path.add("team_id", 42L), + path -> path.add("employee_uuid", employeeId), + path -> path.add("active", true), + path -> path.add("data") + ); + KeySpacePath toResolve = root.path("company"); + for (final Function function : pathNavigation.subList(0, depth)) { + toResolve = function.apply(toResolve); + } + DataInKeySpacePath dataInPath = new DataInKeySpacePath(toResolve, keyValue, context); + + ResolvedKeySpacePath resolved = dataInPath.getResolvedPath().join(); + + // Verify the path + ResolvedKeySpacePath activeLevel = assertNameAndValue(resolved, "data", null); + ResolvedKeySpacePath uuidLevel = assertNameAndValue(activeLevel, "active", true); + ResolvedKeySpacePath teamLevel = assertNameAndValue(uuidLevel, "employee_uuid", employeeId); + ResolvedKeySpacePath deptLevel = assertNameAndValue(teamLevel, "team_id", 42L); + ResolvedKeySpacePath companyLevel = assertNameAndValue(deptLevel, "department", "engineering"); + assertNull(assertNameAndValue(companyLevel, "company", companyUuid)); + + // Verify the resolved path recreates the KeySpacePath portion + assertEquals(TupleHelpers.subTuple(Tuple.fromBytes(keyBytes), 0, 6), resolved.toTuple()); + + // Verify that the remainder contains the additional tuple elements + assertEquals(remainderTuple, resolved.getRemainder()); + + context.commit(); + } + } + + @Test + void pathWithConstantValues() { + final String appUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("application", KeyType.STRING, appUuid) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("environment", KeyType.STRING, "production") + .addSubdirectory(new KeySpaceDirectory("data", KeyType.STRING))))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath dataPath = root.path("application") + .add("version") // Uses constant value 1L + .add("environment") // Uses constant value "production" + .add("data", "user_records"); + // Add additional tuple elements after the KeySpacePath + byte[] keyBytes = dataPath.toSubspace(context).pack( + Tuple.from("config_id", 1001L, "version", "v2.1")); + byte[] valueBytes = Tuple.from("constant_test_data").pack(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the application-level path + KeySpacePath appPath = root.path("application"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(appPath, keyValue, context); + + ResolvedKeySpacePath resolved = dataInPath.getResolvedPath().join(); + + // Verify the path using assertNameAndValue + ResolvedKeySpacePath envLevel = assertNameAndValue(resolved, "data", "user_records"); + ResolvedKeySpacePath versionLevel = assertNameAndValue(envLevel, "environment", "production"); + ResolvedKeySpacePath applicationLevel = assertNameAndValue(versionLevel, "version", 1L); + assertNull(assertNameAndValue(applicationLevel, "application", appUuid)); + + // Verify the resolved path recreates the KeySpacePath portion + assertEquals(TupleHelpers.subTuple(Tuple.fromBytes(keyBytes), 0, 4), resolved.toTuple()); + + // Verify that the remainder contains the additional tuple elements + Tuple remainder = resolved.getRemainder(); + assertNotNull(remainder); + assertEquals(4, remainder.size()); + assertEquals("config_id", remainder.getString(0)); + assertEquals(1001L, remainder.getLong(1)); + assertEquals("version", remainder.getString(2)); + assertEquals("v2.1", remainder.getString(3)); + + context.commit(); + } + } + + @Test + void pathWithDirectoryLayer() { + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG) + .addSubdirectory(new DirectoryLayerDirectory("service")))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath servicePath = root.path("tenant") + .add("user_id", 999L) + .add("service", "analytics"); + + Tuple keyTuple = servicePath.toTuple(context); + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = Tuple.from("directory_layer_data").pack(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the tenant-level path + KeySpacePath tenantPath = root.path("tenant"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(tenantPath, keyValue, context); + + ResolvedKeySpacePath serviceLevel = dataInPath.getResolvedPath().join(); + + final ResolvedKeySpacePath userLevel = assertNameAndDirectoryScopedValue( + serviceLevel, "service", "analytics", servicePath, context); + ResolvedKeySpacePath tenantLevel = assertNameAndValue(userLevel, "user_id", 999L); + + assertNull(assertNameAndDirectoryScopedValue(tenantLevel, "tenant", tenantUuid, tenantPath, context)); + + context.commit(); + } + } + + @Test + void pathWithBinaryData() { + final String storeUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("binary_store", KeyType.STRING, storeUuid) + .addSubdirectory(new KeySpaceDirectory("blob_id", KeyType.BYTES))); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + byte[] blobId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + KeySpacePath blobPath = root.path("binary_store").add("blob_id", blobId); + + Tuple keyTuple = blobPath.toTuple(context); + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = "binary_test_data".getBytes(); + + tr.set(keyBytes, valueBytes); + KeyValue keyValue = new KeyValue(keyBytes, valueBytes); + + // Create DataInKeySpacePath from the binary_store-level path + KeySpacePath storePath = root.path("binary_store"); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(storePath, keyValue, context); + + ResolvedKeySpacePath resolved = dataInPath.getResolvedPath().join(); + + // Verify the path using assertNameAndValue + assertEquals("blob_id", resolved.getDirectoryName()); + byte[] resolvedBytes = (byte[]) resolved.getResolvedValue(); + assertArrayEquals(blobId, resolvedBytes); + + ResolvedKeySpacePath storeLevel = assertNameAndValue(resolved.getParent(), "binary_store", storeUuid); + assertNull(storeLevel); + + // Verify the resolved path can recreate the original key + assertEquals(keyTuple, resolved.toTuple()); + + context.commit(); + } + } + + @ParameterizedTest + @BooleanSource("withRemainder") + void keyValueAccessors(boolean withRemainder) throws ExecutionException, InterruptedException { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath testPath = root.path("test"); + Tuple pathTuple = testPath.toTuple(context); + byte[] keyBytes = withRemainder ? pathTuple.add("Remainder").pack() : pathTuple.pack(); + byte[] valueBytes = Tuple.from("accessor_test").pack(); + + KeyValue originalKeyValue = new KeyValue(keyBytes, valueBytes); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(testPath, originalKeyValue, context); + + // Verify resolved path future is not null + CompletableFuture resolvedFuture = dataInPath.getResolvedPath(); + assertNotNull(resolvedFuture); + assertTrue(resolvedFuture.isDone() || !resolvedFuture.isCancelled()); + + final ResolvedKeySpacePath resolvedPath = resolvedFuture.get(); + assertEquals(pathTuple, resolvedPath.toTuple()); + assertArrayEquals(originalKeyValue.getValue(), dataInPath.getValue()); + if (withRemainder) { + assertEquals(Tuple.from("Remainder"), resolvedPath.getRemainder()); + } else { + assertNull(resolvedPath.getRemainder()); + } + } + } + + @Test + void withWrapper() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Test export at different levels through wrapper methods + try (FDBRecordContext context = database.openContext()) { + // Test 4: Export from specific data store level + final EnvironmentKeySpace.ApplicationPath appPath = keySpace.root().userid(100L).application("app1"); + EnvironmentKeySpace.DataPath dataStore = appPath.dataStore(); + + final byte[] key = dataStore.toTuple(context).add("record2").add(0).pack(); + final byte[] value = Tuple.from("data").pack(); + final DataInKeySpacePath dataInKeySpacePath = new DataInKeySpacePath(dataStore, new KeyValue(key, value), context); + + final ResolvedKeySpacePath resolvedPath = dataInKeySpacePath.getResolvedPath().join(); + assertEquals(dataStore.toResolvedPath(context), withoutRemainder(resolvedPath)); + assertEquals(Tuple.from("record2", 0), resolvedPath.getRemainder()); + + // Verify the path using assertNameAndValue + // Note: We expect the path to be: [environment] -> userid -> application -> data + ResolvedKeySpacePath appLevel; + appLevel = assertNameAndValue(resolvedPath, "data", EnvironmentKeySpace.DATA_VALUE); + ResolvedKeySpacePath userLevel = assertNameAndDirectoryScopedValue(appLevel, "application", "app1", + appPath, context); + ResolvedKeySpacePath envLevel = assertNameAndValue(userLevel, "userid", 100L); + assertNull(assertNameAndDirectoryScopedValue(envLevel, keySpace.root().getDirectoryName(), + keySpace.root().getValue(), keySpace.root(), context)); + } + } + + @Test + void badPath() { + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, rootUuid)); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath testPath = root.path("test"); + byte[] keyBytes = Tuple.from("banana", rootUuid).pack(); + byte[] valueBytes = Tuple.from("accessor_test").pack(); + + KeyValue originalKeyValue = new KeyValue(keyBytes, valueBytes); + DataInKeySpacePath dataInPath = new DataInKeySpacePath(testPath, originalKeyValue, context); + final CompletionException completionException = assertThrows(CompletionException.class, + () -> dataInPath.getResolvedPath().join()); + assertInstanceOf(RecordCoreArgumentException.class, completionException.getCause()); + } + } + + /** + * Test if there is a null value. FDB shouldn't ever return this, and if you got it somehow, you wouldn't be + * able to insert it back into a database because {@link com.apple.foundationdb.FDBTransaction#set(byte[], byte[])} + * does not support a {@code null} key or value. + */ + @Test + void nullValue() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath testPath = root.path("test"); + byte[] keyBytes = testPath.toTuple(context).pack(); + byte[] valueBytes = null; + + KeyValue originalKeyValue = new KeyValue(keyBytes, valueBytes); + assertThrows(RecordCoreArgumentException.class, + () -> new DataInKeySpacePath(testPath, originalKeyValue, context)); + } + } + + private static ResolvedKeySpacePath assertNameAndDirectoryScopedValue(ResolvedKeySpacePath resolved, + String name, Object logicalValue, + KeySpacePath path, FDBRecordContext context) { + assertNotNull(resolved); + assertEquals(name, resolved.getDirectoryName()); + assertEquals(path.toResolvedPath(context).getResolvedValue(), resolved.getResolvedValue()); + assertEquals(logicalValue, resolved.getLogicalValue()); + return resolved.getParent(); + } + + private static ResolvedKeySpacePath assertNameAndValue(ResolvedKeySpacePath resolved, String name, Object value) { + assertNotNull(resolved); + assertEquals(name, resolved.getDirectoryName()); + assertEquals(value, resolved.getResolvedValue()); + assertEquals(value, resolved.getLogicalValue()); + return resolved.getParent(); + } + + private ResolvedKeySpacePath withoutRemainder(final ResolvedKeySpacePath path) { + return new ResolvedKeySpacePath(path.getParent(), path.toPath(), path.getResolvedPathValue(), null); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/EnvironmentKeySpace.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/EnvironmentKeySpace.java new file mode 100644 index 0000000000..c1751395cf --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/EnvironmentKeySpace.java @@ -0,0 +1,195 @@ +/* + * EnvironmentKeySpace.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; +import com.apple.foundationdb.tuple.Tuple; + +import javax.annotation.Nonnull; +import java.util.UUID; + +/** + * This provides an example of a way in which you can define a KeySpace in a relatively clean and type-safe + * manner. It defines a keyspace that looks like: + * 
+ *    [environment]           - A string the identifies the logical environment (like prod, test, qa, etc.).
+ *      |                       This string is converted by the directory layer as a small integer value.
+ *      +- userid             - An integer ID for each user in the system
+ *         |
+ *         +- [application]   - Tne name of an application the user runs (again, converted by the directory
+ *            |                 layer into a small integer value)
+ *            +- data=1       - Constant value of "1", which is the location of a {@link FDBRecordStore}
+ *            |                 in which application data is to be stored
+ *            +- metadata=2   - Constant value of "2", which is the Location of another FDBRecordStore
+ *                              in which application metadata or configuration information can live.
+ *
+ * The main point of this class is to demonstrate how you can use the KeySpacePath wrapping facility to provide + * implementations of the path elements that are meaningful to your application environment and type safe. + */ +class EnvironmentKeySpace { + private final KeySpace root; + private final String rootName; + + static final String USER_KEY = "userid"; + static final String APPLICATION_KEY = "application"; + static final String DATA_KEY = "data"; + static final long DATA_VALUE = 1L; + static final String METADATA_KEY = "metadata"; + static final long METADATA_VALUE = 2L; + + /** + * The EnvironmentKeySpace scopes all of the data it stores underneath of a rootName, + * for example, you could define an instance for prod, test, qa, etc. + * + * @param rootName The root name underwhich all data is stored. + */ + public EnvironmentKeySpace(String rootName) { + this.rootName = rootName; + root = new KeySpace( + new DirectoryLayerDirectory(rootName, rootName, EnvironmentRoot::new) + .addSubdirectory(new KeySpaceDirectory(USER_KEY, KeySpaceDirectory.KeyType.LONG, UserPath::new) + .addSubdirectory(new DirectoryLayerDirectory(APPLICATION_KEY, ApplicationPath::new) + .addSubdirectory(new KeySpaceDirectory(DATA_KEY, KeySpaceDirectory.KeyType.LONG, DATA_VALUE, DataPath::new)) + .addSubdirectory(new KeySpaceDirectory(METADATA_KEY, KeySpaceDirectory.KeyType.LONG, METADATA_VALUE, MetadataPath::new))))); + } + + @Nonnull + static EnvironmentKeySpace setupSampleData(@Nonnull final FDBDatabase database) { + EnvironmentKeySpace keySpace = new EnvironmentKeySpace(UUID.randomUUID().toString()); + + // Store test data at different levels of the hierarchy + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Create paths for different users and applications + ApplicationPath app1User1 = keySpace.root().userid(100L).application("app1"); + ApplicationPath app2User1 = keySpace.root().userid(100L).application("app2"); + ApplicationPath app1User2 = keySpace.root().userid(200L).application("app1"); + + DataPath dataUser1App1 = app1User1.dataStore(); + MetadataPath metaUser1App1 = app1User1.metadataStore(); + DataPath dataUser1App2 = app2User1.dataStore(); + DataPath dataUser2App1 = app1User2.dataStore(); + + // Store data records with additional tuple elements after the KeySpacePath + tr.set(dataUser1App1.toTuple(context).add("record1").pack(), Tuple.from("user100_app1_data1").pack()); + tr.set(dataUser1App1.toTuple(context).add("record2").add(0).pack(), Tuple.from("user100_app1_data2_0").pack()); + tr.set(dataUser1App1.toTuple(context).add("record2").add(1).pack(), Tuple.from("user100_app1_data2_1").pack()); + tr.set(metaUser1App1.toTuple(context).add("config1").pack(), Tuple.from("user100_app1_meta1").pack()); + tr.set(dataUser1App2.toTuple(context).add("record3").pack(), Tuple.from("user100_app2_data3").pack()); + tr.set(dataUser2App1.toTuple(context).add("record4").pack(), Tuple.from("user200_app1_data4").pack()); + + context.commit(); + } + return keySpace; + } + + public String getRootName() { + return rootName; + } + + /** + * Returns an implementation of a KeySpacePath that represents the start of the environment. + */ + public EnvironmentRoot root() { + return (EnvironmentRoot)root.path(rootName); + } + + /** + * Given a tuple that represents an FDB key that came from this KeySpace, returns the leaf-most path + * element in which the tuple resides. + */ + public ResolvedKeySpacePath fromKey(FDBRecordContext context, Tuple tuple) { + return root.resolveFromKey(context, tuple); + } + + /** + * A KeySpacePath that represents the logical root of the environment. + */ + static class EnvironmentRoot extends KeySpacePathWrapper { + public EnvironmentRoot(KeySpacePath path) { + super(path); + } + + public KeySpacePath parent() { + return null; + } + + public UserPath userid(long userid) { + return (UserPath) inner.add(USER_KEY, userid); + } + } + + static class UserPath extends KeySpacePathWrapper { + public UserPath(KeySpacePath path) { + super(path); + } + + public ApplicationPath application(String applicationName) { + return (ApplicationPath) inner.add(APPLICATION_KEY, applicationName); + } + + public EnvironmentRoot parent() { + return (EnvironmentRoot) inner.getParent(); + } + } + + static class ApplicationPath extends KeySpacePathWrapper { + public ApplicationPath(KeySpacePath path) { + super(path); + } + + public DataPath dataStore() { + return (DataPath) inner.add(DATA_KEY); + } + + public MetadataPath metadataStore() { + return (MetadataPath) inner.add(METADATA_KEY); + } + + public UserPath parent() { + return (UserPath) inner.getParent(); + } + } + + static class DataPath extends KeySpacePathWrapper { + public DataPath(KeySpacePath path) { + super(path); + } + + public ApplicationPath parent() { + return (ApplicationPath) inner.getParent(); + } + } + + static class MetadataPath extends KeySpacePathWrapper { + public MetadataPath(KeySpacePath path) { + super(path); + } + + public ApplicationPath parent() { + return (ApplicationPath) inner.getParent(); + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java index 675cad4cbe..6c9d2d07d6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpaceDirectoryTest.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.record.logging.KeyValueLogMessage; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; -import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; import com.apple.foundationdb.record.provider.foundationdb.layers.interning.ScopedInterningLayer; @@ -1335,7 +1334,7 @@ public void testPathWrapperExample() throws Exception { final Tuple dataStoreTuple; final Tuple metadataStoreTuple; try (FDBRecordContext context = database.openContext()) { - ApplicationPath application = keySpace.root().userid(123).application("myApplication"); + EnvironmentKeySpace.ApplicationPath application = keySpace.root().userid(123).application("myApplication"); dataStoreTuple = application.dataStore().toTuple(context); metadataStoreTuple = application.metadataStore().toTuple(context); context.commit(); @@ -1349,9 +1348,9 @@ public void testPathWrapperExample() throws Exception { assertEquals(Tuple.from(entries.get(0), 123L, entries.get(1), EnvironmentKeySpace.METADATA_VALUE), metadataStoreTuple); ResolvedKeySpacePath path = keySpace.fromKey(context, dataStoreTuple); - assertThat(path.toPath(), instanceOf(DataPath.class)); + assertThat(path.toPath(), instanceOf(EnvironmentKeySpace.DataPath.class)); - DataPath mainStorePath = (DataPath) path.toPath(); + EnvironmentKeySpace.DataPath mainStorePath = (EnvironmentKeySpace.DataPath) path.toPath(); assertEquals(EnvironmentKeySpace.DATA_VALUE, mainStorePath.getValue()); assertEquals(EnvironmentKeySpace.DATA_VALUE, mainStorePath.resolveAsync(context).get().getResolvedValue()); assertEquals(entries.get(1), mainStorePath.parent().resolveAsync(context).get().getResolvedValue()); @@ -1361,14 +1360,14 @@ public void testPathWrapperExample() throws Exception { assertEquals("production", mainStorePath.parent().parent().parent().getValue()); assertNull(mainStorePath.parent().parent().parent().parent()); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentRoot.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(UserPath.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(ApplicationPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentKeySpace.EnvironmentRoot.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(EnvironmentKeySpace.UserPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(EnvironmentKeySpace.ApplicationPath.class)); path = keySpace.fromKey(context, metadataStoreTuple); - assertThat(path.toPath(), instanceOf(MetadataPath.class)); + assertThat(path.toPath(), instanceOf(EnvironmentKeySpace.MetadataPath.class)); - MetadataPath metadataPath = (MetadataPath) path.toPath(); + EnvironmentKeySpace.MetadataPath metadataPath = (EnvironmentKeySpace.MetadataPath) path.toPath(); assertEquals(EnvironmentKeySpace.METADATA_VALUE, metadataPath.getValue()); assertEquals(EnvironmentKeySpace.METADATA_VALUE, metadataPath.resolveAsync(context).get().getResolvedValue()); assertEquals(entries.get(1), metadataPath.parent().resolveAsync(context).get().getResolvedValue()); @@ -1378,14 +1377,14 @@ public void testPathWrapperExample() throws Exception { assertEquals("production", metadataPath.parent().parent().parent().getValue()); assertNull(metadataPath.parent().parent().parent().parent()); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentRoot.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(UserPath.class)); - assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(ApplicationPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 1)).toPath(), instanceOf(EnvironmentKeySpace.EnvironmentRoot.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 2)).toPath(), instanceOf(EnvironmentKeySpace.UserPath.class)); + assertThat(keySpace.fromKey(context, TupleHelpers.subTuple(dataStoreTuple, 0, 3)).toPath(), instanceOf(EnvironmentKeySpace.ApplicationPath.class)); // Create a fake main store "record" key to demonstrate that we can get the key as the remainder Tuple recordTuple = dataStoreTuple.add(1L).add("someStr").add(0L); // 1=record space, record id, 0=unsplit record path = keySpace.fromKey(context, recordTuple); - assertThat(path.toPath(), instanceOf(DataPath.class)); + assertThat(path.toPath(), instanceOf(EnvironmentKeySpace.DataPath.class)); assertEquals(Tuple.from(1L, "someStr", 0L), path.getRemainder()); assertEquals(dataStoreTuple, path.toTuple()); } @@ -1596,137 +1595,4 @@ protected CompletableFuture toTupleValueAsyncImpl(@Nonnull FDBRecordC } } - /** - * This provides an example of a way in which you can define a KeySpace in a relatively clean and type-safe - * manner. It defines a keyspace that looks like: - * 
-     *    [environment]           - A string the identifies the logical environment (like prod, test, qa, etc.).
-     *      |                       This string is converted by the directory layer as a small integer value.
-     *      +- userid             - An integer ID for each user in the system
-     *         |
-     *         +- [application]   - Tne name of an application the user runs (again, converted by the directory
-     *            |                 layer into a small integer value)
-     *            +- data=1       - Constant value of "1", which is the location of a {@link FDBRecordStore}
-     *            |                 in which application data is to be stored
-     *            +- metadata=2   - Constant value of "2", which is the Location of another FDBRecordStore
-     *                              in which application metadata or configuration information can live.
-     *
- * The main point of this class is to demonstrate how you can use the KeySpacePath wrapping facility to provide - * implementations of the path elements that are meaningful to your application environment and type safe. - */ - private static class EnvironmentKeySpace { - private final KeySpace root; - private final String rootName; - - public static String USER_KEY = "userid"; - public static String APPLICATION_KEY = "application"; - public static String DATA_KEY = "data"; - public static long DATA_VALUE = 1L; - public static String METADATA_KEY = "metadata"; - public static long METADATA_VALUE = 2L; - - /** - * The EnvironmentKeySpace scopes all of the data it stores underneath of a rootName, - * for example, you could define an instance for prod, test, qa, etc. - * - * @param rootName The root name underwhich all data is stored. - */ - public EnvironmentKeySpace(String rootName) { - this.rootName = rootName; - root = new KeySpace( - new DirectoryLayerDirectory(rootName, rootName, EnvironmentRoot::new) - .addSubdirectory(new KeySpaceDirectory(USER_KEY, KeyType.LONG, UserPath::new) - .addSubdirectory(new DirectoryLayerDirectory(APPLICATION_KEY, ApplicationPath::new) - .addSubdirectory(new KeySpaceDirectory(DATA_KEY, KeyType.LONG, DATA_VALUE, DataPath::new)) - .addSubdirectory(new KeySpaceDirectory(METADATA_KEY, KeyType.LONG, METADATA_VALUE, MetadataPath::new))))); - } - - public String getRootName() { - return rootName; - } - - /** - * Returns an implementation of a KeySpacePath that represents the start of the environment. - */ - public EnvironmentRoot root() { - return (EnvironmentRoot) root.path(rootName); - } - - /** - * Given a tuple that represents an FDB key that came from this KeySpace, returns the leaf-most path - * element in which the tuple resides. - */ - public ResolvedKeySpacePath fromKey(FDBRecordContext context, Tuple tuple) { - return root.resolveFromKey(context, tuple); - } - } - - /** - * A KeySpacePath that represents the logical root of the environment. - */ - private static class EnvironmentRoot extends KeySpacePathWrapper { - public EnvironmentRoot(KeySpacePath path) { - super(path); - } - - public KeySpacePath parent() { - return null; - } - - public UserPath userid(long userid) { - return (UserPath) inner.add(EnvironmentKeySpace.USER_KEY, userid); - } - } - - private static class UserPath extends KeySpacePathWrapper { - public UserPath(KeySpacePath path) { - super(path); - } - - public ApplicationPath application(String applicationName) { - return (ApplicationPath) inner.add(EnvironmentKeySpace.APPLICATION_KEY, applicationName); - } - - public EnvironmentRoot parent() { - return (EnvironmentRoot) inner.getParent(); - } - } - - private static class ApplicationPath extends KeySpacePathWrapper { - public ApplicationPath(KeySpacePath path) { - super(path); - } - - public DataPath dataStore() { - return (DataPath) inner.add(EnvironmentKeySpace.DATA_KEY); - } - - public MetadataPath metadataStore() { - return (MetadataPath) inner.add(EnvironmentKeySpace.METADATA_KEY); - } - - public UserPath parent() { - return (UserPath) inner.getParent(); - } - } - - private static class DataPath extends KeySpacePathWrapper { - public DataPath(KeySpacePath path) { - super(path); - } - - public ApplicationPath parent() { - return (ApplicationPath) inner.getParent(); - } - } - - private static class MetadataPath extends KeySpacePathWrapper { - public MetadataPath(KeySpacePath path) { - super(path); - } - - public ApplicationPath parent() { - return (ApplicationPath) inner.getParent(); - } - } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java new file mode 100644 index 0000000000..52cf9fe423 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -0,0 +1,696 @@ +/* + * KeySpacePathDataExportTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorStartContinuation; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.ParameterizedTestUtils; +import com.apple.test.Tags; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for the new KeySpacePath data export feature that fetches all data stored under a KeySpacePath + * and returns it in a {@code RecordCursor}. + */ +@Tag(Tags.RequiresFDB) +class KeySpacePathDataExportTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + + @Test + void exportAllDataFromSimplePath() throws ExecutionException, InterruptedException { + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + KeySpacePath basePath = root.path("root"); + + // Add data at different levels + for (int i = 0; i < 5; i++) { + Tuple key = basePath.add("level1", (long) i).toTuple(context); + tr.set(key.pack(), Tuple.from("value" + i).pack()); + + // Add some sub-data under each key + for (int j = 0; j < 3; j++) { + Tuple subKey = key.add("sub" + j); + tr.set(subKey.pack(), Tuple.from("subvalue" + i + "_" + j).pack()); + } + } + context.commit(); + } + + // Export all data from the root path + try (FDBRecordContext context = database.openContext()) { + KeySpacePath rootPath = root.path("root"); + final List allData = exportAllData(rootPath, context); + + // Should have 5 main entries + 15 sub-entries = 20 total + assertEquals(20, allData.size()); + + // Verify the data is sorted by key + for (int i = 1; i < allData.size(); i++) { + assertTrue(getKey(allData.get(i - 1)).compareTo(getKey(allData.get(i))) < 0); + } + } + } + + // `toTuple` does not include the remainder, I'm not sure if that is intentional, or an oversight. + private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath) throws ExecutionException, InterruptedException { + final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getResolvedPath().get(); + if (resolvedKeySpacePath.getRemainder() != null) { + return resolvedKeySpacePath.toTuple().addAll(resolvedKeySpacePath.getRemainder()); + } else { + return resolvedKeySpacePath.toTuple(); + } + } + + @Test + void exportAllDataFromSpecificSubPath() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("app", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("user", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.NULL)))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data for multiple users + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + for (long userId = 1; userId <= 3; userId++) { + KeySpacePath userPath = root.path("app").add("user", userId); + KeySpacePath dataPath = userPath.add("data"); + + // Add data for each user + for (int i = 0; i < 4; i++) { + Tuple key = dataPath.toTuple(context).add("record" + i); + tr.set(key.pack(), Tuple.from("user" + userId + "_data" + i).pack()); + } + } + context.commit(); + } + + // Export data only for user 2 + try (FDBRecordContext context = database.openContext()) { + KeySpacePath user2Path = root.path("app").add("user", 2L); + final List userData = exportAllData(user2Path, context); + + // Should have 4 records for user 2 + assertEquals(4, userData.size()); + + // Verify all data belongs to user 2 + for (DataInKeySpacePath data : userData) { + String value = Tuple.fromBytes(data.getValue()).getString(0); + assertTrue(value.startsWith("user2_")); + } + } + } + + @Test + void exportAllDataWithDirectoryLayer() { + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("env", UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("tenant", KeyType.LONG) + .addSubdirectory(new DirectoryLayerDirectory("service")))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath basePath = root.path("env").add("tenant", 100L); + + // Add data for different services + String[] services = {"auth", "storage", "compute"}; + for (String service : services) { + KeySpacePath servicePath = basePath.add("service", service); + Tuple serviceKey = servicePath.toTuple(context); + + for (int i = 0; i < 2; i++) { + tr.set(serviceKey.add("config" + i).pack(), + Tuple.from(service + "_config_" + i).pack()); + } + } + context.commit(); + } + + // Export all data from tenant path + try (FDBRecordContext context = database.openContext()) { + KeySpacePath tenantPath = root.path("env").add("tenant", 100L); + final List allData = exportAllData(tenantPath, context); + + // Should have 6 records (3 services * 2 configs each) + assertEquals(6, allData.size()); + + // Verify we have data for all three services + Set serviceNames = new HashSet<>(); + for (DataInKeySpacePath data : allData) { + String value = Tuple.fromBytes(data.getValue()).getString(0); + String serviceName = value.split("_")[0]; + serviceNames.add(serviceName); + } + assertEquals(3, serviceNames.size()); + assertTrue(serviceNames.containsAll(Arrays.asList("auth", "storage", "compute"))); + } + } + + @Test + void exportAllDataWithDifferentKeyTypes() { + final KeySpaceDirectory rootDirectory = new KeySpaceDirectory("mixed", KeyType.STRING, UUID.randomUUID().toString()); + Map> dataByType = Map.of( + KeyType.STRING, List.of("str0", "str1", "str2"), + KeyType.LONG, List.of(10L, 11L, 12L), + KeyType.BYTES, List.of(new byte[]{0, 1}, new byte[]{1, 2}), + KeyType.UUID, List.of(new UUID(0, 0), new UUID(1, 1)), + KeyType.BOOLEAN, List.of(true, false), + KeyType.NULL, Collections.singletonList(null), + KeyType.DOUBLE, List.of(3.1415, -2.718281, 13.23E8), + KeyType.FLOAT, List.of(1.4142135f, -5.8f, 130.23f) + ); + dataByType.keySet().forEach(keyType -> + rootDirectory.addSubdirectory(new KeySpaceDirectory(keyType.name(), keyType))); + KeySpace root = new KeySpace(rootDirectory); + assertEquals(Set.of(KeyType.values()), dataByType.keySet()); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data with different key types + try (FDBRecordContext context = database.openContext()) { + KeySpacePath basePath = root.path("mixed"); + dataByType.forEach((keyType, data) -> + setData(data, context, basePath, keyType.name(), keyType + "_value_") + ); + context.commit(); + } + + // Export all data and verify different key types + try (FDBRecordContext context = database.openContext()) { + KeySpacePath mixedPath = root.path("mixed"); + final List allData = exportAllData(mixedPath, context); + + assertEquals(dataByType.values().stream().mapToLong(List::size).sum(), + allData.size()); + + // Verify we have different value types + Set valueTypes = allData.stream() + .map(data -> Tuple.fromBytes(data.getValue()).getString(0).split("_")[0]) + .collect(Collectors.toSet()); + assertEquals((Arrays.stream(KeyType.values()).map(Enum::name).collect(Collectors.toSet())), + valueTypes); + } + } + + @Test + void exportAllDataWithConstantValues() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("app", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.STRING, "records")))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data using constant values + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath dataPath = root.path("app").add("version").add("data"); + Tuple baseKey = dataPath.toTuple(context); + + // Add multiple records under the constant path + for (int i = 0; i < 4; i++) { + tr.set(baseKey.add("record" + i).pack(), + Tuple.from("constant_path_data_" + i).pack()); + } + context.commit(); + } + + // Export data from path with constant values + try (FDBRecordContext context = database.openContext()) { + KeySpacePath appPath = root.path("app"); + final List allData = exportAllData(appPath, context); + + // Should have 4 records + assertEquals(4, allData.size()); + + // Verify all data has expected prefix + for (DataInKeySpacePath data : allData) { + String value = Tuple.fromBytes(data.getValue()).getString(0); + assertTrue(value.startsWith("constant_path_data_")); + } + } + } + + @Test + void exportAllDataEmpty() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("empty", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Don't store any data + try (FDBRecordContext context = database.openContext()) { + KeySpacePath emptyPath = root.path("empty"); + final List allData = exportAllData(emptyPath, context); + + // Should be empty + assertEquals(0, allData.size()); + } + } + + @Test + void exportAllDataWithDeepNestedStructure() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("org", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("dept", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("team", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.NULL)))))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Create deep nested structure + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + String[] departments = {"engineering", "sales"}; + for (String dept : departments) { + for (long team = 1; team <= 2; team++) { + for (int member = 0; member < 2; member++) { + UUID memberId = new UUID(dept.hashCode(), team * 100 + member); + KeySpacePath memberPath = root.path("org") + .add("dept", dept) + .add("team", team) + .add("member", memberId) + .add("data"); + + Tuple key = memberPath.toTuple(context); + tr.set(key.add("profile").pack(), + Tuple.from(dept + "_team" + team + "_member" + member).pack()); + tr.set(key.add("settings").pack(), + Tuple.from("settings_" + member).pack()); + } + } + } + context.commit(); + } + + // Export all data from organization root + try (FDBRecordContext context = database.openContext()) { + KeySpacePath orgPath = root.path("org"); + final List allData = exportAllData(orgPath, context); + + // Should have 16 records (2 departments * 2 teams * 2 members * 2 records each) + assertEquals(16, allData.size()); + } + + // Export data from specific department + try (FDBRecordContext context = database.openContext()) { + KeySpacePath engPath = root.path("org").add("dept", "engineering"); + final List allData = exportAllData(engPath, context); + + // Should have 8 records (1 dept * 2 teams * 2 members * 2 records each) + assertEquals(8, allData.size()); + + // Verify all belong to engineering + for (DataInKeySpacePath data : allData) { + String value = Tuple.fromBytes(data.getValue()).getString(0); + if (value.startsWith("engineering_")) { + assertTrue(value.contains("engineering_")); + } + } + } + } + + @Test + void exportAllDataWithBinaryData() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("binary", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("blob", KeyType.BYTES))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store binary data + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + KeySpacePath basePath = root.path("binary"); + + // Store different types of binary data + byte[][] binaryKeys = { + {0x00, 0x01, 0x02}, + {(byte) 0xFF, (byte) 0xFE, (byte) 0xFD}, + {0x7F, 0x00, (byte) 0x80} + }; + + for (int i = 0; i < binaryKeys.length; i++) { + Tuple key = basePath.add("blob", binaryKeys[i]).toTuple(context); + byte[] value = ("binary_data_" + i).getBytes(); + tr.set(key.pack(), value); + } + context.commit(); + } + + // Export binary data + try (FDBRecordContext context = database.openContext()) { + KeySpacePath binaryPath = root.path("binary"); + final List allData = exportAllData(binaryPath, context); + + assertEquals(3, allData.size()); + + // Verify binary data integrity + for (DataInKeySpacePath data : allData) { + String valueStr = new String(data.getValue()); + assertTrue(valueStr.startsWith("binary_data_")); + } + } + } + + static Stream exportAllDataWithContinuation() { + return ParameterizedTestUtils.cartesianProduct( + ParameterizedTestUtils.booleans("forward"), + Stream.of(1, 2, 3, 30) + ); + } + + @ParameterizedTest + @MethodSource + void exportAllDataWithContinuation(boolean forward, int limit) { + KeySpace root = new KeySpace( + new KeySpaceDirectory("continuation", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("item", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + final List> expectedBatches = new ArrayList<>(); + expectedBatches.add(new ArrayList<>()); + final KeySpacePath pathToExport = root.path("continuation"); + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + IntStream sourceStream; + if (forward) { + sourceStream = IntStream.range(0, 20); + } else { + sourceStream = IntStream.iterate(19, i -> i >= 0, i -> i - 1); + } + sourceStream.forEach(i -> { + Tuple key = pathToExport.add("item", (long)i).toTuple(context); + final Tuple value = Tuple.from("continuation_item_" + i); + tr.set(key.pack(), value.pack()); + if (expectedBatches.get(expectedBatches.size() - 1).size() == limit) { + expectedBatches.add(new ArrayList<>()); + } + expectedBatches.get(expectedBatches.size() - 1).add(value); + }); + context.commit(); + } + + if (20 % limit == 0) { + expectedBatches.add(List.of()); + } + + // Export with continuation support + exportWithContinuations(pathToExport, forward, limit, database, expectedBatches); + } + + static Stream exportSingleKeyWithContinuation() { + return ParameterizedTestUtils.cartesianProduct( + ParameterizedTestUtils.booleans("forward"), + ParameterizedTestUtils.booleans("withRemainder"), + Stream.of(1, 3) + ); + } + + @ParameterizedTest + @MethodSource + void exportSingleKeyWithContinuation(boolean forward, boolean withRemainder, int limit) { + KeySpace root = new KeySpace( + new KeySpaceDirectory("continuation", KeyType.STRING, UUID.randomUUID().toString()) + .addSubdirectory(new KeySpaceDirectory("item", KeyType.LONG))); + + final FDBDatabase database = dbExtension.getDatabase(); + + // Store test data + final List> expectedBatches; + final KeySpacePath pathToExport = root.path("continuation"); + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + byte[] key; + if (withRemainder) { + key = pathToExport.toSubspace(context).pack(Tuple.from("continuation")); + } else { + key = pathToExport.toSubspace(context).pack(); + } + final Tuple value = Tuple.from("My Value"); + tr.set(key, value.pack()); + expectedBatches = limit == 1 ? List.of(List.of(value), List.of()) : List.of(List.of(value)); + context.commit(); + } + + // Export with continuation support + exportWithContinuations(pathToExport, forward, limit, database, expectedBatches); + } + + private static void exportWithContinuations(final KeySpacePath pathToExport, + final boolean forward, final int limit, + final FDBDatabase database, + final List> expectedBatches) { + try (FDBRecordContext context = database.openContext()) { + + final ScanProperties directionalProperties = forward ? ScanProperties.FORWARD_SCAN : ScanProperties.REVERSE_SCAN; + + final ScanProperties scanProperties = directionalProperties.with(props -> props.setReturnedRowLimit(limit)); + List> actual = new ArrayList<>(); + RecordCursorContinuation continuation = RecordCursorStartContinuation.START; + while (!continuation.isEnd()) { + final RecordCursor cursor = pathToExport.exportAllData(context, continuation.toBytes(), + scanProperties); + final AtomicReference> tupleResult = new AtomicReference<>(); + final List batch = cursor.map(dataInPath -> { + return Tuple.fromBytes(dataInPath.getValue()); + }).asList(tupleResult).join(); + actual.add(batch); + continuation = tupleResult.get().getContinuation(); + } + assertEquals(expectedBatches, actual); + } + } + + @Test + void exportAllDataThroughKeySpacePathWrapper() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Test export at different levels through wrapper methods + try (FDBRecordContext context = database.openContext()) { + // Export from root level (should get all data) + EnvironmentKeySpace.EnvironmentRoot root = keySpace.root(); + List allData = exportAllData(root, context); + assertEquals(6, allData.size(), "Root level should export all data"); + + // Export from specific user level (should get data for user 100 only) + EnvironmentKeySpace.UserPath user100Path = keySpace.root().userid(100L); + verifyExtractedData(exportAllData(user100Path, context), + 5, "User 100 should have 4 records", + "user100", "All user 100 data should contain 'user100'"); + + // Export from specific application level (app1 for user 100) + EnvironmentKeySpace.ApplicationPath app1User100 = user100Path.application("app1"); + verifyExtractedData(exportAllData(app1User100, context), + 4, "App1 for user 100 should have 4 records (3 data + 1 metadata)", + "user100_app1", "All app1 user100 data should contain 'user100_app1'"); + + // Export from specific data store level + EnvironmentKeySpace.DataPath dataStore = app1User100.dataStore(); + List dataStoreData = exportAllData(dataStore, context); + verifyExtractedData(dataStoreData, + 3, "Data store should have exactly 3 records", + "user100_app1_data", "Data should be from user100 app1 data store"); + + // Export from metadata store level + EnvironmentKeySpace.MetadataPath metadataStore = app1User100.metadataStore(); + verifyExtractedData(exportAllData(metadataStore, context), + 1, "Metadata store should have exactly 1 record", + "user100_app1_meta1", "Metadata value should match"); + + // Verify empty export for user with no data + EnvironmentKeySpace.UserPath user300Path = keySpace.root().userid(300L); + assertEquals(0, exportAllData(user300Path, context).size(), "User 300 should have no data"); + } + } + + @Test + void exportAllDataThroughKeySpacePathWrapperResolvedPaths() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Test export at different levels through wrapper methods + try (FDBRecordContext context = database.openContext()) { + // Test 4: Export from specific data store level + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + final List dataStoreData = dataStore.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .mapPipelined(DataInKeySpacePath::getResolvedPath, 1).asList().join(); + // Verify data store records have correct remainder + final ArrayList remainders = new ArrayList<>(); + for (ResolvedKeySpacePath kv : dataStoreData) { + // Path tuple should be the same + Tuple dataStoreTuple = dataStore.toTuple(context); + assertEquals(dataStoreTuple, kv.toTuple()); + remainders.add(kv.getRemainder()); + } + assertEquals(List.of( + Tuple.from("record1"), + Tuple.from("record2", 0), + Tuple.from("record2", 1) + ), remainders, "remainders should be the same"); + + } + } + + private void setData(List keys, FDBRecordContext context, KeySpacePath basePath, + String subdirectory, String valuePrefix) { + Transaction tr = context.ensureActive(); + for (int i = 0; i < keys.size(); i++) { + Tuple tuple = basePath.add(subdirectory, keys.get(i)).toTuple(context); + tr.set(tuple.pack(), Tuple.from(valuePrefix + i).pack()); + } + } + + /** + * Export all the data, and make some assertions that can always be done. + * This combines a lot of assertions, but most of the underlying behavior should be well covered by the objects + * that {@link KeySpacePath#exportAllData} is built on. + * @param pathToExport the path being exported + * @param context the context in which to export + * @return a list of {@code DataInKeySpacePath}s being exported + */ + private static List exportAllData(final KeySpacePath pathToExport, final FDBRecordContext context) { + final List asSingleExport = pathToExport.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .asList().join(); + + // assert that the resolved paths contain the right prefix + final List resolvedPaths = pathToExport.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .mapPipelined(DataInKeySpacePath::getResolvedPath, 1).asList().join(); + final ResolvedKeySpacePath rootResolvedPath = pathToExport.toResolvedPath(context); + for (ResolvedKeySpacePath resolvedPath : resolvedPaths) { + assertStartsWith(rootResolvedPath, resolvedPath); + } + + // assert that the reverse scan is the same as the forward scan, but in reverse + final List reversed = pathToExport.exportAllData(context, null, ScanProperties.REVERSE_SCAN) + .asList().join(); + Collections.reverse(reversed); + assertDataInKeySpacePathEquals(asSingleExport, reversed); + + // Assert continuations work correctly + final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(1)); + List asContinuations = new ArrayList<>(); + RecordCursorContinuation continuation = RecordCursorStartContinuation.START; + while (!continuation.isEnd()) { + final RecordCursor cursor = pathToExport.exportAllData(context, continuation.toBytes(), + scanProperties); + final AtomicReference> dataInPathResult = new AtomicReference<>(); + final List batch = cursor.asList(dataInPathResult).join(); + asContinuations.addAll(batch); + continuation = dataInPathResult.get().getContinuation(); + if (dataInPathResult.get().hasNext()) { + assertEquals(1, batch.size()); + } else { + assertThat(batch.size()).isLessThanOrEqualTo(1); + } + } + + assertDataInKeySpacePathEquals(asSingleExport, asContinuations); + return asSingleExport; + } + + private static void assertDataInKeySpacePathEquals(final List expectedList, + final List actualList) { + assertThat(actualList).zipSatisfy(expectedList, + (actual, other) -> { + assertThat(actual.getResolvedPath().join()).isEqualTo(other.getResolvedPath().join()); + assertThat(actual.getValue()).isEqualTo(other.getValue()); + }); + } + + private static void assertStartsWith(final ResolvedKeySpacePath rootResolvedPath, ResolvedKeySpacePath resolvedPath) { + ResolvedKeySpacePath searchPath = resolvedPath.withRemainder(null); + do { + if (searchPath.equals(rootResolvedPath)) { + return; + } + searchPath = searchPath.getParent(); + } while (searchPath != null); + Assertions.fail("Expected <" + resolvedPath + "> to start with <" + rootResolvedPath + "> but it didn't"); + } + + private static void verifyExtractedData(final List app1User100Data, + int expectedCount, String expectedCountMessage, + String expectedValueContents, String contentMessage) { + assertEquals(expectedCount, app1User100Data.size(), expectedCountMessage); + + for (DataInKeySpacePath data : app1User100Data) { + String value = Tuple.fromBytes(data.getValue()).getString(0); + assertTrue(value.contains(expectedValueContents), contentMessage); + } + } + +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java new file mode 100644 index 0000000000..a42f08b8c5 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -0,0 +1,447 @@ +/* + * KeySpacePathImportDataTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.Tags; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletionException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link KeySpacePath#importData(FDBRecordContext, Iterable)}. + */ +@Tag(Tags.RequiresFDB) +class KeySpacePathImportDataTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + private FDBDatabase database; + private List databases; + + @BeforeEach + void setUp() { + databases = dbExtension.getDatabases(2); + database = databases.get(0); + } + + @Test + void importComprehensiveData() { + // Test importing data covering ALL KeyType enum values in a single complex directory structure: + // - NULL, BYTES, STRING, LONG, FLOAT, DOUBLE, BOOLEAN, UUID + // - Constant value directories + // - Complex multi-level hierarchy + // - Different data types in remainder elements + // Given that the majority of complexity is in the components under the importData method, this is basically + // an integration test + final String rootUuid = UUID.randomUUID().toString(); + byte[] binaryId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + UUID memberId = UUID.randomUUID(); + + KeySpace root = new KeySpace( + new KeySpaceDirectory("company", KeyType.STRING, rootUuid) // STRING with constant + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) // LONG with constant + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) // STRING variable + .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG) // LONG variable + .addSubdirectory(new KeySpaceDirectory("binary_data", KeyType.BYTES) // BYTES + .addSubdirectory(new KeySpaceDirectory("null_section", KeyType.NULL) // NULL + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) // UUID + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) // BOOLEAN + .addSubdirectory(new KeySpaceDirectory("rating", KeyType.FLOAT)))))))))); // FLOAT + + + // Create comprehensive test data covering ALL KeyType values + KeySpacePath basePath = root.path("company").add("version").add("department", "engineering"); + + // Build paths using all KeyType values + KeySpacePath emp1Path = basePath.add("employee_id", 100L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", true) + .add("rating", 4.5f); + + KeySpacePath emp2Path = basePath.add("employee_id", 200L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", false) + .add("rating", 3.8f); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); + setToTuple(tr, key1, "John Doe"); + + byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); + setToTuple(tr, key2, "Jane Smith"); + + byte[] longKey = emp1Path.toSubspace(context).pack(Tuple.from("salary")); + setToTuple(tr, longKey, 75000); + + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + tr.set(binaryKey, "binary_test_data".getBytes()); + + byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); + setToTuple(tr, complexKey, "Complex Test"); + + context.commit(); + } + + copyData(root.path("company")); + + // Verify all different KeyType values were handled correctly during import + try (FDBRecordContext context = database.openContext()) { + byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); + byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); + assertEquals(Tuple.from("John Doe"), getTuple(context, key1)); + assertEquals(Tuple.from("Jane Smith"), getTuple(context, key2)); + + byte[] longKey = emp1Path.toSubspace(context).pack(Tuple.from("salary")); + assertEquals(Tuple.from(75000), getTuple(context, longKey)); + + // Verify BYTES data (raw binary, not in tuple) + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + assertArrayEquals("binary_test_data".getBytes(), context.ensureActive().get(binaryKey).join()); + + // Verify complex hierarchy with mixed types in remainder (LONG, BOOLEAN, STRING) + byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); + assertEquals(Tuple.from("Complex Test"), getTuple(context, complexKey)); + } + } + + @Test + void importEmptyData() { + // Test importing an empty collection of data + // Should complete successfully without modifying the data under the path + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + importData(database, testPath, Collections.emptyList()); // should not throw any exception + + assertTrue(getExportedData(testPath).isEmpty(), + "there should not have been any data created"); + } + + @Test + void importOverwriteExistingData() { + // Test importing data that overwrites existing keys + // Should verify that new data replaces old data when keys match + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); + setSingleKey(dataPath, Tuple.from("record"), Tuple.from("original_value")); + setSingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); + + // Create import data with same key but different value + List importData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("record")); + KeyValue kv = new KeyValue(key, Tuple.from("new_value").pack()); + importData.add(new DataInKeySpacePath(root.path("root"), kv, context)); + } + + // Verify we can re-import the data multiple times + importData(database, root.path("root"), importData); + importData(database, root.path("root"), importData); + importData(database, root.path("root"), importData); + + // Verify the data was overwritten + verifySingleKey(dataPath, Tuple.from("record"), Tuple.from("new_value")); + verifySingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); + } + + @Test + void importDataWithDirectoryLayer() { + // Test importing data into a keyspace using DirectoryLayer directories + // Should verify that DirectoryLayer mappings work correctly during import + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); + setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); + + copyData(root.path("tenant")); + + verifySingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); + } + + @Test + void importDataWithMismatchedPath() { + // Test importing data that doesn't belong to the target path + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + setSingleKey(keySpace.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); + + // Now try to ipmort that into keySpace2 + List exportedData = getExportedData(keySpace.path("root1")); + assertBadImport(keySpace.path("root2"), exportedData); + } + + @Test + void importDataWithInvalidPath() { + // Test importing data with paths that don't exist in the keyspace + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace1 = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + KeySpace keySpace2 = new KeySpace( + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); + + // Now try to ipmort that into keySpace2 + List exportedData = getExportedData(keySpace1.path("root1")); + assertBadImport(keySpace2.path("root2"), exportedData); + } + + @Test + void importDataWithSubdirectoryPath() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + KeySpacePath level1Path = root.path("root").add("level1", 1L); + + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + + // Export from root, import to subdirectory + List exportedData = getExportedData(root.path("root")); + + clearPath(database, root.path("root")); + + // Import only to level1 subdirectory + importData(database, level1Path, exportedData); + + verifySingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + } + + @Test + void importDataWithSubdirectoryPathFailure() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + KeySpacePath level1Path = root.path("root").add("level1", 1L); + + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + setSingleKey(root.path("root").add("level1", 2L), Tuple.from("item1"), Tuple.from("value1")); + + // Export from root, import to subdirectory + List exportedData = getExportedData(root.path("root")); + + clearPath(database, root.path("root")); + + // Import only to level1 subdirectory + assertBadImport(level1Path, exportedData); + } + + @Test + void importDataWithPartialMismatch() { + // Test importing data where the target path is a parent of some import data paths + // Should throw RecordCoreIllegalImportDataException for paths outside the target + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); + + KeySpacePath path1 = keySpace.path("root1").add("child", 1L); + KeySpacePath path2 = keySpace.path("root2").add("child", 2L); + setSingleKey(path1, Tuple.from("data"), Tuple.from("data1")); + setSingleKey(path2, Tuple.from("data"), Tuple.from("data2")); + + List mixedData = new ArrayList<>(); + mixedData.addAll(getExportedData(path1)); + mixedData.addAll(getExportedData(path2)); + + assertBadImport(keySpace.path("root1"), mixedData); + } + + @Test + void importDataWithWrapperClasses() { + // Test importing data using wrapper classes like EnvironmentKeySpace + // Should verify that wrapper functionality works correctly with import + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + + copyData(keySpace.root()); + + verifySingleKey(dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); + } + + @Test + void importDataWithDuplicateKeys() { + // Test importing data where the same key appears multiple times in the input + // Should verify that the last value wins for duplicate keys + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + KeySpacePath dataPath = root.path("root").add("data", 1L); + try (FDBRecordContext context = database.openContext()) { + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); + + // Create multiple DataInKeySpacePath objects with same key but different values + List duplicateData = Arrays.asList( + new DataInKeySpacePath(root.path("root"), + new KeyValue(key, Tuple.from("first_value").pack()), context), + new DataInKeySpacePath(root.path("root"), + new KeyValue(key, Tuple.from("second_value").pack()), context), + new DataInKeySpacePath(root.path("root"), + new KeyValue(key, Tuple.from("final_value").pack()), context) + ); + + root.path("root").importData(context, duplicateData).join(); + context.commit(); + } + + verifySingleKey(dataPath, Tuple.from("item"), Tuple.from("final_value")); + } + + private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { + try (FDBRecordContext context = database.openContext()) { + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, value.pack()); + context.commit(); + } + } + + private void verifySingleKey(KeySpacePath path, Tuple remainder, Tuple expected) { + try (FDBRecordContext context = database.openContext()) { + byte[] key = path.toSubspace(context).pack(remainder); + assertEquals(expected, getTuple(context, key)); + } + } + + private static void setToTuple(final Transaction tr, final byte[] key1, Object items) { + byte[] value1 = Tuple.from(items).pack(); + tr.set(key1, value1); + } + + private static Tuple getTuple(final FDBRecordContext context, final byte[] key) { + return Tuple.fromBytes(context.ensureActive().get(key).join()); + } + + private void copyData(final KeySpacePath path) { + // Export the data + final List exportedData = getExportedData(path); + + if (databases.size() > 1) { + database = databases.get(1); + } else { + // Clear the data and import it back + clearPath(database, path); + } + + // Import the data + importData(database, path, exportedData); + } + + private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { + try (FDBRecordContext context = database.openContext()) { + path.importData(context, exportedData).join(); + context.commit(); + } + } + + private void assertBadImport(final KeySpacePath path, final List invalidData) { + // Try to import into keySpace1 - should fail + try (FDBRecordContext context = database.openContext()) { + Assertions.assertThatThrownBy(() -> path.importData(context, invalidData).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RecordCoreIllegalImportDataException.class); + } + } + + private void clearPath(final FDBDatabase database, final KeySpacePath path) { + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + tr.clear(path.toSubspace(context).range()); + context.commit(); + } + // just an extra check to make sure the test is working as expected + assertTrue(getExportedData(path).isEmpty(), + "Clearing should remove all the data"); + } + + @Nonnull + private List getExportedData(final KeySpacePath path) { + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + return exportedData; + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathTest.java new file mode 100644 index 0000000000..ce6952d697 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathTest.java @@ -0,0 +1,212 @@ +/* + * KeySpacePathTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.record.RecordCoreArgumentException; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.BooleanSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.mockito.Mockito; + +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests for {@link KeySpacePath}. + * See also {@link KeySpacePathDataExportTest} and {@link ResolvedKeySpacePathTest}. + */ +class KeySpacePathTest { + + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + + private static final String ROOT_UUID = UUID.randomUUID().toString(); + + /** + * Creates a KeySpace for testing with a consistent structure. + * The structure is: root -> branch -> leaf + * @param useDirectoryLayer if true, leaf uses DirectoryLayerDirectory; otherwise uses KeySpaceDirectory + * @param useConstantValue if true, adds a constant value to the leaf directory + * @return a KeySpace with the specified configuration + */ + private KeySpace createKeySpace(boolean useDirectoryLayer, boolean useConstantValue) { + KeySpaceDirectory rootDir = new KeySpaceDirectory("test_root", KeySpaceDirectory.KeyType.STRING, ROOT_UUID); + KeySpaceDirectory branchDir = new KeySpaceDirectory("branch", KeySpaceDirectory.KeyType.STRING, "branch_value"); + + KeySpaceDirectory leafDir; + if (useDirectoryLayer) { + if (useConstantValue) { + leafDir = new DirectoryLayerDirectory("leaf", "leaf_constant"); + } else { + leafDir = new DirectoryLayerDirectory("leaf"); + } + } else { + if (useConstantValue) { + leafDir = new KeySpaceDirectory("leaf", KeySpaceDirectory.KeyType.STRING, "leaf_constant"); + } else { + leafDir = new KeySpaceDirectory("leaf", KeySpaceDirectory.KeyType.STRING); + } + } + + branchDir.addSubdirectory(leafDir); + rootDir.addSubdirectory(branchDir); + return new KeySpace(rootDir); + } + + @ParameterizedTest + @BooleanSource({"withRemainder", "useDirectoryLayer", "useConstantValue"}) + void testToResolvedPathAsync(boolean withRemainder, boolean useDirectoryLayer, boolean useConstantValue) { + final FDBDatabase database = dbExtension.getDatabase(); + final KeySpace keySpace = createKeySpace(useDirectoryLayer, useConstantValue); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath rootPath = keySpace.path("test_root"); + KeySpacePath branchPath = rootPath.add("branch"); + + // Build the full path - add leaf with or without value based on constant + KeySpacePath fullPath; + if (useConstantValue) { + fullPath = branchPath.add("leaf"); + } else { + fullPath = branchPath.add("leaf", "leaf_value"); + } + + // Create a key with or without remainder + byte[] keyBytes; + Tuple expectedRemainder; + if (withRemainder) { + expectedRemainder = Tuple.from("extra", "data"); + keyBytes = fullPath.toSubspace(context).pack(expectedRemainder); + } else { + expectedRemainder = null; + keyBytes = fullPath.toSubspace(context).pack(); + } + + // Test toResolvedPathAsync with the key + ResolvedKeySpacePath resolved = branchPath.toResolvedPathAsync(context, keyBytes).join(); + + assertEquals(fullPath.toResolvedPath(context), resolved.withRemainder(null)); + assertEquals(expectedRemainder, resolved.getRemainder()); + } + } + + @Test + void testToResolvedPathAsyncWithWrapper() { + final FDBDatabase database = dbExtension.getDatabase(); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + try (FDBRecordContext context = database.openContext()) { + // Use the wrapper paths which extend KeySpacePathWrapper + EnvironmentKeySpace.ApplicationPath appPath = keySpace.root().userid(100L).application("app1"); + EnvironmentKeySpace.DataPath dataPath = appPath.dataStore(); + + // Create a key with remainder + Tuple remainderTuple = Tuple.from("record_id", 42L, "version", 1); + byte[] keyBytes = dataPath.toSubspace(context).pack(remainderTuple); + + // Test toResolvedPathAsync on the wrapper - should resolve from appPath through dataPath + ResolvedKeySpacePath resolved = appPath.toResolvedPathAsync(context, keyBytes).join(); + + // Verify the resolved path + assertEquals(EnvironmentKeySpace.DATA_KEY, resolved.getDirectoryName()); + assertEquals(EnvironmentKeySpace.DATA_VALUE, resolved.getResolvedValue()); + assertEquals(remainderTuple, resolved.getRemainder()); + + // Verify parent structure + ResolvedKeySpacePath appLevel = resolved.getParent(); + assertNotNull(appLevel); + assertEquals(EnvironmentKeySpace.APPLICATION_KEY, appLevel.getDirectoryName()); + assertEquals("app1", appLevel.getLogicalValue()); + + ResolvedKeySpacePath userLevel = appLevel.getParent(); + assertNotNull(userLevel); + assertEquals(EnvironmentKeySpace.USER_KEY, userLevel.getDirectoryName()); + assertEquals(100L, userLevel.getResolvedValue()); + } + } + + @Test + void testToResolvedPathAsyncWithKeyNotSubPath() { + final FDBDatabase database = dbExtension.getDatabase(); + final KeySpace keySpace = createKeySpace(false, false); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath rootPath = keySpace.path("test_root"); + KeySpacePath branchPath = rootPath.add("branch"); + + // Create a key that is shorter than branchPath - it stops at root + byte[] shorterKeyBytes = rootPath.toSubspace(context).pack(); + + // Attempting to resolve a key that is not under branchPath should error + ExecutionException ex = assertThrows(ExecutionException.class, + () -> branchPath.toResolvedPathAsync(context, shorterKeyBytes).get()); + assertEquals(RecordCoreArgumentException.class, ex.getCause().getClass()); + } + } + + @Test + void testToResolvedPathAsyncWithInvalidTuple() { + final FDBDatabase database = dbExtension.getDatabase(); + final KeySpace keySpace = createKeySpace(false, false); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath rootPath = keySpace.path("test_root"); + KeySpacePath branchPath = rootPath.add("branch"); + + // Create a byte array that is not a valid tuple + byte[] invalidBytes = new byte[]{(byte) 0xFF, (byte) 0xFF, (byte) 0xFF}; + + // Attempting to resolve invalid tuple bytes should error + // The exception is thrown synchronously from Tuple.fromBytes, not wrapped in ExecutionException + assertThrows(IllegalArgumentException.class, () -> branchPath.toResolvedPathAsync(context, invalidBytes)); + } + } + + /** + * Test of methods with default implementations to ensure backwards compatibility, + * in case someone is implementing {@link KeySpacePath}. + **/ + @Test + void testDefaultMethods() { + final KeySpacePath mock = Mockito.mock(KeySpacePath.class); + final FDBDatabase database = dbExtension.getDatabase(); + + try (FDBRecordContext context = database.openContext()) { + // thenCallReadMethod throws an error if there is not a default implementation + Mockito.when(mock.toResolvedPathAsync(Mockito.any(), Mockito.any())).thenCallRealMethod(); + assertThrows(UnsupportedOperationException.class, + () -> mock.toResolvedPathAsync(context, Tuple.from("foo").pack())); + Mockito.when(mock.exportAllData(Mockito.any(), Mockito.any(), Mockito.any())).thenCallRealMethod(); + assertThrows(UnsupportedOperationException.class, + () -> mock.exportAllData(context, null, ScanProperties.FORWARD_SCAN)); + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePathTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePathTest.java index be82a58eb6..a86f3d4bd5 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePathTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePathTest.java @@ -224,6 +224,80 @@ void testRemainderComparedInEquals(boolean constantDirectory) { .orElseThrow(() -> new AssertionError("Paths with different remainders should sometimes have different hash codes")); } + /** Test withRemainder returns a new path with the updated remainder. */ + @ParameterizedTest + @BooleanSource("constantDirectory") + void testWithRemainder(boolean constantDirectory) { + KeySpacePath innerPath = createKeySpacePath(createRootParent(), KeyType.STRING, "resolved", constantDirectory); + PathValue value = new PathValue("resolved", null); + Tuple originalRemainder = Tuple.from("remainder1"); + + ResolvedKeySpacePath original = new ResolvedKeySpacePath(null, innerPath, value, originalRemainder); + + // Test changing remainder + Tuple newRemainder = Tuple.from("remainder2"); + ResolvedKeySpacePath modified = original.withRemainder(newRemainder); + + assertEquals(newRemainder, modified.getRemainder()); + assertEquals(originalRemainder, original.getRemainder(), "Original should be unchanged"); + assertNotEquals(original, modified, "Paths with different remainders should not be equal"); + + // Verify other fields are preserved + assertEquals(original.toPath(), modified.toPath()); + assertEquals(original.getResolvedValue(), modified.getResolvedValue()); + assertEquals(original.getResolvedPathValue(), modified.getResolvedPathValue()); + + // Verify modified is equal to a freshly created path with the new remainder + ResolvedKeySpacePath freshlyCreated = new ResolvedKeySpacePath(null, innerPath, value, newRemainder); + assertEquals(freshlyCreated, modified); + assertEquals(modified, freshlyCreated); + assertEquals(freshlyCreated.hashCode(), modified.hashCode()); + } + + /** Test withRemainder can set remainder to null. */ + @ParameterizedTest + @BooleanSource("constantDirectory") + void testWithRemainderSetToNull(boolean constantDirectory) { + KeySpacePath innerPath = createKeySpacePath(createRootParent(), KeyType.STRING, "resolved", constantDirectory); + PathValue value = new PathValue("resolved", null); + Tuple originalRemainder = Tuple.from("remainder1"); + + ResolvedKeySpacePath original = new ResolvedKeySpacePath(null, innerPath, value, originalRemainder); + ResolvedKeySpacePath modified = original.withRemainder(null); + + assertNull(modified.getRemainder()); + assertEquals(originalRemainder, original.getRemainder(), "Original should be unchanged"); + assertNotEquals(original, modified); + + // Verify modified is equal to a freshly created path with null remainder + ResolvedKeySpacePath freshlyCreated = new ResolvedKeySpacePath(null, innerPath, value, null); + assertEquals(freshlyCreated, modified); + assertEquals(modified, freshlyCreated); + assertEquals(freshlyCreated.hashCode(), modified.hashCode()); + } + + /** Test withRemainder on a path that already has null remainder. */ + @ParameterizedTest + @BooleanSource("constantDirectory") + void testWithRemainderFromNull(boolean constantDirectory) { + KeySpacePath innerPath = createKeySpacePath(createRootParent(), KeyType.STRING, "resolved", constantDirectory); + PathValue value = new PathValue("resolved", null); + + ResolvedKeySpacePath original = new ResolvedKeySpacePath(null, innerPath, value, null); + Tuple newRemainder = Tuple.from("newRemainder"); + ResolvedKeySpacePath modified = original.withRemainder(newRemainder); + + assertEquals(newRemainder, modified.getRemainder()); + assertNull(original.getRemainder(), "Original should be unchanged"); + assertNotEquals(original, modified); + + // Verify modified is equal to a freshly created path with the new remainder + ResolvedKeySpacePath freshlyCreated = new ResolvedKeySpacePath(null, innerPath, value, newRemainder); + assertEquals(freshlyCreated, modified); + assertEquals(modified, freshlyCreated); + assertEquals(freshlyCreated.hashCode(), modified.hashCode()); + } + @Nonnull private KeySpacePath createKeySpacePath(@Nonnull ResolvedKeySpacePath parent, @Nonnull KeyType keyType, @Nullable Object value, boolean constantDirectory) { @@ -235,7 +309,7 @@ private KeySpacePath createKeySpacePath(@Nonnull ResolvedKeySpacePath parent, @N childDir = new KeySpaceDirectory("test", keyType); } parent.getDirectory().addSubdirectory(childDir); - + if (constantDirectory) { return parent.toPath().add("test"); } else { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java index a1309989f7..7d7f84bc09 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java @@ -36,10 +36,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -161,6 +165,12 @@ public FDBDatabase getDatabase(@Nullable String clusterFile) { }); } + public List getDatabases(int count) { + List clusterFiles = new ArrayList<>(FDBTestEnvironment.allClusterFiles()); + Collections.shuffle(clusterFiles); + return clusterFiles.stream().limit(count).map(this::getDatabase).collect(Collectors.toList()); + } + public void checkForOpenContexts() { for (final Map.Entry clusterFileToDatabase : databases.entrySet()) { assertEquals(0, clusterFileToDatabase.getValue().warnAndCloseOldTrackedOpenContexts(0),