diff --git a/libs/common/Crc64.cs b/libs/common/Crc64.cs new file mode 100644 index 0000000000..d1422ff7e3 --- /dev/null +++ b/libs/common/Crc64.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Garnet.common; + +/// +/// Port of redis crc64 from https://github.com/redis/redis/blob/7.2/src/crc64.c +/// +public static class Crc64 +{ + /// + /// Polynomial (same as redis) + /// + private const ulong POLY = 0xad93d23594c935a9UL; + + /// + /// Reverse all bits in a 64-bit value (bit reflection). + /// Only used for data_len == 64 in this code. + /// + private static ulong Reflect64(ulong data) + { + // swap odd/even bits + data = ((data >> 1) & 0x5555555555555555UL) | ((data & 0x5555555555555555UL) << 1); + // swap consecutive pairs + data = ((data >> 2) & 0x3333333333333333UL) | ((data & 0x3333333333333333UL) << 2); + // swap nibbles + data = ((data >> 4) & 0x0F0F0F0F0F0F0F0FUL) | ((data & 0x0F0F0F0F0F0F0F0FUL) << 4); + // swap bytes, then 2-byte pairs, then 4-byte pairs + data = System.Buffers.Binary.BinaryPrimitives.ReverseEndianness(data); + return data; + } + + /// + /// A direct bit-by-bit CRC64 calculation (like _crc64 in C). + /// + private static ulong Crc64Bitwise(ReadOnlySpan data) + { + ulong crc = 0; + + foreach (var c in data) + { + for (byte i = 1; i != 0; i <<= 1) + { + // interpret the top bit of 'crc' and current bit of 'c' + var bitSet = (crc & 0x8000000000000000UL) != 0; + var cbit = (c & i) != 0; + + // if cbit flips the sense, invert bitSet + if (cbit) + bitSet = !bitSet; + + // shift + crc <<= 1; + + // apply polynomial if needed + if (bitSet) + crc ^= POLY; + } + + // ensure it stays in 64 bits + crc &= 0xffffffffffffffffUL; + } + + // reflect and XOR, per standard + crc &= 0xffffffffffffffffUL; + crc = Reflect64(crc) ^ 0x0000000000000000UL; + return crc; + } + + /// + /// Computes crc64 + /// + /// + /// + public static byte[] Hash(ReadOnlySpan data) + { + var bitwiseCrc = Crc64Bitwise(data); + return BitConverter.GetBytes(bitwiseCrc); + } +} \ No newline at end of file diff --git a/libs/common/RespLengthEncodingUtils.cs b/libs/common/RespLengthEncodingUtils.cs new file mode 100644 index 0000000000..3116f34ae3 --- /dev/null +++ b/libs/common/RespLengthEncodingUtils.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers.Binary; + +namespace Garnet.common; + +/// +/// Utils for working with RESP length encoding +/// +public static class RespLengthEncodingUtils +{ + /// + /// Maximum length that can be encoded + /// + private const int MaxLength = 0xFFFFFF; + + /// + /// Try read RESP-encoded length + /// + /// + /// + /// + /// + public static bool TryReadLength(ReadOnlySpan input, out int length, out int bytesRead) + { + length = 0; + bytesRead = 0; + if (input.Length < 1) + { + return false; + } + + var firstByte = input[0]; + switch (firstByte >> 6) + { + case 0: + bytesRead = 1; + length = firstByte & 0x3F; + return true; + case 1 when input.Length > 1: + bytesRead = 2; + length = ((firstByte & 0x3F) << 8) | input[1]; + return true; + case 2: + bytesRead = 5; + return BinaryPrimitives.TryReadInt32BigEndian(input, out length); + default: + return false; + } + } + + /// + /// Try to write RESP-encoded length + /// + /// + /// + /// + /// + public static bool TryWriteLength(int length, Span output, out int bytesWritten) + { + bytesWritten = 0; + + if (length > MaxLength) + { + return false; + } + + // 6-bit encoding (length ≤ 63) + if (length < 1 << 6) + { + if (output.Length < 1) + { + return false; + } + + output[0] = (byte)(length & 0x3F); + + bytesWritten = 1; + return true; + } + + // 14-bit encoding (64 ≤ length ≤ 16,383) + if (length < 1 << 14) + { + if (output.Length < 2) + { + return false; + } + + output[0] = (byte)(((length >> 8) & 0x3F) | (1 << 6)); + output[1] = (byte)(length & 0xFF); + + bytesWritten = 2; + return true; + } + + // 32-bit encoding (length ≤ 4,294,967,295) + if (output.Length < 5) + { + return false; + } + + output[0] = 2 << 6; + BinaryPrimitives.WriteUInt32BigEndian(output.Slice(1), (uint)length); + + bytesWritten = 5; + return true; + } +} \ No newline at end of file diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 8922292bf9..eba93826c8 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -509,6 +509,10 @@ internal sealed class Options [Option("fail-on-recovery-error", Default = false, Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")] public bool? FailOnRecoveryError { get; set; } + [OptionValidation] + [Option("skip-rdb-restore-checksum-validation", Default = false, Required = false, HelpText = "Skip RDB restore checksum validation")] + public bool? SkipRDBRestoreChecksumValidation { get; set; } + [Option("lua-memory-management-mode", Default = LuaMemoryManagementMode.Native, Required = false, HelpText = "Memory management mode for Lua scripts, must be set to LimittedNative or Managed to impose script limits")] public LuaMemoryManagementMode LuaMemoryManagementMode { get; set; } @@ -728,6 +732,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) IndexResizeThreshold = IndexResizeThreshold, LoadModuleCS = LoadModuleCS, FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(), + SkipRDBRestoreChecksumValidation = SkipRDBRestoreChecksumValidation.GetValueOrDefault(), LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, logger) : null, }; } diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index 4c08ebec06..5c82c4b37d 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -344,6 +344,9 @@ /* Fails if encounters error during AOF replay or checkpointing */ "FailOnRecoveryError": false, + /* Skips crc64 validation in restore command */ + "SkipRDBRestoreChecksumValidation": false, + /* Lua uses the default, unmanaged and untracked, allocator */ "LuaMemoryManagementMode": "Native", diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index b57c74dab4..ed2d937ffe 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -748,7 +748,6 @@ "Summary": "Pops an element from a list, pushes it to another list and returns it. Block until an element is available otherwise. Deletes the list if the last element was popped.", "Group": "List", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060BLMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments", "Arguments": [ { @@ -1427,7 +1426,6 @@ "Summary": "Returns the mapping of cluster slots to nodes.", "Group": "Cluster", "Complexity": "O(N) where N is the total number of Cluster nodes", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060CLUSTER SHARDS\u0060" } ] @@ -1654,6 +1652,22 @@ "Group": "Transactions", "Complexity": "O(N), when N is the number of queued commands" }, + { + "Command": "DUMP", + "Name": "DUMP", + "Summary": "Returns a serialized representation of the value stored at a key.", + "Group": "Generic", + "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1).", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, { "Command": "ECHO", "Name": "ECHO", @@ -2821,7 +2835,6 @@ "Summary": "Returns the previous string value of a key after setting it to a new value.", "Group": "String", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060SET\u0060 with the \u0060!GET\u0060 argument", "Arguments": [ { @@ -3107,7 +3120,6 @@ "Summary": "Sets the values of multiple fields.", "Group": "Hash", "Complexity": "O(N) where N is the number of fields being set.", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060HSET\u0060 with multiple field-value pairs", "Arguments": [ { @@ -4452,7 +4464,6 @@ "Summary": "Sets both string value and expiration time in milliseconds of a key. The key is created if it doesn\u0027t exist.", "Group": "String", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060SET\u0060 with the \u0060PX\u0060 argument", "Arguments": [ { @@ -4635,7 +4646,6 @@ "Summary": "Closes the connection.", "Group": "Connection", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "just closing the connection" }, { @@ -4878,6 +4888,34 @@ } ] }, + { + "Command": "RESTORE", + "Name": "RESTORE", + "Summary": "Creates a key from the serialized representation of a value.", + "Group": "Generic", + "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "TTL", + "DisplayText": "ttl", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SERIALIZEDVALUE", + "DisplayText": "serialized-value", + "Type": "String" + } + ] + }, { "Command": "RPOP", "Name": "RPOP", @@ -4907,7 +4945,6 @@ "Summary": "Returns the last element of a list after removing and pushing it to another list. Deletes the list if the last element was popped.", "Group": "List", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060LMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments", "Arguments": [ { @@ -5400,7 +5437,6 @@ "Summary": "Sets the string value and expiration time of a key. Creates the key if it doesn\u0027t exist.", "Group": "String", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060SET\u0060 with the \u0060EX\u0060 argument", "Arguments": [ { @@ -5480,7 +5516,6 @@ "Summary": "Set the string value of a key only when the key doesn\u0027t exist.", "Group": "String", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060SET\u0060 with the \u0060NX\u0060 argument", "Arguments": [ { @@ -5626,7 +5661,6 @@ "Summary": "Sets a Redis server as a replica of another, or promotes it to being a master.", "Group": "Server", "Complexity": "O(1)", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060REPLICAOF\u0060", "Arguments": [ { @@ -5891,7 +5925,6 @@ "Summary": "Returns a substring from a string value.", "Group": "String", "Complexity": "O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060GETRANGE\u0060", "Arguments": [ { @@ -6763,7 +6796,6 @@ "Summary": "Returns members in a sorted set within a range of scores.", "Group": "SortedSet", "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060BYSCORE\u0060 argument", "Arguments": [ { @@ -7045,7 +7077,6 @@ "Summary": "Returns members in a sorted set within a range of indexes in reverse order.", "Group": "SortedSet", "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements returned.", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 argument", "Arguments": [ { @@ -7083,7 +7114,6 @@ "Summary": "Returns members in a sorted set within a lexicographical range in reverse order.", "Group": "SortedSet", "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYLEX\u0060 arguments", "Arguments": [ { @@ -7134,7 +7164,6 @@ "Summary": "Returns members in a sorted set within a range of scores in reverse order.", "Group": "SortedSet", "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", - "DocFlags": "Deprecated", "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYSCORE\u0060 arguments", "Arguments": [ { diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 340f4c2d17..dcd2da2d49 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -1034,6 +1034,31 @@ "Flags": "Fast, Loading, NoScript, Stale, AllowBusy", "AclCategories": "Fast, Transaction" }, + { + "Command": "DUMP", + "Name": "DUMP", + "Arity": 2, + "Flags": "ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "KeySpace, Read", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "ECHO", "Name": "ECHO", @@ -3139,6 +3164,31 @@ "Flags": "Admin, NoAsyncLoading, NoScript, Stale", "AclCategories": "Admin, Dangerous, Slow" }, + { + "Command": "RESTORE", + "Name": "RESTORE", + "Arity": -4, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Dangerous, KeySpace, Slow, Write", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 0, + "Limit": 0 + }, + "Flags": "OW, Update" + } + ] + }, { "Command": "RPOP", "Name": "RPOP", diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index 8f32e0bbea..4aa8278453 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -227,6 +227,7 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_INCR_SUPPORTS_ONLY_SINGLE_PAIR => "ERR INCR option supports a single increment-element pair"u8; public static ReadOnlySpan RESP_ERR_INVALID_BITFIELD_TYPE => "ERR Invalid bitfield type. Use something like i16 u8. Note that u64 is not supported but i64 is"u8; public static ReadOnlySpan RESP_ERR_SCRIPT_FLUSH_OPTIONS => "ERR SCRIPT FLUSH only support SYNC|ASYNC option"u8; + public static ReadOnlySpan RESP_ERR_BUSSYKEY => "BUSYKEY Target key name already exists."u8; public static ReadOnlySpan RESP_ERR_LENGTH_AND_INDEXES => "If you want both the length and indexes, please just use IDX."u8; /// diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index 1ce25dd487..972edea133 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -2,14 +2,228 @@ // Licensed under the MIT license. using System; +using System.Buffers; using System.Diagnostics; using Garnet.common; +using Garnet.server.Auth; using Tsavorite.core; namespace Garnet.server { internal sealed unsafe partial class RespServerSession : ServerSessionBase { + /// + /// RDB format version + /// + private readonly byte RDB_VERSION = 11; + + /// + /// RESTORE + /// + /// + /// + /// + bool NetworkRESTORE(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 3) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RESTORE)); + } + + var key = parseState.GetArgSliceByRef(0); + + if (!parseState.TryGetInt(parseState.Count - 2, out var expiry)) + { + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_TIMEOUT_NOT_VALID_FLOAT, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var value = parseState.GetArgSliceByRef(2); + + var valueSpan = value.ReadOnlySpan; + + // Restore is only implemented for string type + if (valueSpan[0] != 0x00) + { + while (!RespWriteUtils.WriteError("ERR RESTORE currently only supports string types", ref dcurr, dend)) + SendAndReset(); + return true; + } + + // check if length of value is at least 10 + if (valueSpan.Length < 10) + { + while (!RespWriteUtils.WriteError("ERR DUMP payload version or checksum are wrong", ref dcurr, dend)) + SendAndReset(); + return true; + } + + // get footer (2 bytes of rdb version + 8 bytes of crc) + var footer = valueSpan[^10..]; + + var rdbVersion = (footer[1] << 8) | footer[0]; + + if (rdbVersion > RDB_VERSION) + { + while (!RespWriteUtils.WriteError("ERR DUMP payload version or checksum are wrong", ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (storeWrapper.serverOptions.SkipRDBRestoreChecksumValidation) + { + // crc is calculated over the encoded payload length, payload and the rdb version bytes + // skip's the value type byte and crc64 bytes + var calculatedCrc = new ReadOnlySpan(Crc64.Hash(valueSpan.Slice(0, valueSpan.Length - 8))); + + // skip's rdb version bytes + var payloadCrc = footer[2..]; + + if (calculatedCrc.SequenceCompareTo(payloadCrc) != 0) + { + while (!RespWriteUtils.WriteError("ERR DUMP payload version or checksum are wrong", ref dcurr, dend)) + SendAndReset(); + return true; + } + } + + // decode the length of payload + if (!RespLengthEncodingUtils.TryReadLength(valueSpan.Slice(1), out var length, out var payloadStart)) + { + while (!RespWriteUtils.WriteError("ERR DUMP payload length format is invalid", ref dcurr, dend)) + SendAndReset(); + return true; + } + + // Start from payload start and skip the value type byte + var val = value.ReadOnlySpan.Slice(payloadStart + 1, length); + + var valArgSlice = scratchBufferManager.CreateArgSlice(val); + + var sbKey = key.SpanByte; + + parseState.InitializeWithArgument(valArgSlice); + + RawStringInput input; + if (expiry > 0) + { + var inputArg = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromSeconds(expiry).Ticks; + input = new RawStringInput(RespCommand.SETEXNX, ref parseState, arg1: inputArg); + } + else + { + input = new RawStringInput(RespCommand.SETEXNX, ref parseState); + } + + var status = storageApi.SET_Conditional(ref sbKey, ref input); + + if (status is GarnetStatus.NOTFOUND) + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + return true; + } + + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_BUSSYKEY, ref dcurr, dend)) + SendAndReset(); + + return true; + } + + /// + /// DUMP + /// + bool NetworkDUMP(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 1) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.DUMP)); + } + + var key = parseState.GetArgSliceByRef(0); + + var status = storageApi.GET(key, out var value); + + if (status is GarnetStatus.NOTFOUND) + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + return true; + } + + Span encodedLength = stackalloc byte[5]; + + if (!RespLengthEncodingUtils.TryWriteLength(value.ReadOnlySpan.Length, encodedLength, out var bytesWritten)) + { + while (!RespWriteUtils.WriteError("ERR DUMP payload length is invalid", ref dcurr, dend)) + SendAndReset(); + return true; + } + + encodedLength = encodedLength.Slice(0, bytesWritten); + + // Len of the dump (payload type + redis encoded payload len + payload len + rdb version + crc64) + var len = 1 + encodedLength.Length + value.ReadOnlySpan.Length + 2 + 8; + Span lengthInASCIIBytes = stackalloc byte[NumUtils.NumDigitsInLong(len)]; + var lengthInASCIIBytesLen = NumUtils.LongToSpanByte(len, lengthInASCIIBytes); + + // Total len (% + length of ascii bytes + CR LF + payload type + redis encoded payload len + payload len + rdb version + crc64 + CR LF) + var totalLength = 1 + lengthInASCIIBytesLen + 2 + 1 + encodedLength.Length + value.ReadOnlySpan.Length + 2 + 8 + 2; + + byte[] rentedBuffer = null; + var buffer = totalLength <= 256 + ? stackalloc byte[256] + : (rentedBuffer = ArrayPool.Shared.Rent(totalLength)); + + var offset = 0; + + // Write RESP bulk string prefix and length + buffer[offset++] = 0x24; // '$' + lengthInASCIIBytes.CopyTo(buffer[offset..]); + offset += lengthInASCIIBytes.Length; + buffer[offset++] = 0x0D; // CR + buffer[offset++] = 0x0A; // LF + + // value type byte + buffer[offset++] = 0x00; + + // length of the span + encodedLength.CopyTo(buffer[offset..]); + offset += encodedLength.Length; + + // copy value to buffer + value.ReadOnlySpan.CopyTo(buffer[offset..]); + offset += value.ReadOnlySpan.Length; + + // Write RDB version + buffer[offset++] = (byte)(RDB_VERSION & 0xff); + buffer[offset++] = (byte)((RDB_VERSION >> 8) & 0xff); + + // Compute and write CRC64 checksum + var payloadToHash = buffer.Slice(1 + lengthInASCIIBytes.Length + 2 + 1, + encodedLength.Length + value.ReadOnlySpan.Length + 2); + + var crcBytes = Crc64.Hash(payloadToHash); + crcBytes.CopyTo(buffer[offset..]); + offset += crcBytes.Length; + + // Write final CRLF + buffer[offset++] = 0x0D; // CR + buffer[offset] = 0x0A; // LF + + while (!RespWriteUtils.WriteDirect(buffer.Slice(0, totalLength), ref dcurr, dend)) + SendAndReset(); + + if (rentedBuffer is not null) + { + ArrayPool.Shared.Return(rentedBuffer); + } + + return true; + } /// /// TryRENAME diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index c2e961b7d5..cf771c5b5a 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -25,6 +25,7 @@ public enum RespCommand : ushort BITPOS, COSCAN, DBSIZE, + DUMP, EXISTS, EXPIRETIME, GEODIST, @@ -142,6 +143,7 @@ public enum RespCommand : ushort PFMERGE, PSETEX, RENAME, + RESTORE, RENAMENX, RPOP, RPOPLPUSH, @@ -668,6 +670,7 @@ private RespCommand FastParseCommand(out int count) (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDUMP\r\n"u8) => RespCommand.DUMP, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, @@ -689,6 +692,7 @@ private RespCommand FastParseCommand(out int count) (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SETBIT\r\n"u8) => RespCommand.SETBIT, (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SUBSTR\r\n"u8) => RespCommand.SUBSTR, + (3 << 4) | 7 when lastWord == MemoryMarshal.Read("ESTORE\r\n"u8) && ptr[8] == 'R' => RespCommand.RESTORE, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SE"u8) => RespCommand.SETRANGE, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("GE"u8) => RespCommand.GETRANGE, diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index aaf064f57a..8e387b8e26 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -577,6 +577,8 @@ private bool ProcessBasicCommands(RespCommand cmd, ref TGarnetApi st RespCommand.READWRITE => NetworkREADWRITE(), RespCommand.EXPIREAT => NetworkEXPIREAT(RespCommand.EXPIREAT, ref storageApi), RespCommand.PEXPIREAT => NetworkEXPIREAT(RespCommand.PEXPIREAT, ref storageApi), + RespCommand.DUMP => NetworkDUMP(ref storageApi), + RespCommand.RESTORE => NetworkRESTORE(ref storageApi), _ => ProcessArrayCommands(cmd, ref storageApi) }; diff --git a/libs/server/Servers/ServerOptions.cs b/libs/server/Servers/ServerOptions.cs index efa6208859..92795ba60d 100644 --- a/libs/server/Servers/ServerOptions.cs +++ b/libs/server/Servers/ServerOptions.cs @@ -98,6 +98,11 @@ public class ServerOptions /// public bool FailOnRecoveryError = false; + /// + /// Skip RDB restore checksum validation + /// + public bool SkipRDBRestoreChecksumValidation = false; + /// /// Logger /// diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index dbef2f2b7f..d972c08e98 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -110,6 +110,7 @@ public class SupportedCommand new("CustomTxn", RespCommand.CustomTxn), new("CustomProcedure", RespCommand.CustomProcedure), new("DBSIZE", RespCommand.DBSIZE), + new("DUMP", RespCommand.DUMP), new("DECR", RespCommand.DECR), new("DECRBY", RespCommand.DECRBY), new("DEL", RespCommand.DEL), @@ -220,6 +221,7 @@ public class SupportedCommand new("READONLY", RespCommand.READONLY), new("READWRITE", RespCommand.READWRITE), new("RENAME", RespCommand.RENAME), + new("RESTORE", RespCommand.RESTORE), new("RENAMENX", RespCommand.RENAMENX), new("REPLICAOF", RespCommand.REPLICAOF), new("RPOP", RespCommand.RPOP), diff --git a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs index 5d7da8fd4e..e73698b415 100644 --- a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs +++ b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs @@ -663,6 +663,40 @@ public override string[] GetSingleSlotRequest() public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); } + internal class DUMP : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(DUMP); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0]]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class RESTORE : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(RESTORE); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], ssk[1], ssk[2]]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + internal class WATCH : BaseCommand { public override bool IsArrayCommand => true; diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs index e1ae064223..05693c702c 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs @@ -58,6 +58,8 @@ public class ClusterSlotVerificationTests new PERSIST(), new EXPIRE(), new TTL(), + new DUMP(), + new RESTORE(), new SDIFFSTORE(), new SDIFF(), new SMOVE(), @@ -250,6 +252,8 @@ public virtual void OneTimeTearDown() [TestCase("PERSIST")] [TestCase("EXPIRE")] [TestCase("TTL")] + [TestCase("DUMP")] + [TestCase("RESTORE")] [TestCase("SDIFFSTORE")] [TestCase("SDIFF")] [TestCase("SMOVE")] @@ -402,6 +406,8 @@ void GarnetClientSessionClusterDown(BaseCommand command) [TestCase("PERSIST")] [TestCase("EXPIRE")] [TestCase("TTL")] + [TestCase("DUMP")] + [TestCase("RESTORE")] [TestCase("SDIFFSTORE")] [TestCase("SDIFF")] [TestCase("SMOVE")] @@ -565,6 +571,8 @@ void GarnetClientSessionOK(BaseCommand command) [TestCase("PERSIST")] [TestCase("EXPIRE")] [TestCase("TTL")] + [TestCase("DUMP")] + [TestCase("RESTORE")] [TestCase("SDIFFSTORE")] [TestCase("SDIFF")] [TestCase("SMOVE")] @@ -883,6 +891,8 @@ void GarnetClientSessionMOVEDTest(BaseCommand command) [TestCase("PERSIST")] [TestCase("EXPIRE")] [TestCase("TTL")] + [TestCase("DUMP")] + [TestCase("RESTORE")] [TestCase("SDIFFSTORE")] [TestCase("SDIFF")] [TestCase("SMOVE")] @@ -1063,6 +1073,8 @@ void GarnetClientSessionASKTest(BaseCommand command) [TestCase("PERSIST")] [TestCase("EXPIRE")] [TestCase("TTL")] + [TestCase("DUMP")] + [TestCase("RESTORE")] [TestCase("SDIFFSTORE")] [TestCase("SDIFF")] [TestCase("SMOVE")] diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index de4ba5bdfd..8dc10d1655 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,6 +6,7 @@ using System.Linq; using System.Numerics; using System.Reflection; +using System.Text; using System.Threading.Tasks; using Garnet.client; using Garnet.server; @@ -6397,6 +6398,56 @@ static async Task DoTTLAsync(GarnetClient client) } } + [Test] + public async Task DumpACLsAsync() + { + await CheckCommandsAsync( + "DUMP", + [DoDUMPAsync] + ); + + static async Task DoDUMPAsync(GarnetClient client) + { + string val = await client.ExecuteForStringResultAsync("DUMP", ["foo"]); + ClassicAssert.IsNull(val); + } + } + + [Test] + public async Task RestoreACLsAsync() + { + var count = 0; + + await CheckCommandsAsync( + "RESTORE", + [DoRestoreAsync] + ); + + async Task DoRestoreAsync(GarnetClient client) + { + var payload = new byte[] + { + 0x00, // value type + 0x03, // length of payload + 0x76, 0x61, 0x6C, // 'v', 'a', 'l' + 0x0B, 0x00, // RDB version + 0xDB, 0x82, 0x3C, 0x30, 0x38, 0x78, 0x5A, 0x99 // Crc64 + }; + + count++; + + var val = await client.ExecuteForStringResultAsync( + "$7\r\nRESTORE\r\n"u8.ToArray(), + [ + Encoding.UTF8.GetBytes($"foo-{count}"), + "0"u8.ToArray(), + payload + ]); + + ClassicAssert.AreEqual("OK", val); + } + } + [Test] public async Task TypeACLsAsync() { @@ -6845,6 +6896,5 @@ private static async Task CheckAuthFailureAsync(Func act) return false; } } - } } \ No newline at end of file diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 62127deb27..670b0f4f69 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -136,6 +136,312 @@ public void IsClusterSubCommand() } } + /// + /// Tests RESTORE value that is not string + /// + [Test] + public void TryRestoreKeyNonStringType() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var payload = new byte[] + { + 0x14, 0xf, 0xf, 0x0, 0x0, 0x0, 0x1, 0x0, 0xc2, 0x86, 0x62, 0x69, 0x6b, 0x65, 0x3a, 0x31, 0x7, 0xc3, 0xbf, 0xb, 0x0, 0xc3, 0xaa, 0x33, 0x68, 0x7b, 0x2a, 0xc3, 0xa6, 0xc3, 0xbf, 0xc3, 0xb9 + }; + + Assert.Throws(() => db.KeyRestore("mykey", payload)); + } + + /// + /// Tests RESTORE command on existing key + /// + [Test] + public void TryRestoreExistingKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey")!; + + Assert.Throws(() => db.KeyRestore("mykey", dump)); + } + + /// + /// Tests that RESTORE command restores payload with 32 bit encoded length + /// + [Test] + public void SingleRestore32Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383; i++) + valueBuilder.Append('a'); + + var val = valueBuilder.ToString(); + + db.StringSet("mykey", val); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual(val, value.ToString()); + } + + /// + /// Tests that RESTORE command restores payload with 14 bit encoded length + /// + [Test] + public void SingleRestore14Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383 - 1; i++) + valueBuilder.Append('a'); + + var val = valueBuilder.ToString(); + + db.StringSet("mykey", val); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual(val, value.ToString()); + } + + /// + /// Tests that RESTORE command restores payload with 6 bit encoded length + /// + [Test] + public void SingleRestore6Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump, TimeSpan.FromHours(3)); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual("val", value.ToString()); + } + + /// + /// Tests that RESTORE command restores payload with 6 bit encoded length without TTL + /// + [Test] + public void SingleRestore6BitWithoutTtl() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey")!; + + db.KeyDelete("mykey"); + + db.KeyRestore("mykey", dump); + + var value = db.StringGet("mykey"); + + ClassicAssert.AreEqual("val", value.ToString()); + } + + /// + /// Tests that DUMP command returns payload with 6 bit encoded length + /// + [Test] + public void SingleDump6Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("mykey", "val"); + + var dump = db.KeyDump("mykey"); + + var expectedValue = new byte[] + { + 0x00, // value type + 0x03, // length of payload + 0x76, 0x61, 0x6C, // 'v', 'a', 'l' + 0x0B, 0x00, // RDB version + }; + + var crc = new byte[] + { + 0xDB, + 0x82, + 0x3C, + 0x30, + 0x38, + 0x78, + 0x5A, + 0x99 + }; + + expectedValue = [.. expectedValue, .. crc]; + + ClassicAssert.AreEqual(expectedValue, dump); + } + + /// + /// Tests that DUMP command returns payload with 14 bit encoded length + /// + [Test] + public void SingleDump14Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383 - 1; i++) + valueBuilder.Append('a'); + + var value = valueBuilder.ToString(); + + db.StringSet("mykey", value); + + var dump = db.KeyDump("mykey"); + + var expectedValue = new byte[] + { + 0x00, // value type + 0x7F, 0xFE, // length of payload + }; + + expectedValue = [.. expectedValue, .. Encoding.UTF8.GetBytes(value)]; + + var rdbVersion = new byte[] + { + 0x0B, 0x00, // RDB version + }; + + expectedValue = [.. expectedValue, .. rdbVersion]; + + var crc = new byte[] + { + 0x7C, + 0x09, + 0x2D, + 0x16, + 0x73, + 0xAE, + 0x7C, + 0xCF + }; + + expectedValue = [.. expectedValue, .. crc]; + + ClassicAssert.AreEqual(expectedValue, dump); + } + + /// + /// Tests that DUMP command returns payload with 32 bit encoded length + /// + [Test] + public void SingleDump32Bit() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var valueBuilder = new StringBuilder(); + + for (var i = 0; i < 16_383 + 1; i++) + valueBuilder.Append('a'); + + var value = valueBuilder.ToString(); + + db.StringSet("mykey", value); + + var dump = db.KeyDump("mykey"); + + var expectedValue = new byte[] + { + 0x00, // value type + 0x80, 0x00, 0x00, 0x40, 0x00, // length of payload + }; + + expectedValue = [.. expectedValue, .. Encoding.UTF8.GetBytes(value)]; + + var rdbVersion = new byte[] + { + 0x0B, 0x00, // RDB version + }; + + expectedValue = [.. expectedValue, .. rdbVersion]; + + var crc = new byte[] + { + 0x7F, + 0x73, + 0x7E, + 0xA9, + 0x87, + 0xD9, + 0x90, + 0x14 + }; + + expectedValue = [.. expectedValue, .. crc]; + + ClassicAssert.AreEqual(expectedValue, dump); + } + + /// + /// Tests DUMP on non string type which is currently not supported + /// + [Test] + public void TryDumpKeyNonString() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.SetAdd("mykey", "val1"); + db.SetAdd("mykey", "val2"); + + var value = db.KeyDump("mykey"); + + ClassicAssert.AreEqual(null, value); + } + + /// + /// Try DUMP key that does not exist + /// + [Test] + public void TryDumpKeyThatDoesNotExist() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var value = db.KeyDump("mykey"); + + ClassicAssert.AreEqual(null, value); + } + [Test] public void SingleSetGet() { diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index 68a653a5fc..f913114e8d 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -125,13 +125,13 @@ Note that this list is subject to change as we continue to expand our API comman | **FUNCTIONS** | FCALL | ➖ | | | | FCALL_RO | ➖ | | | | DELETE | ➖ | -| | DUMP | ➖ | +| | DUMP | ➖ | | | | FLUSH | ➖ | | | HELP | ➖ | | | KILL | ➖ | | | LIST | ➖ | | | LOAD | ➖ | -| | RESTORE | ➖ | +| | RESTORE | ➖ | | | | STATS | ➖ | | **GENERIC** | [PERSIST](generic-commands.md#persist) | ➕ | | | | [PEXPIRE](generic-commands.md#pexpire) | ➕ | | @@ -141,7 +141,7 @@ Note that this list is subject to change as we continue to expand our API comman | | RANDOMKEY | ➖ | | | | [RENAME](generic-commands.md#rename) | ➕ | | | | [RENAMENX](generic-commands.md#renamenx) | ➕ | | -| | RESTORE | ➖ | | +| | [RESTORE](generic-commands.md#restore) | ➕ | | | [SCAN](generic-commands.md#scan) | ➕ | | | | SORT | ➖ | | | | SORT_RO | ➖ | | @@ -193,7 +193,7 @@ Note that this list is subject to change as we continue to expand our API comman | | PFSELFTEST | ➖ | Internal command | | **KEYS** | COPY | ➖ | | | | [DEL](generic-commands.md#del) | ➕ | | -| | DUMP | ➖ | | +| | [DUMP](generic-commands.md#dump) | ➕ | | | [EXISTS](generic-commands.md#exists) | ➕ | | | | [EXPIRE](generic-commands.md#expire) | ➕ | | | | [EXPIREAT](generic-commands.md#expireat) | ➕ | | diff --git a/website/docs/commands/generic-commands.md b/website/docs/commands/generic-commands.md index 27c8ae6946..8f6a34fe16 100644 --- a/website/docs/commands/generic-commands.md +++ b/website/docs/commands/generic-commands.md @@ -481,5 +481,51 @@ Integer reply: the number of keys that were unlinked. --- +### DUMP + +> [!IMPORTANT] +> DUMP currently only supports string types without lzf compression + +#### Syntax + +```bash +DUMP mykey +``` + +Serialize the value stored at key in a Redis-specific format and return it to the user. The returned value can be synthesized back into a Redis key using the [RESTORE](#restore) command. + +#### Resp Reply + +String reply: The serialization format is opaque and non-standard, however it has a few semantic characteristics: + +- It contains a 64-bit checksum that is used to make sure errors will be detected. The [RESTORE](#restore) command makes sure to check the checksum before synthesizing a key using the serialized value. +- Values are encoded in the same format used by RDB. +- An RDB version is encoded inside the serialized value, so that different Redis versions with incompatible RDB formats will refuse to process the serialized value. + +--- + +### RESTORE + +> [!IMPORTANT] +> RESTORE currently only supports string types without lzf compression + +#### Syntax + +```bash +restore mykey 0 "\x00\x0evallllllllllll\x0b\x00|\xeb\xe2|\xd2.\xfa7" +``` + +Create a key associated with a value that is obtained by deserializing the provided serialized value (obtained via [DUMP](#dump)). + +If ttl is 0 the key is created without any expire, otherwise the specified expire time (in milliseconds) is set. + +#### Resp Reply + +Simple string reply: OK. + +--- + + +