Skip to content

Commit

Permalink
Supported RESP Commands Info (#287)
Browse files Browse the repository at this point in the history
* wip

* WIP

* fixes

* wip

* wip

* wip

* Added command info for custom commands

* Added command count

* Moved StreamProvider to common & added RespCommandsInfoProvider

* Error handling, logging + bugfix

* Added some tests

* bugfixes

* Added RespSerializableBase

* wip

* wip

* Adding logic to CommandInfoUpdater tool

* Finishing CommandInfoUpdater app

* Moving LightClientRequest to common + moving CommandInfoUpdater under playground

* running dotnet format

* Adding some comments

* some cleanup

* Added comments to CommandInfoUpdater

* dotnet format

* Small bugfixes, added more supported commands

* Some cleanup & small changes

* Addressing comments + fixing some namings

* Added test to verify coverage of command info, added some more missing commands info

* Added WATCHOS, WATCHMS as internal commands

* dotnet format

* removing redis references from CommandInfoUpdater, adding "force" option

* Updating RespCommandsInfo.json

* Addressing comments

* formatting

* Update playground/CommandInfoUpdater/SupportedCommand.cs

Co-authored-by: Lukas Maas <[email protected]>

* Update libs/server/Resp/RespCommandInfoParser.cs

Co-authored-by: Lukas Maas <[email protected]>

* Clean up singleton comments in RespCommandInfoParser.cs

* Fixing naming + checking output from calls to RespReadUtils

* Some small fixes

* Update test/Garnet.test/RespCommandTests.cs

Co-authored-by: Lukas Maas <[email protected]>

* Update playground/CommandInfoUpdater/CommandInfoUpdater.cs

Co-authored-by: Lukas Maas <[email protected]>

* Update playground/CommandInfoUpdater/CommandInfoUpdater.cs

Co-authored-by: Lukas Maas <[email protected]>

* Update playground/CommandInfoUpdater/CommandInfoUpdater.cs

Co-authored-by: Lukas Maas <[email protected]>

* Disabling Nullable in CommandInfoUpdater.csproj + Fixing thread-safety issue in RespCommandsInfo initialization

* Adding documentation

* Docs fix

---------

Co-authored-by: Lukas Maas <[email protected]>
Co-authored-by: Badrish Chandramouli <[email protected]>
  • Loading branch information
3 people authored May 9, 2024
1 parent ed4b9ab commit 7f1184d
Show file tree
Hide file tree
Showing 52 changed files with 8,435 additions and 433 deletions.
11 changes: 11 additions & 0 deletions Garnet.sln
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Embedded.perftest", "playgr
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BDN.benchmark", "benchmark\BDN.benchmark\BDN.benchmark.csproj", "{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommandInfoUpdater", "playground\CommandInfoUpdater\CommandInfoUpdater.csproj", "{9BE474A2-1547-43AC-B4F2-FB48A01FA995}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -259,6 +261,14 @@ Global
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}.Release|Any CPU.Build.0 = Release|Any CPU
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}.Release|x64.ActiveCfg = Release|Any CPU
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}.Release|x64.Build.0 = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|x64.ActiveCfg = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|x64.Build.0 = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|Any CPU.Build.0 = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|x64.ActiveCfg = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -286,6 +296,7 @@ Global
{8941A05C-099B-45AC-A7BF-F0E226BD59A8} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
{5BEDAC1F-6458-4EBA-8174-EC06B07F2132} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD} = {346A5A53-51E4-4A75-B7E6-491D950382CE}
{9BE474A2-1547-43AC-B4F2-FB48A01FA995} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {2C02C405-4798-41CA-AF98-61EDFEF6772E}
Expand Down
7 changes: 6 additions & 1 deletion benchmark/Resp.benchmark/BenchUtils.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System.Collections.Generic;
using System.Diagnostics;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
Expand Down Expand Up @@ -31,10 +32,14 @@ public static SslClientAuthenticationOptions GetTlsOptions(string tlsHost, strin

public static ConfigurationOptions GetConfig(string address, int port = default, bool allowAdmin = false, bool useTLS = false, string tlsHost = null)
{
var commands = RespCommandsInfo.TryGetRespCommandNames(out var cmds)
? new HashSet<string>(cmds)
: new HashSet<string>();

var configOptions = new ConfigurationOptions
{
EndPoints = { { address, port }, },
CommandMap = CommandMap.Create(RespInfo.GetCommands()),
CommandMap = CommandMap.Create(commands),
ConnectTimeout = 100_000,
SyncTimeout = 100_000,
AllowAdmin = allowAdmin,
Expand Down
7 changes: 2 additions & 5 deletions benchmark/Resp.benchmark/Resp.benchmark.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,6 @@
<ServerGarbageCollection>true</ServerGarbageCollection>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\..\libs\server\Resp\RespInfo.cs" Link="RespInfo.cs" />
</ItemGroup>

<ItemGroup>
<None Include="..\..\test\testcerts\testcert.pfx" Link="testcert.pfx">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
Expand All @@ -25,7 +21,8 @@

<ItemGroup>
<ProjectReference Include="..\..\libs\client\Garnet.client.csproj" />
<ProjectReference Include="..\..\metrics\HdrHistogram\HdrHistogram.csproj" />
<ProjectReference Include="..\..\libs\server\Garnet.server.csproj" />
<ProjectReference Include="..\..\metrics\HdrHistogram\HdrHistogram.csproj" />
</ItemGroup>

</Project>
113 changes: 113 additions & 0 deletions libs/common/EnumUtils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;

namespace Garnet.common
{
/// <summary>
/// Utilities for enums
/// </summary>
public static class EnumUtils
{
private static readonly Dictionary<Type, IDictionary<string, string>> EnumNameToDescriptionCache = new();
private static readonly Dictionary<Type, IDictionary<string, List<string>>> EnumDescriptionToNameCache = new();

/// <summary>
/// Gets a mapping between an enum's string value to its description, for each of the enum's values
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <returns>A dictionary mapping between the enum's string value to its description</returns>
public static IDictionary<string, string> GetEnumNameToDescription<T>() where T : Enum
{
// Check if mapping is already in the cache. If not, add it to the cache.
if (!EnumNameToDescriptionCache.ContainsKey(typeof(T)))
AddTypeToCache<T>();

return EnumNameToDescriptionCache[typeof(T)];
}

/// <summary>
/// If enum does not have the 'Flags' attribute, gets an array of size 1 with the description of the enum's value.
/// If no description exists, returns the ToString() value of the input value.
/// If enum has the 'Flags' attribute, gets an array with all the descriptions of the flags which are turned on in the input value.
/// If no description exists, returns the ToString() value of the flag.
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <param name="value">Enum value</param>
/// <returns>Array of descriptions</returns>
public static string[] GetEnumDescriptions<T>(T value) where T : Enum
{
var nameToDesc = GetEnumNameToDescription<T>();
return value.ToString().Split(',').Select(f => nameToDesc.ContainsKey(f.Trim()) ? nameToDesc[f.Trim()] : f).ToArray();
}

/// <summary>
/// Gets an enum's values based on the description attribute
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <param name="strVal">Enum description</param>
/// <param name="vals">Enum values</param>
/// <returns>True if matched more than one value successfully</returns>
public static bool TryParseEnumsFromDescription<T>(string strVal, out IEnumerable<T> vals) where T : struct, Enum
{
vals = new List<T>();

if (!EnumDescriptionToNameCache.ContainsKey(typeof(T)))
AddTypeToCache<T>();

if (!EnumDescriptionToNameCache[typeof(T)].ContainsKey(strVal))
return false;

foreach (var enumName in EnumDescriptionToNameCache[typeof(T)][strVal])
{
if (Enum.TryParse(enumName, out T enumVal))
{
((List<T>)vals).Add(enumVal);
}
}

return ((List<T>)vals).Count > 0;
}

/// <summary>
/// Gets an enum's value based on its description attribute
/// If more than one values match the same description, returns the first one
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <param name="strVal">Enum description</param>
/// <param name="val">Enum value</param>
/// <returns>True if successful</returns>
public static bool TryParseEnumFromDescription<T>(string strVal, out T val) where T : struct, Enum
{
var isSuccessful = TryParseEnumsFromDescription(strVal, out IEnumerable<T> vals);
val = isSuccessful ? vals.First() : default;
return isSuccessful;
}


private static void AddTypeToCache<T>()
{
var valToDesc = new Dictionary<string, string>();
var descToVals = new Dictionary<string, List<string>>();

foreach (var flagFieldInfo in typeof(T).GetFields())
{
var descAttr = (DescriptionAttribute)flagFieldInfo.GetCustomAttributes(typeof(DescriptionAttribute), false).FirstOrDefault();
if (descAttr != null)
{
valToDesc.Add(flagFieldInfo.Name, descAttr.Description);
if (!descToVals.ContainsKey(descAttr.Description))
descToVals.Add(descAttr.Description, new List<string>());
descToVals[descAttr.Description].Add(flagFieldInfo.Name);
}
}

EnumNameToDescriptionCache.Add(typeof(T), valToDesc);
EnumDescriptionToNameCache.Add(typeof(T), descToVals);
}
}
}
5 changes: 5 additions & 0 deletions libs/common/Garnet.common.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,9 @@
<PackageReference Include="Microsoft.SourceLink.GitHub" PrivateAssets="All" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\storage\Tsavorite\cs\src\core\Tsavorite.core.csproj" />
<ProjectReference Include="..\storage\Tsavorite\cs\src\devices\AzureStorageDevice\Tsavorite.devices.AzureStorageDevice.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@
using System;
using System.Net.Security;
using System.Text;
using Garnet.common;
using Garnet.networking;

namespace Garnet.test
namespace Garnet.common
{
public unsafe class LightClientRequest : IDisposable
{
Expand Down
32 changes: 20 additions & 12 deletions libs/host/StreamProvider.cs → libs/common/StreamProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
using Tsavorite.core;
using Tsavorite.devices;

namespace Garnet
namespace Garnet.common
{
internal enum FileLocationType
public enum FileLocationType
{
Local,
AzureStorage,
Expand All @@ -22,7 +22,7 @@ internal enum FileLocationType
/// <summary>
/// Interface for reading / writing into local / remote files
/// </summary>
internal interface IStreamProvider
public interface IStreamProvider
{
/// <summary>
/// Read data from file specified in path
Expand Down Expand Up @@ -112,15 +112,16 @@ private static void IOCallback(uint errorCode, uint numBytes, object context)
/// <summary>
/// Provides a StreamProvider instance
/// </summary>
internal class StreamProviderFactory
public class StreamProviderFactory
{
/// <summary>
/// Get a StreamProvider instance
/// </summary>
/// <param name="locationType">Type of location of files the stream provider reads from / writes to</param>
/// <param name="connectionString">Connection string to Azure Storage, if applicable</param>
/// <param name="resourceAssembly">Assembly from which to load the embedded resource, if applicable</param>
/// <returns>StreamProvider instance</returns>
internal static IStreamProvider GetStreamProvider(FileLocationType locationType, string connectionString = null)
public static IStreamProvider GetStreamProvider(FileLocationType locationType, string connectionString = null, Assembly resourceAssembly = null)
{
switch (locationType)
{
Expand All @@ -131,7 +132,7 @@ internal static IStreamProvider GetStreamProvider(FileLocationType locationType,
case FileLocationType.Local:
return new LocalFileStreamProvider();
case FileLocationType.EmbeddedResource:
return new EmbeddedResourceStreamProvider();
return new EmbeddedResourceStreamProvider(resourceAssembly);
default:
throw new NotImplementedException();
}
Expand Down Expand Up @@ -199,22 +200,29 @@ protected override long GetBytesToWrite(byte[] bytes, IDevice device)
/// </summary>
internal class EmbeddedResourceStreamProvider : IStreamProvider
{
private readonly Assembly assembly;

public EmbeddedResourceStreamProvider(Assembly assembly)
{
this.assembly = assembly;
}

public Stream Read(string path)
{
var assembly = Assembly.GetExecutingAssembly();
var resourceName = assembly.GetManifestResourceNames().FirstOrDefault(rn => rn.EndsWith(path));
var resourceName = assembly.GetManifestResourceNames()
.FirstOrDefault(rn => rn.EndsWith($".{path}"));
if (resourceName == null) return null;

return Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName);
return assembly.GetManifestResourceStream(resourceName);
}

public void Write(string path, byte[] data)
{
var assembly = Assembly.GetExecutingAssembly();
var resourceName = assembly.GetManifestResourceNames().FirstOrDefault(rn => rn.EndsWith(path));
var resourceName = assembly.GetManifestResourceNames()
.FirstOrDefault(rn => rn.EndsWith($".{path}"));
if (resourceName == null) return;

using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName);
using var stream = assembly.GetManifestResourceStream(resourceName);
if (stream != null)
stream.Write(data, 0, data.Length);
}
Expand Down
1 change: 1 addition & 0 deletions libs/host/Configuration/ConfigProviders.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.IO;
using System.Text;
using System.Text.Json;
using Garnet.common;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using ErrorEventArgs = Newtonsoft.Json.Serialization.ErrorEventArgs;
Expand Down
9 changes: 7 additions & 2 deletions libs/host/ServerSettingsManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Text.RegularExpressions;
using CommandLine;
using CommandLine.Text;
using Garnet.common;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;

Expand Down Expand Up @@ -237,7 +238,9 @@ private static Dictionary<string, object> GetArgumentNameToValue(Options options
/// <returns>True if import succeeded</returns>
private static bool TryImportServerOptions(string path, ConfigFileType configFileType, Options options, ILogger logger, FileLocationType fileLocationType, string connString = null)
{
var streamProvider = StreamProviderFactory.GetStreamProvider(fileLocationType, connString);
var assembly = fileLocationType == FileLocationType.EmbeddedResource ? Assembly.GetExecutingAssembly() : null;

var streamProvider = StreamProviderFactory.GetStreamProvider(fileLocationType, connString, assembly);
var configProvider = ConfigProviderFactory.GetConfigProvider(configFileType);

using var stream = streamProvider.Read(path);
Expand Down Expand Up @@ -269,7 +272,9 @@ private static bool TryImportServerOptions(string path, ConfigFileType configFil
/// <returns>True if export succeeded</returns>
private static bool TryExportServerOptions(string path, ConfigFileType configFileType, Options options, ILogger logger, FileLocationType fileLocationType, string connString = null)
{
var streamProvider = StreamProviderFactory.GetStreamProvider(fileLocationType, connString);
var assembly = fileLocationType == FileLocationType.EmbeddedResource ? Assembly.GetExecutingAssembly() : null;

var streamProvider = StreamProviderFactory.GetStreamProvider(fileLocationType, connString, assembly);
var configProvider = ConfigProviderFactory.GetConfigProvider(configFileType);

var exportSucceeded = configProvider.TryExportOptions(path, streamProvider, options, logger);
Expand Down
Loading

0 comments on commit 7f1184d

Please sign in to comment.