Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding ObjectStoreOutputFlags to GarnetObjectStoreOutput #923

Merged
merged 16 commits into from
Jan 23, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Adding SizeChange to GOSO
  • Loading branch information
TalZaccai committed Jan 21, 2025
commit 20d7749f40625cb46a7699036d424ff4ba55a1c6
2 changes: 2 additions & 0 deletions libs/server/Custom/CustomObjectBase.cs
Original file line number Diff line number Diff line change
@@ -70,6 +70,8 @@ public sealed override void DoSerialize(BinaryWriter writer)
/// <inheritdoc />
public sealed override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output)
{
output.SizeChange = 0;

switch (input.header.cmd)
{
// Scan Command
6 changes: 6 additions & 0 deletions libs/server/Objects/Hash/HashObject.cs
Original file line number Diff line number Diff line change
@@ -110,16 +110,20 @@
/// <inheritdoc />
public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output)
{
output.SizeChange = 0;

fixed (byte* outputSpan = output.SpanByteAndMemory.SpanByte.AsSpan())
{
if (input.header.type != GarnetObjectType.Hash)
{
//Indicates when there is an incorrect type
output.OutputFlags |= ObjectStoreOutputFlags.WrongType;
output.SpanByteAndMemory.Length = 0;
output.SizeChange = 0;
return true;
}

var previousSize = this.Size;
switch (input.header.HashOp)
{
case HashOperation.HSET:
@@ -183,6 +187,8 @@
default:
throw new GarnetException($"Unsupported operation {input.header.HashOp} in HashObject.Operate");
}

output.SizeChange = this.Size - previousSize;
}

if (hash.Count == 0)
@@ -195,13 +201,13 @@
{
var size = Utility.RoundUp(key.Length, IntPtr.Size) + Utility.RoundUp(value.Length, IntPtr.Size)
+ (2 * MemoryUtils.ByteArrayOverhead) + MemoryUtils.DictionaryEntryOverhead;
this.Size += add ? size : -size;

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 204 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'
Debug.Assert(this.Size >= MemoryUtils.DictionaryOverhead);
}

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 207 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'
/// <inheritdoc />
public override unsafe void Scan(long start, out List<byte[]> items, out long cursor, int count = 10, byte* pattern = default, int patternLength = 0, bool isNoValue = false)
{

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test.cluster)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'

Check failure on line 210 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test)

Argument 2: cannot convert from 'ref Garnet.server.GarnetObjectStoreOutput' to 'ref Tsavorite.core.SpanByteAndMemory'
cursor = start;
items = new List<byte[]>();

@@ -222,7 +228,7 @@
continue;
}

if (patternLength == 0)

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test.cluster)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test.cluster)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Release, Garnet.test)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (ubuntu-latest, net8.0, Debug, Garnet.test)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test.cluster)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Debug, Garnet.test)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test.cluster)

The name '_output' does not exist in the current context

Check failure on line 231 in libs/server/Objects/Hash/HashObject.cs

GitHub Actions / Garnet (windows-latest, net8.0, Release, Garnet.test)

The name '_output' does not exist in the current context
{
items.Add(item.Key);
if (!isNoValue)
3 changes: 3 additions & 0 deletions libs/server/Objects/List/ListObject.cs
Original file line number Diff line number Diff line change
@@ -137,6 +137,7 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore
// Indicates an incorrect type of key
output.OutputFlags |= ObjectStoreOutputFlags.WrongType;
output.SpanByteAndMemory.Length = 0;
output.SizeChange = 0;
return true;
}

@@ -185,6 +186,8 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore
default:
throw new GarnetException($"Unsupported operation {input.header.ListOp} in ListObject.Operate");
}

output.SizeChange = this.Size - previousSize;
}

if (list.Count == 0)
3 changes: 3 additions & 0 deletions libs/server/Objects/Set/SetObject.cs
Original file line number Diff line number Diff line change
@@ -114,6 +114,7 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore
// Indicates an incorrect type of key
output.OutputFlags |= ObjectStoreOutputFlags.WrongType;
output.SpanByteAndMemory.Length = 0;
output.SizeChange = 0;
return true;
}

@@ -160,6 +161,8 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore
default:
throw new GarnetException($"Unsupported operation {input.header.SetOp} in SetObject.Operate");
}

output.SizeChange = this.Size - prevSize;
}

if (set.Count == 0)
3 changes: 3 additions & 0 deletions libs/server/Objects/SortedSet/SortedSetObject.cs
Original file line number Diff line number Diff line change
@@ -222,6 +222,7 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore
// Indicates an incorrect type of key
output.OutputFlags |= ObjectStoreOutputFlags.WrongType;
output.SpanByteAndMemory.Length = 0;
output.SizeChange = 0;
return true;
}

@@ -323,6 +324,8 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore
default:
throw new GarnetException($"Unsupported operation {op} in SortedSetObject.Operate");
}

output.SizeChange = this.Size - prevSize;
}

if (sortedSetDict.Count == 0)
5 changes: 5 additions & 0 deletions libs/server/Objects/Types/GarnetObjectStoreOutput.cs
Original file line number Diff line number Diff line change
@@ -48,6 +48,11 @@ public struct GarnetObjectStoreOutput
/// </summary>
public ObjectStoreOutputFlags OutputFlags;

/// <summary>
/// Object size change
/// </summary>
public long SizeChange;

public void ConvertToHeap()
{
// Does not convert to heap when going pending, because we immediately complete pending operations for object store.
8 changes: 3 additions & 5 deletions libs/server/Storage/Functions/ObjectStore/RMWMethods.cs
Original file line number Diff line number Diff line change
@@ -81,21 +81,19 @@ public void PostInitialUpdater(ref byte[] key, ref ObjectInput input, ref IGarne
/// <inheritdoc />
public bool InPlaceUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo)
{
if (InPlaceUpdaterWorker(ref key, ref input, ref value, ref output, ref rmwInfo, out long sizeChange))
if (InPlaceUpdaterWorker(ref key, ref input, ref value, ref output, ref rmwInfo))
{
if (!rmwInfo.RecordInfo.Modified)
functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash);
if (functionsState.appendOnlyFile != null) WriteLogRMW(ref key, ref input, rmwInfo.Version, rmwInfo.SessionID);
functionsState.objectStoreSizeTracker?.AddTrackedSize(sizeChange);
functionsState.objectStoreSizeTracker?.AddTrackedSize(output.SizeChange);
return true;
}
return false;
}

bool InPlaceUpdaterWorker(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, out long sizeChange)
bool InPlaceUpdaterWorker(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo)
{
sizeChange = 0;

// Expired data
if (value.Expiration > 0 && input.header.CheckExpiry(value.Expiration))
{
Loading