Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more test coverage for Tsavorite primitives #1121

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
+245 −155
Draft

Add more test coverage for Tsavorite primitives #1121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a2a9a81
Remove repeat length check from SpanByte.Equals
PaulusParssinen Mar 21, 2025
7e90426
Revert "Remove repeat length check from SpanByte.Equals"
PaulusParssinen Mar 21, 2025
8b6ce52
Initial unit tests for SpanByteComparer.Equals
PaulusParssinen Mar 21, 2025
3b6f6f3
Use Utf8 instead of non-portable MemoryMarshal.Cast in RandomReadCach…
PaulusParssinen Mar 21, 2025
066c25c
Use UTF-8 instead UTF-16 MM.Cast in SpanByteLogScanTests
PaulusParssinen Mar 21, 2025
22cbea4
Use UTF-8 instead UTF-16 MM.Cast in SpanByteTests
PaulusParssinen Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 55 additions & 55 deletions libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using NUnit.Framework;
using NUnit.Framework.Legacy;
Expand All @@ -27,12 +26,11 @@ public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref

public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo)
{
var keyString = new string(MemoryMarshal.Cast<byte, char>(key.AsReadOnlySpan()));
var inputString = new string(MemoryMarshal.Cast<byte, char>(input.AsReadOnlySpan()));
var valueString = new string(MemoryMarshal.Cast<byte, char>(value.AsReadOnlySpan()));
var actualValue = long.Parse(valueString);
ClassicAssert.AreEqual(long.Parse(keyString) * 2, actualValue);
ClassicAssert.AreEqual(long.Parse(inputString), actualValue);
var parsedKey = long.Parse(key.AsReadOnlySpan());
var parsedInput = long.Parse(input.AsReadOnlySpan());
var actualValue = long.Parse(value.AsReadOnlySpan());
ClassicAssert.AreEqual(parsedKey * 2, actualValue);
ClassicAssert.AreEqual(parsedInput, actualValue);

value.CopyTo(ref dst, MemoryPool<byte>.Shared);
return true;
Expand All @@ -41,13 +39,12 @@ public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref Span
public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, Empty context, Status status, RecordMetadata recordMetadata)
{
ClassicAssert.IsTrue(status.Found);
var keyString = new string(MemoryMarshal.Cast<byte, char>(key.AsReadOnlySpan()));
var inputString = new string(MemoryMarshal.Cast<byte, char>(input.AsReadOnlySpan()));
var outputString = new string(MemoryMarshal.Cast<byte, char>(output.AsReadOnlySpan()));
var actualValue = long.Parse(outputString);
ClassicAssert.AreEqual(long.Parse(keyString) * 2, actualValue);
ClassicAssert.AreEqual(long.Parse(inputString), actualValue);
ClassicAssert.IsNotNull(output.Memory, $"key {keyString}, in ReadCC");
var parsedKey = long.Parse(key.AsReadOnlySpan());
var parsedInput = long.Parse(input.AsReadOnlySpan());
var actualValue = long.Parse(output.AsReadOnlySpan());
ClassicAssert.AreEqual(parsedKey * 2, actualValue);
ClassicAssert.AreEqual(parsedInput, actualValue);
ClassicAssert.IsNotNull(output.Memory, $"key {parsedKey}, in ReadCC");
}
}

Expand Down Expand Up @@ -111,7 +108,7 @@ public void TearDown()
[Category(ReadCacheTestCategory)]
[Category(StressTestCategory)]
//[Repeat(300)]
public unsafe void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values] KeyContentionMode keyContentionMode, [Values] ReadCacheMode readCacheMode)
public void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values] KeyContentionMode keyContentionMode, [Values] ReadCacheMode readCacheMode)
{
if (numThreads == 1 && keyContentionMode == KeyContentionMode.Contention)
Assert.Ignore("Skipped because 1 thread cannot have contention");
Expand All @@ -124,30 +121,30 @@ public unsafe void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values

void LocalRead(BasicContext<SpanByte, SpanByte, SpanByte, SpanByteAndMemory, Empty, Functions, SpanByteStoreFunctions, SpanByteAllocator<SpanByteStoreFunctions>> sessionContext, int i, ref int numPending, bool isLast)
{
var keyString = $"{i}";
var inputString = $"{i * 2}";
var key = MemoryMarshal.Cast<char, byte>(keyString.AsSpan());
var input = MemoryMarshal.Cast<char, byte>(inputString.AsSpan());
Span<byte> keySpan = stackalloc byte[64];
Span<byte> inputSpan = stackalloc byte[64];

fixed (byte* kptr = key, iptr = input)
{
var sbKey = SpanByte.FromPinnedSpan(key);
var sbInput = SpanByte.FromPinnedSpan(input);
SpanByteAndMemory output = default;
var key = i;
var input = i * 2;

var status = sessionContext.Read(ref sbKey, ref sbInput, ref output);
_ = key.TryFormat(keySpan, out var keyBytesWritten);
_ = input.TryFormat(inputSpan, out var inputBytesWritten);

if (status.Found)
{
var outputString = new string(MemoryMarshal.Cast<byte, char>(output.AsReadOnlySpan()));
ClassicAssert.AreEqual(i * 2, long.Parse(outputString));
output.Memory.Dispose();
}
else
{
ClassicAssert.IsTrue(status.IsPending, $"was not Pending: {keyString}; status {status}");
++numPending;
}
var sbKey = SpanByte.FromPinnedSpan(keySpan.Slice(0, keyBytesWritten));
var sbInput = SpanByte.FromPinnedSpan(inputSpan.Slice(0, inputBytesWritten));
SpanByteAndMemory output = default;

var status = sessionContext.Read(ref sbKey, ref sbInput, ref output);

if (status.Found)
{
ClassicAssert.AreEqual(i * 2, long.Parse(output.AsReadOnlySpan()));
output.Memory.Dispose();
}
else
{
ClassicAssert.IsTrue(status.IsPending, $"was not Pending: {key}; status {status}");
++numPending;
}

if (numPending > 0 && ((numPending % PendingMod) == 0 || isLast))
Expand All @@ -157,16 +154,15 @@ void LocalRead(BasicContext<SpanByte, SpanByte, SpanByte, SpanByteAndMemory, Emp
{
while (completedOutputs.Next())
{
var status = completedOutputs.Current.Status;
var output = completedOutputs.Current.Output;
var completedStatus = completedOutputs.Current.Status;
var completedOutput = completedOutputs.Current.Output;
// Note: do NOT overwrite 'key' here
long keyLong = long.Parse(new string(MemoryMarshal.Cast<byte, char>(completedOutputs.Current.Key.AsReadOnlySpan())));
long keyLong = long.Parse(completedOutputs.Current.Key.AsReadOnlySpan());

ClassicAssert.IsTrue(status.Found, $"key {keyLong}, {status}, wasPending {true}, pt 1");
ClassicAssert.IsNotNull(output.Memory, $"key {keyLong}, wasPending {true}, pt 2");
var outputString = new string(MemoryMarshal.Cast<byte, char>(output.AsReadOnlySpan()));
ClassicAssert.AreEqual(keyLong * 2, long.Parse(outputString), $"key {keyLong}, wasPending {true}, pt 3");
output.Memory.Dispose();
ClassicAssert.IsTrue(completedStatus.Found, $"key {keyLong}, {completedStatus}, wasPending {true}, pt 1");
ClassicAssert.IsNotNull(completedOutput.Memory, $"key {keyLong}, wasPending {true}, pt 2");
ClassicAssert.AreEqual(keyLong * 2, long.Parse(completedOutput.AsReadOnlySpan()), $"key {keyLong}, wasPending {true}, pt 3");
completedOutput.Memory.Dispose();
}
}
}
Expand Down Expand Up @@ -194,19 +190,23 @@ void LocalRun(int startKey, int endKey)
{ // Write the values first (single-threaded, all keys)
var session = store.NewSession<SpanByte, SpanByteAndMemory, Empty, Functions>(new Functions());
var bContext = session.BasicContext;

Span<byte> keySpan = stackalloc byte[64];
Span<byte> valueSpan = stackalloc byte[64];

for (int i = 0; i < MaxKeys; i++)
{
var keyString = $"{i}";
var valueString = $"{i * 2}";
var key = MemoryMarshal.Cast<char, byte>(keyString.AsSpan());
var value = MemoryMarshal.Cast<char, byte>(valueString.AsSpan());
fixed (byte* k = key, v = value)
{
var sbKey = SpanByte.FromPinnedSpan(key);
var sbValue = SpanByte.FromPinnedSpan(value);
var status = bContext.Upsert(sbKey, sbValue);
ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString());
}
var key = i;
var value = i * 2;

_ = key.TryFormat(keySpan, out var keyBytesWritten);
_ = value.TryFormat(valueSpan, out var valueBytesWritten);

var sbKey = SpanByte.FromPinnedSpan(keySpan.Slice(0, keyBytesWritten));
var sbValue = SpanByte.FromPinnedSpan(valueSpan.Slice(0, valueBytesWritten));

var status = bContext.Upsert(sbKey, sbValue);
ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString());
}
}

Expand Down
83 changes: 83 additions & 0 deletions libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System;
using NUnit.Framework;
using NUnit.Framework.Legacy;
using Tsavorite.core;

namespace Tsavorite.test.spanbyte
{
[TestFixture]
[Category("TsavoriteKV")]
[Category("Smoke")]
internal class SpanByteComparerTests
{
[Test]
public void Equals_ReturnsTrue_ForIdenticalSpanBytes()
{
var left = SpanByte.FromPinnedSpan([1, 2, 3, 4]);
var right = SpanByte.FromPinnedSpan([1, 2, 3, 4]);

ClassicAssert.IsTrue(SpanByteComparer.Instance.Equals(ref left, ref right));
}

[Test]
public void Equals_ReturnsFalse_ForDifferentPayloads()
{
var left = SpanByte.FromPinnedSpan([1, 2, 3, 4]);
var right = SpanByte.FromPinnedSpan([1, 2, 3, 5]);

ClassicAssert.IsFalse(SpanByteComparer.Instance.Equals(ref left, ref right));
}

[Test]
public void Equals_ReturnsFalse_ForDifferentMetadataSizes()
{
Span<byte> data = [1, 2, 3, 4, 5, 6, 7, 8];
var left = SpanByte.FromPinnedSpan(data);
var right = SpanByte.FromPinnedSpan(data);

left.ExtraMetadata = 1234; // sets MetadataSize to 8

ClassicAssert.AreEqual(8, left.MetadataSize);
ClassicAssert.AreEqual(0, right.MetadataSize);
ClassicAssert.IsFalse(SpanByteComparer.Instance.Equals(ref left, ref right));
}

[Test]
public void Equals_ReturnsTrue_SerializedAndUnserializedWithSamePayload()
{
Span<byte> payload = [1, 2, 3, 4];
var unserialized = SpanByte.FromPinnedSpan(payload);

Span<byte> serializedSpan = stackalloc byte[sizeof(int) + payload.Length];
payload.CopyTo(serializedSpan.Slice(sizeof(int)));

var serialized = SpanByte.Reinterpret(serializedSpan);

ClassicAssert.AreNotEqual(unserialized.Serialized, serialized.Serialized);
ClassicAssert.True(SpanByteComparer.Instance.Equals(ref serialized, ref unserialized));
}

[Test]
public void Equals_ReturnsTrue_ForTwoEmpty()
{
var left = new SpanByte();
var right = new SpanByte();

ClassicAssert.True(SpanByteComparer.Instance.Equals(ref left, ref right));
}

[Test]
public void Equals_ReturnsTrue_ForEmptyAndInvalid()
{
var invalid = new SpanByte();
invalid.Invalid = true;

var valid = new SpanByte();

ClassicAssert.IsTrue(SpanByteComparer.Instance.Equals(ref invalid, ref valid));
}
}
}
Loading
Loading