From a2a9a813d285f29ba93d27b6c8fe0e3f1c5f8554 Mon Sep 17 00:00:00 2001 From: PaulusParssinen Date: Fri, 21 Mar 2025 16:57:12 +0200 Subject: [PATCH 1/6] Remove repeat length check from SpanByte.Equals --- libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs index 6ab7bbf1d1b..f6952ef408e 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs @@ -47,8 +47,7 @@ public static unsafe long StaticGetHashCode64(ref SpanByte spanByte) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe bool StaticEquals(ref SpanByte k1, ref SpanByte k2) { - return k1.AsReadOnlySpanWithMetadata().SequenceEqual(k2.AsReadOnlySpanWithMetadata()) - && (k1.MetadataSize == k2.MetadataSize); + return k1.AsReadOnlySpanWithMetadata().SequenceEqual(k2.AsReadOnlySpanWithMetadata()); } } } \ No newline at end of file From 7e904268e63990ecb447662161433cc4079961df Mon Sep 17 00:00:00 2001 From: PaulusParssinen Date: Fri, 21 Mar 2025 20:36:23 +0200 Subject: [PATCH 2/6] Revert "Remove repeat length check from SpanByte.Equals" This reverts commit a2a9a813d285f29ba93d27b6c8fe0e3f1c5f8554. --- libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs index f6952ef408e..6ab7bbf1d1b 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs @@ -47,7 +47,8 @@ public static unsafe long StaticGetHashCode64(ref SpanByte spanByte) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe bool StaticEquals(ref SpanByte k1, ref SpanByte k2) { - return k1.AsReadOnlySpanWithMetadata().SequenceEqual(k2.AsReadOnlySpanWithMetadata()); + return k1.AsReadOnlySpanWithMetadata().SequenceEqual(k2.AsReadOnlySpanWithMetadata()) + && (k1.MetadataSize == k2.MetadataSize); } } } \ No newline at end of file From 8b6ce52a8e5c31c56b685ccecadebf845dcba8bb Mon Sep 17 00:00:00 2001 From: PaulusParssinen Date: Fri, 21 Mar 2025 23:31:09 +0200 Subject: [PATCH 3/6] Initial unit tests for SpanByteComparer.Equals --- .../cs/test/SpanByteComparerTests.cs | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs diff --git a/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs b/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs new file mode 100644 index 00000000000..449809191e9 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; + +namespace Tsavorite.test.spanbyte +{ + [TestFixture] + [Category("TsavoriteKV")] + [Category("Smoke")] + internal class SpanByteComparerTests + { + [Test] + public void Equals_ReturnsTrue_ForIdenticalSpanBytes() + { + var left = SpanByte.FromPinnedSpan([1, 2, 3, 4]); + var right = SpanByte.FromPinnedSpan([1, 2, 3, 4]); + + ClassicAssert.IsTrue(SpanByteComparer.Instance.Equals(ref left, ref right)); + } + + [Test] + public void Equals_ReturnsFalse_ForDifferentPayloads() + { + var left = SpanByte.FromPinnedSpan([1, 2, 3, 4]); + var right = SpanByte.FromPinnedSpan([1, 2, 3, 5]); + + ClassicAssert.IsFalse(SpanByteComparer.Instance.Equals(ref left, ref right)); + } + + [Test] + public void Equals_ReturnsFalse_ForDifferentMetadataSizes() + { + Span data = [1, 2, 3, 4, 5, 6, 7, 8]; + var left = SpanByte.FromPinnedSpan(data); + var right = SpanByte.FromPinnedSpan(data); + + left.ExtraMetadata = 1234; // sets MetadataSize to 8 + + ClassicAssert.AreEqual(8, left.MetadataSize); + ClassicAssert.AreEqual(0, right.MetadataSize); + ClassicAssert.IsFalse(SpanByteComparer.Instance.Equals(ref left, ref right)); + } + + [Test] + public void Equals_ReturnsTrue_SerializedAndUnserializedWithSamePayload() + { + Span payload = [1, 2, 3, 4]; + var unserialized = SpanByte.FromPinnedSpan(payload); + + Span serializedSpan = stackalloc byte[sizeof(int) + payload.Length]; + payload.CopyTo(serializedSpan.Slice(sizeof(int))); + + var serialized = SpanByte.Reinterpret(serializedSpan); + + ClassicAssert.AreNotEqual(unserialized.Serialized, serialized.Serialized); + ClassicAssert.True(SpanByteComparer.Instance.Equals(ref serialized, ref unserialized)); + } + + [Test] + public void Equals_ReturnsTrue_ForTwoEmpty() + { + var left = new SpanByte(); + var right = new SpanByte(); + + ClassicAssert.True(SpanByteComparer.Instance.Equals(ref left, ref right)); + } + + [Test] + public void Equals_ReturnsTrue_ForEmptyAndInvalid() + { + var invalid = new SpanByte(); + invalid.Invalid = true; + + var valid = new SpanByte(); + + ClassicAssert.IsTrue(SpanByteComparer.Instance.Equals(ref invalid, ref valid)); + } + } +} From 3b6f6f3ff6b8defbc93598dceb620e6cbf7801aa Mon Sep 17 00:00:00 2001 From: PaulusParssinen Date: Sat, 22 Mar 2025 00:04:24 +0200 Subject: [PATCH 4/6] Use Utf8 instead of non-portable MemoryMarshal.Cast in RandomReadCacheTest --- .../Tsavorite/cs/test/ReproReadCacheTest.cs | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs b/libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs index 280f6b3938f..9342b974874 100644 --- a/libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs +++ b/libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Runtime.InteropServices; using System.Threading.Tasks; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -27,12 +26,11 @@ public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) { - var keyString = new string(MemoryMarshal.Cast(key.AsReadOnlySpan())); - var inputString = new string(MemoryMarshal.Cast(input.AsReadOnlySpan())); - var valueString = new string(MemoryMarshal.Cast(value.AsReadOnlySpan())); - var actualValue = long.Parse(valueString); - ClassicAssert.AreEqual(long.Parse(keyString) * 2, actualValue); - ClassicAssert.AreEqual(long.Parse(inputString), actualValue); + var parsedKey = long.Parse(key.AsReadOnlySpan()); + var parsedInput = long.Parse(input.AsReadOnlySpan()); + var actualValue = long.Parse(value.AsReadOnlySpan()); + ClassicAssert.AreEqual(parsedKey * 2, actualValue); + ClassicAssert.AreEqual(parsedInput, actualValue); value.CopyTo(ref dst, MemoryPool.Shared); return true; @@ -41,13 +39,12 @@ public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref Span public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, Empty context, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); - var keyString = new string(MemoryMarshal.Cast(key.AsReadOnlySpan())); - var inputString = new string(MemoryMarshal.Cast(input.AsReadOnlySpan())); - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - var actualValue = long.Parse(outputString); - ClassicAssert.AreEqual(long.Parse(keyString) * 2, actualValue); - ClassicAssert.AreEqual(long.Parse(inputString), actualValue); - ClassicAssert.IsNotNull(output.Memory, $"key {keyString}, in ReadCC"); + var parsedKey = long.Parse(key.AsReadOnlySpan()); + var parsedInput = long.Parse(input.AsReadOnlySpan()); + var actualValue = long.Parse(output.AsReadOnlySpan()); + ClassicAssert.AreEqual(parsedKey * 2, actualValue); + ClassicAssert.AreEqual(parsedInput, actualValue); + ClassicAssert.IsNotNull(output.Memory, $"key {parsedKey}, in ReadCC"); } } @@ -111,7 +108,7 @@ public void TearDown() [Category(ReadCacheTestCategory)] [Category(StressTestCategory)] //[Repeat(300)] - public unsafe void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values] KeyContentionMode keyContentionMode, [Values] ReadCacheMode readCacheMode) + public void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values] KeyContentionMode keyContentionMode, [Values] ReadCacheMode readCacheMode) { if (numThreads == 1 && keyContentionMode == KeyContentionMode.Contention) Assert.Ignore("Skipped because 1 thread cannot have contention"); @@ -124,30 +121,30 @@ public unsafe void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values void LocalRead(BasicContext> sessionContext, int i, ref int numPending, bool isLast) { - var keyString = $"{i}"; - var inputString = $"{i * 2}"; - var key = MemoryMarshal.Cast(keyString.AsSpan()); - var input = MemoryMarshal.Cast(inputString.AsSpan()); + Span keySpan = stackalloc byte[64]; + Span inputSpan = stackalloc byte[64]; - fixed (byte* kptr = key, iptr = input) - { - var sbKey = SpanByte.FromPinnedSpan(key); - var sbInput = SpanByte.FromPinnedSpan(input); - SpanByteAndMemory output = default; + var key = i; + var input = i * 2; - var status = sessionContext.Read(ref sbKey, ref sbInput, ref output); + _ = key.TryFormat(keySpan, out var keyBytesWritten); + _ = input.TryFormat(inputSpan, out var inputBytesWritten); - if (status.Found) - { - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - ClassicAssert.AreEqual(i * 2, long.Parse(outputString)); - output.Memory.Dispose(); - } - else - { - ClassicAssert.IsTrue(status.IsPending, $"was not Pending: {keyString}; status {status}"); - ++numPending; - } + var sbKey = SpanByte.FromPinnedSpan(keySpan.Slice(0, keyBytesWritten)); + var sbInput = SpanByte.FromPinnedSpan(inputSpan.Slice(0, inputBytesWritten)); + SpanByteAndMemory output = default; + + var status = sessionContext.Read(ref sbKey, ref sbInput, ref output); + + if (status.Found) + { + ClassicAssert.AreEqual(i * 2, long.Parse(output.AsReadOnlySpan())); + output.Memory.Dispose(); + } + else + { + ClassicAssert.IsTrue(status.IsPending, $"was not Pending: {key}; status {status}"); + ++numPending; } if (numPending > 0 && ((numPending % PendingMod) == 0 || isLast)) @@ -157,16 +154,15 @@ void LocalRead(BasicContext(completedOutputs.Current.Key.AsReadOnlySpan()))); + long keyLong = long.Parse(completedOutputs.Current.Key.AsReadOnlySpan()); - ClassicAssert.IsTrue(status.Found, $"key {keyLong}, {status}, wasPending {true}, pt 1"); - ClassicAssert.IsNotNull(output.Memory, $"key {keyLong}, wasPending {true}, pt 2"); - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - ClassicAssert.AreEqual(keyLong * 2, long.Parse(outputString), $"key {keyLong}, wasPending {true}, pt 3"); - output.Memory.Dispose(); + ClassicAssert.IsTrue(completedStatus.Found, $"key {keyLong}, {completedStatus}, wasPending {true}, pt 1"); + ClassicAssert.IsNotNull(completedOutput.Memory, $"key {keyLong}, wasPending {true}, pt 2"); + ClassicAssert.AreEqual(keyLong * 2, long.Parse(completedOutput.AsReadOnlySpan()), $"key {keyLong}, wasPending {true}, pt 3"); + completedOutput.Memory.Dispose(); } } } @@ -194,19 +190,23 @@ void LocalRun(int startKey, int endKey) { // Write the values first (single-threaded, all keys) var session = store.NewSession(new Functions()); var bContext = session.BasicContext; + + Span keySpan = stackalloc byte[64]; + Span valueSpan = stackalloc byte[64]; + for (int i = 0; i < MaxKeys; i++) { - var keyString = $"{i}"; - var valueString = $"{i * 2}"; - var key = MemoryMarshal.Cast(keyString.AsSpan()); - var value = MemoryMarshal.Cast(valueString.AsSpan()); - fixed (byte* k = key, v = value) - { - var sbKey = SpanByte.FromPinnedSpan(key); - var sbValue = SpanByte.FromPinnedSpan(value); - var status = bContext.Upsert(sbKey, sbValue); - ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); - } + var key = i; + var value = i * 2; + + _ = key.TryFormat(keySpan, out var keyBytesWritten); + _ = value.TryFormat(valueSpan, out var valueBytesWritten); + + var sbKey = SpanByte.FromPinnedSpan(keySpan.Slice(0, keyBytesWritten)); + var sbValue = SpanByte.FromPinnedSpan(valueSpan.Slice(0, valueBytesWritten)); + + var status = bContext.Upsert(sbKey, sbValue); + ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } } From 066c25c2844d46bfbe03382f338d8d3f75b7e600 Mon Sep 17 00:00:00 2001 From: PaulusParssinen Date: Sat, 22 Mar 2025 01:10:25 +0200 Subject: [PATCH 5/6] Use UTF-8 instead UTF-16 MM.Cast in SpanByteLogScanTests --- .../Tsavorite/cs/test/SpanByteLogScanTests.cs | 140 ++++++++++-------- 1 file changed, 76 insertions(+), 64 deletions(-) diff --git a/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs b/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs index 788523d7953..f2fd587ff8c 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs +++ b/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs @@ -3,7 +3,7 @@ using System; using System.IO; -using System.Runtime.InteropServices; +using System.Text.Unicode; using System.Threading.Tasks; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -96,7 +96,7 @@ public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref Span [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) + public void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { const long PageSize = 1L << PageSizeBits; @@ -105,17 +105,20 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H Random rng = new(101); + Span keySpan = stackalloc byte[8]; + Span valueSpan = stackalloc byte[128]; + + valueSpan.Fill((byte)'v'); + for (int i = 0; i < TotalRecords; i++) { - var valueFill = new string('x', rng.Next(120)); // Make the record lengths random - var key = MemoryMarshal.Cast($"key_{i}".AsSpan()); - var value = MemoryMarshal.Cast($"v{valueFill}_{i}".AsSpan()); + _ = Utf8.TryWrite(keySpan, $"key_{i:0000}", out _); + _ = Utf8.TryWrite(valueSpan, $"{i:0000}_", out int valueBytesWritten); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) - { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); - } + var keySpanByte = SpanByte.FromPinnedSpan(keySpan); + var valueSpanByte = SpanByte.FromPinnedSpan(valueSpan.Slice(0, rng.Next(valueBytesWritten + 1, valueSpan.Length))); // Make the record lengths random + + _ = bContext.Upsert(keySpanByte, valueSpanByte); } var scanCursorFuncs = new ScanCursorFuncs(store); @@ -149,17 +152,15 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H ClassicAssert.AreEqual(0, cursor, "Expected cursor to be 0, pt 2"); // Add another totalRecords, with keys incremented by totalRecords to remain distinct, and verify we see all keys. - for (int i = 0; i < TotalRecords; i++) + for (int i = TotalRecords; i < TotalRecords * 2; i++) { - var valueFill = new string('x', rng.Next(120)); // Make the record lengths random - var key = MemoryMarshal.Cast($"key_{i + TotalRecords}".AsSpan()); - var value = MemoryMarshal.Cast($"v{valueFill}_{i + TotalRecords}".AsSpan()); + _ = Utf8.TryWrite(keySpan, $"key_{i:0000}", out _); + _ = Utf8.TryWrite(valueSpan, $"{i:0000}_", out int valueBytesWritten); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) - { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); - } + var keySpanByte = SpanByte.FromPinnedSpan(keySpan); + var valueSpanByte = SpanByte.FromPinnedSpan(valueSpan.Slice(0, rng.Next(valueBytesWritten + 1, valueSpan.Length))); // Make the record lengths random + + _ = bContext.Upsert(keySpanByte, valueSpanByte); } scanCursorFuncs.Initialize(verifyKeys); ClassicAssert.IsFalse(session.ScanCursor(ref cursor, long.MaxValue, scanCursorFuncs, long.MaxValue), "Expected scan to finish and return false, pt 2"); @@ -182,12 +183,13 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H ReadOptions readOptions = default; var readStatus = bContext.ReadAtAddress(store.hlogBase.HeadAddress, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(readStatus.Found, $"Could not read at HeadAddress; {readStatus}"); - var keyString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - var keyOrdinal = int.Parse(keyString.Substring(keyString.IndexOf('_') + 1)); + + var outputKeySpan = output.AsReadOnlySpan(); + var outputKeyOrdinal = int.Parse(outputKeySpan.Slice(0, outputKeySpan.IndexOf((byte)'_'))); output.Memory.Dispose(); scanCursorFuncs.Initialize(verifyKeys); - scanCursorFuncs.numRecords = keyOrdinal; + scanCursorFuncs.numRecords = outputKeyOrdinal; cursor = store.Log.HeadAddress + 1; do { @@ -199,24 +201,27 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void SpanByteScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) + public void SpanByteScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; Random rng = new(101); + + Span keySpan = stackalloc byte[8]; + Span valueSpan = stackalloc byte[128]; + + valueSpan.Fill((byte)'v'); for (int i = 0; i < TotalRecords; i++) { - var valueFill = new string('x', rng.Next(120)); // Make the record lengths random - var key = MemoryMarshal.Cast($"key_{i}".AsSpan()); - var value = MemoryMarshal.Cast($"v{valueFill}_{i}".AsSpan()); + _ = Utf8.TryWrite(keySpan, $"key_{i:0000}", out _); + _ = Utf8.TryWrite(valueSpan, $"{i:0000}_", out int valueBytesWritten); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) - { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); - } + var keySpanByte = SpanByte.FromPinnedSpan(keySpan); + var valueSpanByte = SpanByte.FromPinnedSpan(valueSpan.Slice(0, rng.Next(valueBytesWritten + 1, valueSpan.Length))); // Make the record lengths random + + _ = bContext.Upsert(keySpanByte, valueSpanByte); } var scanCursorFuncs = new ScanCursorFuncs(store); @@ -240,24 +245,27 @@ internal enum RCULocation { RCUNone, RCUBefore, RCUAfter }; [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void SpanByteScanCursorWithRCUTest([Values(RCULocation.RCUBefore, RCULocation.RCUAfter)] RCULocation rcuLocation, [Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) + public void SpanByteScanCursorWithRCUTest([Values(RCULocation.RCUBefore, RCULocation.RCUAfter)] RCULocation rcuLocation, [Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; Random rng = new(101); + Span keySpan = stackalloc byte[8]; + Span valueSpan = stackalloc byte[128]; + + valueSpan.Fill((byte)'v'); + for (int i = 0; i < TotalRecords; i++) { - var valueFill = new string('x', rng.Next(120)); // Make the record lengths random - var key = MemoryMarshal.Cast($"key_{i}".AsSpan()); - var value = MemoryMarshal.Cast($"v{valueFill}_{i}".AsSpan()); + _ = Utf8.TryWrite(keySpan, $"key_{i:0000}", out _); + _ = Utf8.TryWrite(valueSpan, $"{i:0000}_", out int valueBytesWritten); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) - { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); - } + var keySpanByte = SpanByte.FromPinnedSpan(keySpan); + var valueSpanByte = SpanByte.FromPinnedSpan(valueSpan.Slice(0, rng.Next(valueBytesWritten + 1, valueSpan.Length))); // Make the record lengths random + + _ = bContext.Upsert(keySpanByte, valueSpanByte); } var scanCursorFuncs = new ScanCursorFuncs(store) @@ -314,7 +322,7 @@ internal void Initialize(bool verifyKeys, Func filter) this.filter = filter; } - unsafe void CheckForRCU() + void CheckForRCU() { if (rcuLocation == RCULocation.RCUBefore && rcuRecord == numRecords + 1 || rcuLocation == RCULocation.RCUAfter && rcuRecord == numRecords - 1) @@ -325,15 +333,19 @@ unsafe void CheckForRCU() using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; - var valueFill = new string('x', 220); // Update the specified key with a longer value that requires RCU. - var key = MemoryMarshal.Cast($"key_{rcuRecord}".AsSpan()); - var value = MemoryMarshal.Cast($"v{valueFill}_{rcuRecord}".AsSpan()); + Span keySpan = stackalloc byte[8]; + Span valueSpan = stackalloc byte[225]; + + _ = Utf8.TryWrite(keySpan, $"key_{rcuRecord:0000}", out int keyBytesWritten); + + valueSpan.Fill((byte)'v'); + _ = Utf8.TryWrite(valueSpan, $"{rcuRecord:0000}_", out int valueBytesWritten); + + var keySpanByte = SpanByte.FromPinnedSpan(keySpan); + var valueSpanByte = SpanByte.FromPinnedSpan(valueSpan); // Update the specified key with a longer value that requires RCU. - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) - { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); - } + _ = bContext.Upsert(keySpanByte, valueSpanByte); + }).Wait(); // If we RCU before Scan arrives at the record, then we won't see it and the values will be off by one (higher). @@ -345,8 +357,8 @@ unsafe void CheckForRCU() public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) { - var keyString = new string(MemoryMarshal.Cast(key.AsReadOnlySpan())); - var kfield1 = int.Parse(keyString.Substring(keyString.IndexOf('_') + 1)); + var keySpan = key.AsReadOnlySpan(); + var kfield1 = int.Parse(keySpan.Slice(keySpan.IndexOf((byte)'_') + 1)); cursorRecordResult = filter(kfield1) ? CursorRecordResult.Accept : CursorRecordResult.Skip; if (cursorRecordResult != CursorRecordResult.Accept) @@ -383,8 +395,7 @@ public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata re [Test] [Category("TsavoriteKV")] [Category("Smoke")] - - public unsafe void SpanByteJumpToBeginAddressTest() + public void SpanByteJumpToBeginAddressTest() { DeleteDirectory(MethodTestDir, wait: true); using var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "test.log"), deleteOnClose: true); @@ -404,6 +415,9 @@ public unsafe void SpanByteJumpToBeginAddressTest() const int numRecords = 200; const int numTailRecords = 10; + + Span keyValueSpan = stackalloc byte[8]; + long shiftBeginAddressTo = 0; int shiftToKey = 0; for (int i = 0; i < numRecords; i++) @@ -414,14 +428,12 @@ public unsafe void SpanByteJumpToBeginAddressTest() shiftToKey = i; } - var key = MemoryMarshal.Cast($"{i}".AsSpan()); - var value = MemoryMarshal.Cast($"{i}".AsSpan()); + _ = i.TryFormat(keyValueSpan, out int keyValueBytesWritten); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) - { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); - } + var keySpanByte = SpanByte.FromPinnedSpan(keyValueSpan.Slice(0, keyValueBytesWritten)); + var valueSpanByte = SpanByte.FromPinnedSpan(keyValueSpan.Slice(0, keyValueBytesWritten)); + + _ = bContext.Upsert(keySpanByte, valueSpanByte); } using var iter = store.Log.Scan(store.Log.HeadAddress, store.Log.TailAddress); @@ -429,8 +441,8 @@ public unsafe void SpanByteJumpToBeginAddressTest() for (int i = 0; i < 100; ++i) { ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); - ClassicAssert.AreEqual(i, int.Parse(MemoryMarshal.Cast(iter.GetKey().AsSpan()))); - ClassicAssert.AreEqual(i, int.Parse(MemoryMarshal.Cast(iter.GetValue().AsSpan()))); + ClassicAssert.AreEqual(i, int.Parse(iter.GetKey().AsReadOnlySpan())); + ClassicAssert.AreEqual(i, int.Parse(iter.GetValue().AsReadOnlySpan())); } store.Log.ShiftBeginAddress(shiftBeginAddressTo); @@ -441,8 +453,8 @@ public unsafe void SpanByteJumpToBeginAddressTest() if (i == 0) ClassicAssert.AreEqual(store.Log.BeginAddress, iter.CurrentAddress); var expectedKey = numRecords - numTailRecords + i; - ClassicAssert.AreEqual(expectedKey, int.Parse(MemoryMarshal.Cast(iter.GetKey().AsSpan()))); - ClassicAssert.AreEqual(expectedKey, int.Parse(MemoryMarshal.Cast(iter.GetValue().AsSpan()))); + ClassicAssert.AreEqual(expectedKey, int.Parse(iter.GetKey().AsReadOnlySpan())); + ClassicAssert.AreEqual(expectedKey, int.Parse(iter.GetValue().AsReadOnlySpan())); } } From 22cbea4087414b7cd3e51125df4883e78bfb2313 Mon Sep 17 00:00:00 2001 From: PaulusParssinen Date: Sat, 22 Mar 2025 01:34:47 +0200 Subject: [PATCH 6/6] Use UTF-8 instead UTF-16 MM.Cast in SpanByteTests --- .../cs/test/SpanByteComparerTests.cs | 2 +- .../Tsavorite/cs/test/SpanByteLogScanTests.cs | 8 +-- .../Tsavorite/cs/test/SpanByteTests.cs | 67 +++++++++---------- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs b/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs index 449809191e9..6fbb5993772 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs +++ b/libs/storage/Tsavorite/cs/test/SpanByteComparerTests.cs @@ -80,4 +80,4 @@ public void Equals_ReturnsTrue_ForEmptyAndInvalid() ClassicAssert.IsTrue(SpanByteComparer.Instance.Equals(ref invalid, ref valid)); } } -} +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs b/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs index f2fd587ff8c..bf50e3a6d87 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs +++ b/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs @@ -183,7 +183,7 @@ public void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred) ReadOptions readOptions = default; var readStatus = bContext.ReadAtAddress(store.hlogBase.HeadAddress, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(readStatus.Found, $"Could not read at HeadAddress; {readStatus}"); - + var outputKeySpan = output.AsReadOnlySpan(); var outputKeyOrdinal = int.Parse(outputKeySpan.Slice(0, outputKeySpan.IndexOf((byte)'_'))); output.Memory.Dispose(); @@ -207,7 +207,7 @@ public void SpanByteScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hu var bContext = session.BasicContext; Random rng = new(101); - + Span keySpan = stackalloc byte[8]; Span valueSpan = stackalloc byte[128]; @@ -337,7 +337,7 @@ void CheckForRCU() Span valueSpan = stackalloc byte[225]; _ = Utf8.TryWrite(keySpan, $"key_{rcuRecord:0000}", out int keyBytesWritten); - + valueSpan.Fill((byte)'v'); _ = Utf8.TryWrite(valueSpan, $"{rcuRecord:0000}_", out int valueBytesWritten); @@ -345,7 +345,7 @@ void CheckForRCU() var valueSpanByte = SpanByte.FromPinnedSpan(valueSpan); // Update the specified key with a longer value that requires RCU. _ = bContext.Upsert(keySpanByte, valueSpanByte); - + }).Wait(); // If we RCU before Scan arrives at the record, then we won't see it and the values will be off by one (higher). diff --git a/libs/storage/Tsavorite/cs/test/SpanByteTests.cs b/libs/storage/Tsavorite/cs/test/SpanByteTests.cs index 2d5ed4fab7d..af646f2eb78 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteTests.cs +++ b/libs/storage/Tsavorite/cs/test/SpanByteTests.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; @@ -20,7 +19,7 @@ internal class SpanByteTests [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void SpanByteTest1() + public void SpanByteTest1() { Span output = stackalloc byte[20]; SpanByte input = default; @@ -42,40 +41,31 @@ public unsafe void SpanByteTest1() using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; - var key1 = MemoryMarshal.Cast("key1".AsSpan()); - var value1 = MemoryMarshal.Cast("value1".AsSpan()); + var key1 = "key1"u8; + var value1 = "value1"u8; var output1 = SpanByteAndMemory.FromPinnedSpan(output); - fixed (byte* key1Ptr = key1) - fixed (byte* value1Ptr = value1) - { - var key1SpanByte = SpanByte.FromPinnedPointer(key1Ptr, key1.Length); - var value1SpanByte = SpanByte.FromPinnedPointer(value1Ptr, value1.Length); + var key1SpanByte = SpanByte.FromPinnedSpan(key1); + var value1SpanByte = SpanByte.FromPinnedSpan(value1); - _ = bContext.Upsert(key1SpanByte, value1SpanByte); - _ = bContext.Read(ref key1SpanByte, ref input, ref output1); - } + _ = bContext.Upsert(key1SpanByte, value1SpanByte); + _ = bContext.Read(ref key1SpanByte, ref input, ref output1); ClassicAssert.IsTrue(output1.IsSpanByte); - ClassicAssert.IsTrue(output1.SpanByte.AsReadOnlySpan().SequenceEqual(value1)); + ClassicAssert.IsTrue(output1.AsReadOnlySpan().SequenceEqual(value1)); - var key2 = MemoryMarshal.Cast("key2".AsSpan()); - var value2 = MemoryMarshal.Cast("value2value2value2".AsSpan()); + var key2 = "key2"u8; + var value2 = "value2value2value2"u8; var output2 = SpanByteAndMemory.FromPinnedSpan(output); - fixed (byte* key2Ptr = key2) - fixed (byte* value2Ptr = value2) - { - var key2SpanByte = SpanByte.FromPinnedPointer(key2Ptr, key2.Length); - var value2SpanByte = SpanByte.FromPinnedPointer(value2Ptr, value2.Length); + var key2SpanByte = SpanByte.FromPinnedSpan(key2); + var value2SpanByte = SpanByte.FromPinnedSpan(value2); - _ = bContext.Upsert(key2SpanByte, value2SpanByte); - _ = bContext.Read(ref key2SpanByte, ref input, ref output2); - } + _ = bContext.Upsert(key2SpanByte, value2SpanByte); + _ = bContext.Read(ref key2SpanByte, ref input, ref output2); - ClassicAssert.IsTrue(!output2.IsSpanByte); - ClassicAssert.IsTrue(output2.Memory.Memory.Span.Slice(0, output2.Length).SequenceEqual(value2)); - output2.Memory.Dispose(); + ClassicAssert.IsTrue(output2.IsSpanByte); + ClassicAssert.IsTrue(output2.AsReadOnlySpan().SequenceEqual(value2)); } finally { @@ -86,7 +76,7 @@ public unsafe void SpanByteTest1() [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void MultiRead_SpanByte_Test() + public void MultiRead_SpanByte_Test() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); @@ -106,12 +96,17 @@ public unsafe void MultiRead_SpanByte_Test() using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; + Span keySpan = stackalloc byte[8]; + Span valueSpan = stackalloc byte[8]; + for (int i = 0; i < 200; i++) { - var key = MemoryMarshal.Cast($"{i}".AsSpan()); - var value = MemoryMarshal.Cast($"{i + 1000}".AsSpan()); - fixed (byte* k = key, v = value) - _ = bContext.Upsert(SpanByte.FromPinnedSpan(key), SpanByte.FromPinnedSpan(value)); + _ = i.TryFormat(keySpan, out var keyBytesWritten); + _ = (i + 1000).TryFormat(valueSpan, out var valueBytesWritten); + + _ = bContext.Upsert( + SpanByte.FromPinnedSpan(keySpan.Slice(0, keyBytesWritten)), + SpanByte.FromPinnedSpan(valueSpan.Slice(0, valueBytesWritten))); } // Read, evict all records to disk, read again @@ -134,9 +129,10 @@ void ReadKey(long key, long value, bool evicted) Status status; SpanByteAndMemory output = default; - var keyBytes = MemoryMarshal.Cast($"{key}".AsSpan()); - fixed (byte* _ = keyBytes) - status = bContext.Read(key: SpanByte.FromPinnedSpan(keyBytes), out output); + Span keyBytes = stackalloc byte[8]; + _ = key.TryFormat(keyBytes, out var keyBytesWritten); + + status = bContext.Read(SpanByte.FromPinnedSpan(keyBytes.Slice(0, keyBytesWritten)), out output); ClassicAssert.AreEqual(evicted, status.IsPending, "evicted/pending mismatch"); if (evicted) @@ -144,8 +140,7 @@ void ReadKey(long key, long value, bool evicted) ClassicAssert.IsTrue(status.Found, $"expected to find key; status = {status}, pending = {evicted}"); ClassicAssert.IsFalse(output.IsSpanByte, "Output should not have a valid SpanByte"); - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - ClassicAssert.AreEqual(value, long.Parse(outputString), $"outputString mismatch; pending = {evicted}"); + ClassicAssert.AreEqual(value, long.Parse(output.AsReadOnlySpan()), $"outputString mismatch; pending = {evicted}"); output.Memory.Dispose(); } }