Skip to content

Commit aa8a3de

Browse files
adding ORSet benchmarks (#4990)
* adding ORSet benchmarks Working on #4956 * added VersionVectorBenchmark Using this to help gauge ORSet / other CRDT merge performance * moved all `VersionVector` comparisons to use `ValueTuple` instead of `KeyValuePair` * temporarily lowered ORSetBenchmark parameters * added VersionVector merge benchmarks and cleaned up code * working on more benchmarking for ORSet
1 parent b5f6552 commit aa8a3de

File tree

8 files changed

+521
-70
lines changed

8 files changed

+521
-70
lines changed

src/Akka.sln

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,9 +240,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Akka.DependencyInjection.Te
240240
EndProject
241241
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AspNetCore", "AspNetCore", "{162F5991-EA57-4221-9B70-F9B6FEC18036}"
242242
EndProject
243-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.Akka.AspNetCore", "examples\AspNetCore\Samples.Akka.AspNetCore\Samples.Akka.AspNetCore.csproj", "{D62F4AD6-318F-4ECC-B875-83FA9933A81B}"
243+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.Akka.AspNetCore", "examples\AspNetCore\Samples.Akka.AspNetCore\Samples.Akka.AspNetCore.csproj", "{D62F4AD6-318F-4ECC-B875-83FA9933A81B}"
244244
EndProject
245-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SerializationBenchmarks", "benchmark\SerializationBenchmarks\SerializationBenchmarks.csproj", "{2E4B9584-42CC-4D17-B719-9F462B16C94D}"
245+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SerializationBenchmarks", "benchmark\SerializationBenchmarks\SerializationBenchmarks.csproj", "{2E4B9584-42CC-4D17-B719-9F462B16C94D}"
246+
EndProject
247+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DDataStressTest", "examples\Cluster\DData\DDataStressTest\DDataStressTest.csproj", "{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}"
246248
EndProject
247249
Global
248250
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -1123,6 +1125,18 @@ Global
11231125
{2E4B9584-42CC-4D17-B719-9F462B16C94D}.Release|x64.Build.0 = Release|Any CPU
11241126
{2E4B9584-42CC-4D17-B719-9F462B16C94D}.Release|x86.ActiveCfg = Release|Any CPU
11251127
{2E4B9584-42CC-4D17-B719-9F462B16C94D}.Release|x86.Build.0 = Release|Any CPU
1128+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1129+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Debug|Any CPU.Build.0 = Debug|Any CPU
1130+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Debug|x64.ActiveCfg = Debug|Any CPU
1131+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Debug|x64.Build.0 = Debug|Any CPU
1132+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Debug|x86.ActiveCfg = Debug|Any CPU
1133+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Debug|x86.Build.0 = Debug|Any CPU
1134+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Release|Any CPU.ActiveCfg = Release|Any CPU
1135+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Release|Any CPU.Build.0 = Release|Any CPU
1136+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Release|x64.ActiveCfg = Release|Any CPU
1137+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Release|x64.Build.0 = Release|Any CPU
1138+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Release|x86.ActiveCfg = Release|Any CPU
1139+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50}.Release|x86.Build.0 = Release|Any CPU
11261140
EndGlobalSection
11271141
GlobalSection(SolutionProperties) = preSolution
11281142
HideSolutionNode = FALSE
@@ -1230,6 +1244,7 @@ Global
12301244
{162F5991-EA57-4221-9B70-F9B6FEC18036} = {D3AF8295-AEB5-4324-AA82-FCC0014AC310}
12311245
{D62F4AD6-318F-4ECC-B875-83FA9933A81B} = {162F5991-EA57-4221-9B70-F9B6FEC18036}
12321246
{2E4B9584-42CC-4D17-B719-9F462B16C94D} = {73108242-625A-4D7B-AA09-63375DBAE464}
1247+
{44B3DDD6-6103-4E8F-8AC2-0F4BA3CF6B50} = {C50E1A9E-820C-4E75-AE39-6F96A99AC4A7}
12331248
EndGlobalSection
12341249
GlobalSection(ExtensibilityGlobals) = postSolution
12351250
SolutionGuid = {03AD8E21-7507-4E68-A4E9-F4A7E7273164}
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using Akka.Actor;
6+
using Akka.Benchmarks.Configurations;
7+
using Akka.Cluster;
8+
using Akka.DistributedData;
9+
using BenchmarkDotNet.Attributes;
10+
11+
namespace Akka.Benchmarks.DData
12+
{
13+
[Config(typeof(MicroBenchmarkConfig))]
14+
public class ORSetBenchmarks
15+
{
16+
[Params(25)]
17+
public int NumElements;
18+
19+
[Params(10)]
20+
public int NumNodes;
21+
22+
[Params(100)]
23+
public int Iterations;
24+
25+
private UniqueAddress[] _nodes;
26+
private string[] _elements;
27+
28+
private readonly string _user1 = "{\"username\":\"john\",\"password\":\"coltrane\"}";
29+
private readonly string _user2 = "{\"username\":\"sonny\",\"password\":\"rollins\"}";
30+
private readonly string _user3 = "{\"username\":\"charlie\",\"password\":\"parker\"}";
31+
private readonly string _user4 = "{\"username\":\"charles\",\"password\":\"mingus\"}";
32+
33+
// has data from all nodes
34+
private ORSet<string> _c1 = ORSet<String>.Empty;
35+
36+
// has additional items from all nodes
37+
private ORSet<string> _c2 = ORSet<String>.Empty;
38+
39+
// has removed items from all nodes
40+
private ORSet<string> _c3 = ORSet<String>.Empty;
41+
42+
[GlobalSetup]
43+
public void Setup()
44+
{
45+
var newNodes = new List<UniqueAddress>(NumNodes);
46+
foreach(var i in Enumerable.Range(0, NumNodes)){
47+
var address = new Address("akka.tcp", "Sys", "localhost", 2552 + i);
48+
var uniqueAddress = new UniqueAddress(address, i);
49+
newNodes.Add(uniqueAddress);
50+
}
51+
_nodes = newNodes.ToArray();
52+
53+
var newElements = new List<string>(NumNodes);
54+
foreach(var i in Enumerable.Range(0, NumElements)){
55+
newElements.Add(i.ToString());
56+
}
57+
_elements = newElements.ToArray();
58+
59+
_c1 = ORSet<String>.Empty;
60+
foreach(var node in _nodes){
61+
_c1 = _c1.Add(node, _elements[0]);
62+
}
63+
64+
// add some data that _c2 doesn't have
65+
_c2 = _c1;
66+
foreach(var node in _nodes.Skip(NumNodes/2)){
67+
_c2 = _c2.Add(node, _elements[1]);
68+
}
69+
70+
_c3 = _c1;
71+
foreach(var node in _nodes.Take(NumNodes/2)){
72+
_c3 = _c3.Remove(node, _elements[0]);
73+
}
74+
}
75+
76+
[Benchmark]
77+
public void Should_add_node_to_ORSet()
78+
{
79+
for (var i = 0; i < Iterations; i++)
80+
{
81+
var init = ORSet<string>.Empty;
82+
foreach (var node in _nodes)
83+
{
84+
init = init.Add(node, _elements[0]);
85+
}
86+
}
87+
88+
}
89+
90+
[Benchmark]
91+
public void Should_add_elements_for_Same_node()
92+
{
93+
for (var i = 0; i < Iterations; i++)
94+
{
95+
var init = ORSet<string>.Empty;
96+
foreach (var element in _elements)
97+
{
98+
init = init.Add(_nodes[0], element);
99+
}
100+
}
101+
}
102+
103+
[Benchmark]
104+
public void Should_merge_in_new_Elements_from_other_nodes(){
105+
for (var i = 0; i < Iterations; i++)
106+
{
107+
var c4 = _c1.Merge(_c2);
108+
}
109+
110+
}
111+
112+
[Benchmark]
113+
public void Should_merge_in_removed_Elements_from_other_nodes(){
114+
for (var i = 0; i < Iterations; i++)
115+
{
116+
var c4 = _c1.Merge(_c3);
117+
}
118+
119+
}
120+
121+
[Benchmark]
122+
public void Should_merge_in_add_and_removed_Elements_from_other_nodes(){
123+
for (var i = 0; i < Iterations; i++)
124+
{
125+
var c4 = _c1.Merge(_c2).Merge(_c3);
126+
}
127+
}
128+
}
129+
}
Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Collections.Immutable;
4+
using System.Linq;
5+
using System.Text;
6+
using Akka.Benchmarks.Configurations;
7+
using Akka.Cluster;
8+
using Akka.DistributedData;
9+
using BenchmarkDotNet.Attributes;
10+
using FluentAssertions;
11+
using static Akka.DistributedData.VersionVector;
12+
13+
namespace Akka.Benchmarks.DData
14+
{
15+
[Config(typeof(MicroBenchmarkConfig))]
16+
public class VersionVectorBenchmarks
17+
{
18+
[Params(100)]
19+
public int ClockSize;
20+
21+
[Params(1000)]
22+
public int Iterations;
23+
24+
internal (VersionVector clock, ImmutableSortedSet<UniqueAddress> nodes) CreateVectorClockOfSize(int size)
25+
{
26+
UniqueAddress GenerateUniqueAddress(int nodeCount){
27+
return new UniqueAddress(new Akka.Actor.Address("akka.tcp", "ClusterSys", "localhost", nodeCount), nodeCount);
28+
}
29+
30+
return Enumerable.Range(1, size)
31+
.Aggregate((VersionVector.Empty, ImmutableSortedSet<UniqueAddress>.Empty),
32+
(tuple, i) =>
33+
{
34+
var (vc, nodes) = tuple;
35+
var node = GenerateUniqueAddress(i);
36+
return (vc.Increment(node), nodes.Add(node));
37+
});
38+
}
39+
40+
internal VersionVector CopyVectorClock(VersionVector vc)
41+
{
42+
var versions = ImmutableDictionary<UniqueAddress, long>.Empty;
43+
var enumerator = vc.VersionEnumerator;
44+
while(enumerator.MoveNext()){
45+
var nodePair = enumerator.Current;
46+
versions = versions.SetItem(nodePair.Key, nodePair.Value);
47+
}
48+
49+
return VersionVector.Create(versions);
50+
}
51+
52+
private UniqueAddress _firstNode;
53+
private UniqueAddress _lastNode;
54+
private UniqueAddress _middleNode;
55+
private ImmutableSortedSet<UniqueAddress> _nodes;
56+
private VersionVector _vcBefore;
57+
private VersionVector _vcBaseLast;
58+
private VersionVector _vcAfterLast;
59+
private VersionVector _vcConcurrentLast;
60+
private VersionVector _vcBaseMiddle;
61+
private VersionVector _vcAfterMiddle;
62+
private VersionVector _vcConcurrentMiddle;
63+
64+
[GlobalSetup]
65+
public void Setup()
66+
{
67+
var (vcBefore, nodes) = CreateVectorClockOfSize(ClockSize);
68+
_vcBefore = vcBefore;
69+
_nodes = nodes;
70+
71+
_firstNode = nodes.First();
72+
_lastNode = nodes.Last();
73+
_middleNode = nodes[ClockSize / 2];
74+
75+
_vcBaseLast = vcBefore.Increment(_lastNode);
76+
_vcAfterLast = _vcBaseLast.Increment(_firstNode);
77+
_vcConcurrentLast = _vcBaseLast.Increment(_lastNode);
78+
_vcBaseMiddle = _vcBefore.Increment(_middleNode);
79+
_vcAfterMiddle = _vcBaseMiddle.Increment(_firstNode);
80+
_vcConcurrentMiddle = _vcBaseMiddle.Increment(_middleNode);
81+
}
82+
83+
private void CheckThunkFor(VersionVector vc1, VersionVector vc2, Action<VersionVector, VersionVector> thunk, int times)
84+
{
85+
var vcc1 = CopyVectorClock(vc1);
86+
var vcc2 = CopyVectorClock(vc2);
87+
for (var i = 0; i < times; i++)
88+
{
89+
thunk(vcc1, vcc2);
90+
}
91+
}
92+
93+
private void CompareTo(VersionVector vc1, VersionVector vc2, Ordering ordering)
94+
{
95+
vc1.Compare(vc2).Should().Be(ordering);
96+
}
97+
98+
private void NotEqual(VersionVector vc1, VersionVector vc2)
99+
{
100+
(vc1 == vc2).Should().BeFalse();
101+
}
102+
103+
private void Merge(VersionVector vc1, VersionVector vc2)
104+
{
105+
vc1.Merge(vc2);
106+
}
107+
108+
[Benchmark]
109+
public void VectorClock_comparisons_should_compare_same()
110+
{
111+
CheckThunkFor(_vcBaseLast, _vcBaseLast, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.Same), Iterations);
112+
}
113+
114+
[Benchmark]
115+
public void VectorClock_comparisons_should_compare_Before_last()
116+
{
117+
CheckThunkFor(_vcBefore, _vcBaseLast, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.Before), Iterations);
118+
}
119+
120+
[Benchmark]
121+
public void VectorClock_comparisons_should_compare_After_last()
122+
{
123+
CheckThunkFor(_vcAfterLast, _vcBaseLast, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.After), Iterations);
124+
}
125+
126+
[Benchmark]
127+
public void VectorClock_comparisons_should_compare_Concurrent_last()
128+
{
129+
CheckThunkFor(_vcAfterLast, _vcConcurrentLast, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.Concurrent), Iterations);
130+
}
131+
132+
[Benchmark]
133+
public void VectorClock_comparisons_should_compare_Before_middle()
134+
{
135+
CheckThunkFor(_vcBefore, _vcBaseMiddle, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.Before), Iterations);
136+
}
137+
138+
[Benchmark]
139+
public void VectorClock_comparisons_should_compare_After_middle()
140+
{
141+
CheckThunkFor(_vcAfterMiddle, _vcBaseMiddle, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.After), Iterations);
142+
}
143+
144+
[Benchmark]
145+
public void VectorClock_comparisons_should_compare_Concurrent_middle()
146+
{
147+
CheckThunkFor(_vcAfterMiddle, _vcConcurrentMiddle, (clock, vectorClock) => CompareTo(clock, vectorClock, Ordering.Concurrent), Iterations);
148+
}
149+
150+
[Benchmark]
151+
public void VectorClock_comparisons_should_compare_notEquals_Before_Middle()
152+
{
153+
CheckThunkFor(_vcBefore, _vcBaseMiddle, (clock, vectorClock) => NotEqual(clock, vectorClock), Iterations);
154+
}
155+
156+
[Benchmark]
157+
public void VectorClock_comparisons_should_compare_notEquals_After_Middle()
158+
{
159+
CheckThunkFor(_vcAfterMiddle, _vcBaseMiddle, (clock, vectorClock) => NotEqual(clock, vectorClock), Iterations);
160+
}
161+
162+
[Benchmark]
163+
public void VectorClock_comparisons_should_compare_notEquals_Concurrent_Middle()
164+
{
165+
CheckThunkFor(_vcAfterMiddle, _vcConcurrentMiddle, (clock, vectorClock) => NotEqual(clock, vectorClock), Iterations);
166+
}
167+
168+
[Benchmark]
169+
public void VersionVector_merge_Multi_Multi()
170+
{
171+
CheckThunkFor(_vcBefore, _vcAfterLast, (vector, versionVector) => Merge(vector, versionVector), Iterations);
172+
}
173+
}
174+
}

src/contrib/cluster/Akka.DistributedData.Tests/ORSetSpec.cs

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,20 @@ public ORSetSpec(ITestOutputHelper output)
5757
[Fact]
5858
public void ORSet_must_be_able_to_add_element()
5959
{
60-
var c1 = ORSet<string>.Empty;
61-
var c2 = c1.Add(_node1, _user1);
62-
var c3 = c2.Add(_node1, _user2);
63-
var c4 = c3.Add(_node1, _user4);
64-
var c5 = c4.Add(_node1, _user3);
65-
66-
Assert.Contains(_user1, c5.Elements);
67-
Assert.Contains(_user2, c5.Elements);
68-
Assert.Contains(_user3, c5.Elements);
69-
Assert.Contains(_user4, c5.Elements);
60+
for (var i = 0; i < 100; i++)
61+
{
62+
var c1 = ORSet<string>.Empty;
63+
var c2 = c1.Add(_node1, _user1);
64+
var c3 = c2.Add(_node1, _user2);
65+
var c4 = c3.Add(_node1, _user4);
66+
var c5 = c4.Add(_node1, _user3);
67+
68+
Assert.Contains(_user1, c5.Elements);
69+
Assert.Contains(_user2, c5.Elements);
70+
Assert.Contains(_user3, c5.Elements);
71+
Assert.Contains(_user4, c5.Elements);
72+
}
73+
7074
}
7175

7276
[Fact]

0 commit comments

Comments
 (0)