feat(ReconnectBench): fuzz slow I/O delay emulation

platform-sdk/swirlds-benchmarks/src/jmh/java/com/swirlds/benchmark/ReconnectBench.java

@@ -71,13 +71,27 @@ public class ReconnectBench extends VirtualMapBaseBench {
     @Param({"0"})
     public long delayStorageMicroseconds;
 
+    /**
+     * A percentage fuzz range for the delayStorageMicroseconds values,
+     * e.g. 0.15 for a -15%..+15% range around the value.
+     */
+    @Param({"0.15"})
+    public double delayStorageFuzzRangePercent;
+
     /**
      * Emulated delay for serializeMessage() calls in both Teaching- and Learning-Synchronizers,
      * or zero for no delay. This emulates slow network I/O when sending data.
      */
     @Param({"0"})
     public long delayNetworkMicroseconds;
 
+    /**
+     * A percentage fuzz range for the delayNetworkMicroseconds values,
+     * e.g. 0.15 for a -15%..+15% range around the value.
+     */
+    @Param({"0.15"})
+    public double delayNetworkFuzzRangePercent;
+
     private VirtualMap<BenchmarkKey, BenchmarkValue> teacherMap;
     private VirtualMap<BenchmarkKey, BenchmarkValue> learnerMap;
     private MerkleInternal teacherTree;
@@ -214,6 +228,13 @@ public void tearDownInvocation() throws Exception {
     @Benchmark
     public void reconnect() throws Exception {
         node = MerkleBenchmarkUtils.hashAndTestSynchronization(
-                learnerTree, teacherTree, delayStorageMicroseconds, delayNetworkMicroseconds, configuration);
+                learnerTree,
+                teacherTree,
+                randomSeed,
+                delayStorageMicroseconds,
+                delayStorageFuzzRangePercent,
+                delayNetworkMicroseconds,
+                delayNetworkFuzzRangePercent,
+                configuration);
     }
 }

platform-sdk/swirlds-benchmarks/src/jmh/java/com/swirlds/benchmark/reconnect/MerkleBenchmarkUtils.java

@@ -55,8 +55,11 @@ public static MerkleInternal createTreeForMap(final VirtualMap<BenchmarkKey, Ben
     public static <T extends MerkleNode> T hashAndTestSynchronization(
             final MerkleNode startingTree,
             final MerkleNode desiredTree,
+            final long randomSeed,
             final long delayStorageMicroseconds,
+            final double delayStorageFuzzRangePercent,
             final long delayNetworkMicroseconds,
+            final double delayNetworkFuzzRangePercent,
             final Configuration configuration)
             throws Exception {
         System.out.println("------------");
@@ -74,8 +77,11 @@ public static <T extends MerkleNode> T hashAndTestSynchronization(
         return testSynchronization(
                 startingTree,
                 desiredTree,
+                randomSeed,
                 delayStorageMicroseconds,
+                delayStorageFuzzRangePercent,
                 delayNetworkMicroseconds,
+                delayNetworkFuzzRangePercent,
                 configuration,
                 reconnectConfig);
     }
@@ -87,8 +93,11 @@ public static <T extends MerkleNode> T hashAndTestSynchronization(
     private static <T extends MerkleNode> T testSynchronization(
             final MerkleNode startingTree,
             final MerkleNode desiredTree,
+            final long randomSeed,
             final long delayStorageMicroseconds,
+            final double delayStorageFuzzRangePercent,
             final long delayNetworkMicroseconds,
+            final double delayNetworkFuzzRangePercent,
             final Configuration configuration,
             final ReconnectConfig reconnectConfig)
             throws Exception {
@@ -132,8 +141,11 @@ private static <T extends MerkleNode> T testSynchronization(
                         streams.getLearnerInput(),
                         streams.getLearnerOutput(),
                         startingTree,
+                        randomSeed,
                         delayStorageMicroseconds,
+                        delayStorageFuzzRangePercent,
                         delayNetworkMicroseconds,
+                        delayNetworkFuzzRangePercent,
                         () -> {
                             try {
                                 streams.disconnect();
@@ -148,8 +160,11 @@ private static <T extends MerkleNode> T testSynchronization(
                         streams.getTeacherInput(),
                         streams.getTeacherOutput(),
                         desiredTree,
+                        randomSeed,
                         delayStorageMicroseconds,
+                        delayStorageFuzzRangePercent,
                         delayNetworkMicroseconds,
+                        delayNetworkFuzzRangePercent,
                         () -> {
                             try {
                                 streams.disconnect();

platform-sdk/swirlds-benchmarks/src/jmh/java/com/swirlds/benchmark/reconnect/lag/BenchmarkSlowAsyncOutputStream.java

@@ -24,6 +24,7 @@
 import com.swirlds.common.threading.pool.StandardWorkGroup;
 import java.io.IOException;
 import java.time.Duration;
+import java.util.Random;
 
 /**
  * This variant of the async output stream introduces an extra delay for every single
@@ -32,18 +33,28 @@
  */
 public class BenchmarkSlowAsyncOutputStream<T extends SelfSerializable> extends AsyncOutputStream<T> {
 
-    private final long delayStorageMicroseconds;
-    private final long delayNetworkMicroseconds;
+    private final LongFuzzer delayStorageMicrosecondsFuzzer;
+    private final LongFuzzer delayNetworkMicrosecondsFuzzer;
 
     public BenchmarkSlowAsyncOutputStream(
             final SerializableDataOutputStream out,
             final StandardWorkGroup workGroup,
+            final long randomSeed,
             final long delayStorageMicroseconds,
+            final double delayStorageFuzzRangePercent,
             final long delayNetworkMicroseconds,
+            final double delayNetworkFuzzRangePercent,
             final ReconnectConfig reconnectConfig) {
         super(out, workGroup, reconnectConfig);
-        this.delayStorageMicroseconds = delayStorageMicroseconds;
-        this.delayNetworkMicroseconds = delayNetworkMicroseconds;
+
+        // Note that we use randomSeed and -randomSeed for the two fuzzers
+        // to ensure that they don't end up returning the exact same
+        // (relatively, that is, in percentages) delay
+        // for both the storage and network.
+        delayStorageMicrosecondsFuzzer =
+                new LongFuzzer(delayStorageMicroseconds, new Random(randomSeed), delayStorageFuzzRangePercent);
+        delayNetworkMicrosecondsFuzzer =
+                new LongFuzzer(delayNetworkMicroseconds, new Random(-randomSeed), delayNetworkFuzzRangePercent);
     }
 
     /**
@@ -54,7 +65,7 @@ public void sendAsync(final T message) throws InterruptedException {
         if (!isAlive()) {
             throw new MerkleSynchronizationException("Messages can not be sent after close has been called.");
         }
-        sleepMicros(delayStorageMicroseconds);
+        sleepMicros(delayStorageMicrosecondsFuzzer.next());
         getOutgoingMessages().put(message);
     }
 
@@ -63,7 +74,7 @@ public void sendAsync(final T message) throws InterruptedException {
      */
     @Override
     protected void serializeMessage(final T message) throws IOException {
-        sleepMicros(delayNetworkMicroseconds);
+        sleepMicros(delayNetworkMicrosecondsFuzzer.next());
         message.serialize(getOutputStream());
     }
 

platform-sdk/swirlds-benchmarks/src/jmh/java/com/swirlds/benchmark/reconnect/lag/BenchmarkSlowLearningSynchronizer.java

@@ -33,8 +33,11 @@
  */
 public class BenchmarkSlowLearningSynchronizer extends LearningSynchronizer {
 
+    private final long randomSeed;
     private final long delayStorageMicroseconds;
+    private final double delayStorageFuzzRangePercent;
     private final long delayNetworkMicroseconds;
+    private final double delayNetworkFuzzRangePercent;
 
     /**
      * Create a new learning synchronizer with simulated latency.
@@ -43,14 +46,20 @@ public BenchmarkSlowLearningSynchronizer(
             final MerkleDataInputStream in,
             final MerkleDataOutputStream out,
             final MerkleNode root,
+            final long randomSeed,
             final long delayStorageMicroseconds,
+            final double delayStorageFuzzRangePercent,
             final long delayNetworkMicroseconds,
+            final double delayNetworkFuzzRangePercent,
             final Runnable breakConnection,
             final ReconnectConfig reconnectConfig) {
         super(getStaticThreadManager(), in, out, root, breakConnection, reconnectConfig);
 
+        this.randomSeed = randomSeed;
         this.delayStorageMicroseconds = delayStorageMicroseconds;
+        this.delayStorageFuzzRangePercent = delayStorageFuzzRangePercent;
         this.delayNetworkMicroseconds = delayNetworkMicroseconds;
+        this.delayNetworkFuzzRangePercent = delayNetworkFuzzRangePercent;
     }
 
     /**
@@ -60,6 +69,13 @@ public BenchmarkSlowLearningSynchronizer(
     protected AsyncOutputStream<QueryResponse> buildOutputStream(
             final StandardWorkGroup workGroup, final SerializableDataOutputStream out) {
         return new BenchmarkSlowAsyncOutputStream<>(
-                out, workGroup, delayStorageMicroseconds, delayNetworkMicroseconds, reconnectConfig);
+                out,
+                workGroup,
+                randomSeed,
+                delayStorageMicroseconds,
+                delayStorageFuzzRangePercent,
+                delayNetworkMicroseconds,
+                delayNetworkFuzzRangePercent,
+                reconnectConfig);
     }
 }

platform-sdk/swirlds-benchmarks/src/jmh/java/com/swirlds/benchmark/reconnect/lag/BenchmarkSlowTeachingSynchronizer.java

@@ -36,8 +36,11 @@
  */
 public class BenchmarkSlowTeachingSynchronizer extends TeachingSynchronizer {
 
+    private final long randomSeed;
     private final long delayStorageMicroseconds;
+    private final double delayStorageFuzzRangePercent;
     private final long delayNetworkMicroseconds;
+    private final double delayNetworkFuzzRangePercent;
 
     /**
      * Create a new teaching synchronizer with simulated latency.
@@ -47,8 +50,11 @@ public BenchmarkSlowTeachingSynchronizer(
             final MerkleDataInputStream in,
             final MerkleDataOutputStream out,
             final MerkleNode root,
+            final long randomSeed,
             final long delayStorageMicroseconds,
+            final double delayStorageFuzzRangePercent,
             final long delayNetworkMicroseconds,
+            final double delayNetworkFuzzRangePercent,
             final Runnable breakConnection,
             final ReconnectConfig reconnectConfig) {
         super(
@@ -60,8 +66,12 @@ public BenchmarkSlowTeachingSynchronizer(
                 root,
                 breakConnection,
                 reconnectConfig);
+
+        this.randomSeed = randomSeed;
         this.delayStorageMicroseconds = delayStorageMicroseconds;
+        this.delayStorageFuzzRangePercent = delayStorageFuzzRangePercent;
         this.delayNetworkMicroseconds = delayNetworkMicroseconds;
+        this.delayNetworkFuzzRangePercent = delayNetworkFuzzRangePercent;
     }
 
     /**
@@ -71,6 +81,13 @@ public BenchmarkSlowTeachingSynchronizer(
     protected <T> AsyncOutputStream<Lesson<T>> buildOutputStream(
             final StandardWorkGroup workGroup, final SerializableDataOutputStream out) {
         return new BenchmarkSlowAsyncOutputStream<>(
-                out, workGroup, delayStorageMicroseconds, delayNetworkMicroseconds, reconnectConfig);
+                out,
+                workGroup,
+                randomSeed,
+                delayStorageMicroseconds,
+                delayStorageFuzzRangePercent,
+                delayNetworkMicroseconds,
+                delayNetworkFuzzRangePercent,
+                reconnectConfig);
     }
 }

platform-sdk/swirlds-benchmarks/src/jmh/java/com/swirlds/benchmark/reconnect/lag/LongFuzzer.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.swirlds.benchmark.reconnect.lag;
+
+import java.util.Random;
+
+/**
+ * A generator of random long values in a given percentage range around a given base value.
+ *
+ * @param value a base value
+ * @param random a Random instance, or null for no fuzz
+ * @param rangePercent a rangePercent, e.g. 0.15 for a -15%..+15% range around the base value.
+ */
+public record LongFuzzer(long value, Random random, double rangePercent) {
+    /**
+     * Returns the next long value in the range -rangePercent..+rangePercent around the base value.
+     * @return the next long value
+     */
+    public long next() {
+        if (random == null || rangePercent == .0) return value;
+
+        // Generate a random fuzz percentage in the range -rangePercent..+rangePercent, e.g. -0.15..+0.15 for 15% range
+        final double fuzzPercent = random.nextDouble(rangePercent * 2.) - rangePercent;
+        return (long) ((double) value * (1. + fuzzPercent));
+    }
+}