Added MerkleDbConfig.goodAverageBucketEntryCount config to replace HDHM.GOOD_AVERAGE_BUCKET_ENTRY_COUNT

Signed-off-by: Artem Ananev <[email protected]>

Author: artemananiev

platform-sdk/swirlds-merkledb/src/main/java/com/swirlds/merkledb/config/MerkleDbConfig.java

@@ -46,6 +46,10 @@
  * @param indexRebuildingEnforced
  * 		Configuration used to avoid reading stored indexes from a saved state and enforce rebuilding those indexes from
  * 		data files.
+ * @param goodAverageBucketEntryCount
+ *      Target average number of entries in HalfDiskHashMap buckets. This number is used to calculate the number
+ *      of buckets to allocate based on projected virtual map size, and also to check if it's time to double the
+ *      number of HalfDiskHashMap buckets.
  * @param tablesToRepairHdhm
  *      Comma-delimited list of data source names, may be empty. When a MerkleDb data source with a name from the
  *      list is loaded from a snapshot, its key to path map will be rebuilt from path to KV data files. Note that
@@ -80,6 +84,7 @@ public record MerkleDbConfig(
         @Positive @ConfigProperty(defaultValue = "16777216") int iteratorInputBufferBytes,
         @ConfigProperty(defaultValue = "false") boolean reconnectKeyLeakMitigationEnabled,
         @ConfigProperty(defaultValue = "false") boolean indexRebuildingEnforced,
+        @ConfigProperty(defaultValue = "32") int goodAverageBucketEntryCount,
         @ConfigProperty(defaultValue = "") String tablesToRepairHdhm,
         @ConfigProperty(defaultValue = "75.0") double percentHalfDiskHashMapFlushThreads,
         @ConfigProperty(defaultValue = "-1") int numHalfDiskHashMapFlushThreads,

platform-sdk/swirlds-merkledb/src/main/java/com/swirlds/merkledb/files/hashmap/HalfDiskHashMap.java

@@ -75,7 +75,7 @@ public class HalfDiskHashMap implements AutoCloseable, Snapshotable, FileStatist
      * starts to exceed the number of buckets times this average number of entries, the
      * map is resized by doubling the number of buckets.
      */
-    private static final long GOOD_AVERAGE_BUCKET_ENTRY_COUNT = 32;
+    private final int goodAverageBucketEntryCount;
 
     /** The limit on the number of concurrent read tasks in {@code endWriting()} */
     private static final int MAX_IN_FLIGHT = 1024;
@@ -208,10 +208,11 @@ public HalfDiskHashMap(
             throws IOException {
         this.config = requireNonNull(configuration);
         final MerkleDbConfig merkleDbConfig = this.config.getConfigData(MerkleDbConfig.class);
+        this.goodAverageBucketEntryCount = merkleDbConfig.goodAverageBucketEntryCount();
         // Max number of keys is limited by merkleDbConfig.maxNumberOfKeys. Number of buckets is,
         // on average, GOOD_AVERAGE_BUCKET_ENTRY_COUNT times smaller than the number of keys. To
         // be on the safe side, double that amount and use as a hard limit for bucket index size
-        final long bucketIndexCapacity = merkleDbConfig.maxNumOfKeys() * 2 / GOOD_AVERAGE_BUCKET_ENTRY_COUNT;
+        final long bucketIndexCapacity = merkleDbConfig.maxNumOfKeys() * 2 / goodAverageBucketEntryCount;
         this.storeName = storeName;
         Path indexFile = storeDir.resolve(storeName + BUCKET_INDEX_FILENAME_SUFFIX);
         // create bucket pool
@@ -274,7 +275,7 @@ public HalfDiskHashMap(
             // create store dir
             Files.createDirectories(storeDir);
             // calculate number of entries we can store in a disk page
-            final int minimumBuckets = (int) (initialCapacity / GOOD_AVERAGE_BUCKET_ENTRY_COUNT);
+            final int minimumBuckets = (int) (initialCapacity / goodAverageBucketEntryCount);
             // numOfBuckets is the nearest power of two greater than minimumBuckets with a min of 2
             setNumberOfBuckets(Math.max(Integer.highestOneBit(minimumBuckets) * 2, 2));
             // create new index
@@ -886,15 +887,15 @@ private Bucket readBucket(final int bucketIndex) throws IOException {
 
     /**
      * Check if this map should be resized, given the new virtual map size. If the new map size
-     * exceeds 80% of the current number of buckets times {@link #GOOD_AVERAGE_BUCKET_ENTRY_COUNT},
+     * exceeds 80% of the current number of buckets times {@link #goodAverageBucketEntryCount},
      * the map is resized by doubling the number of buckets.
      *
      * @param firstLeafPath The first leaf virtual path
      * @param lastLeafPath The last leaf virtual path
      */
     public void resizeIfNeeded(final long firstLeafPath, final long lastLeafPath) {
         final long currentSize = lastLeafPath - firstLeafPath + 1;
-        if (currentSize / numOfBuckets.get() <= GOOD_AVERAGE_BUCKET_ENTRY_COUNT * 80 / 100) {
+        if (currentSize / numOfBuckets.get() <= goodAverageBucketEntryCount * 80 / 100) {
             // No need to resize yet
             return;
         }
@@ -928,10 +929,10 @@ public void printStats() {
                 """
                         HalfDiskHashMap Stats {
                         	numOfBuckets = {}
-                        	GOOD_AVERAGE_BUCKET_ENTRY_COUNT = {}
+                        	goodAverageBucketEntryCount = {}
                         }""",
                 numOfBuckets,
-                GOOD_AVERAGE_BUCKET_ENTRY_COUNT);
+                goodAverageBucketEntryCount);
     }
 
     public DataFileCollection getFileCollection() {
@@ -954,6 +955,11 @@ private void setNumberOfBuckets(final int newValue) {
         bucketMaskBits.set(trailingZeroes);
     }
 
+    // For testing purposes
+    int getNumOfBuckets() {
+        return numOfBuckets.get();
+    }
+
     /**
      * Computes which bucket a key with the given hash falls. Depends on the fact the numOfBuckets
      * is a power of two. Based on same calculation that is used in java HashMap.