Skip to content

Commit d5acbd8

Browse files
authored
feat: 8841: Need a way to change the number of object key buckets in MerkleDb (#18910)
Fixes: #8841 Reviewed-by: Ivan Malygin <[email protected]>, Jasper Potts <[email protected]>, Kelly Greco <[email protected]>, Michael Tinker <[email protected]>, Neeharika Sompalli <[email protected]>, Nikita Lebedev <[email protected]>, Oleg Mazurov <[email protected]> Signed-off-by: Artem Ananev <[email protected]>
1 parent 95d642b commit d5acbd8

File tree

41 files changed

+1099
-431
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1099
-431
lines changed

hedera-node/hedera-app/src/main/java/com/hedera/node/app/hints/schemas/V059HintsSchema.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ public class V059HintsSchema extends Schema {
3434
private static final SemanticVersion VERSION =
3535
SemanticVersion.newBuilder().minor(59).build();
3636

37-
private static final long MAX_HINTS = 1L << 31;
38-
private static final long MAX_PREPROCESSING_VOTES = 1L << 31;
37+
private static final long MAX_HINTS = 1L << 10;
38+
private static final long MAX_PREPROCESSING_VOTES = 1L << 10;
3939

4040
public static final String HINTS_KEY_SETS_KEY = "HINTS_KEY_SETS";
4141
public static final String ACTIVE_HINT_CONSTRUCTION_KEY = "ACTIVE_HINT_CONSTRUCTION";

hedera-node/hedera-app/src/main/java/com/hedera/node/app/hints/schemas/V060HintsSchema.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public class V060HintsSchema extends Schema {
3030
public static final String CRS_PUBLICATIONS_KEY = "CRS_PUBLICATIONS";
3131
private static final Logger log = LogManager.getLogger(V060HintsSchema.class);
3232

33-
private static final long MAX_CRS_PUBLICATIONS = 1L << 31;
33+
private static final long MAX_CRS_PUBLICATIONS = 1L << 10;
3434

3535
private final HintsContext signingContext;
3636
private final HintsLibrary library;

hedera-node/hedera-app/src/test/java/com/hedera/node/app/state/merkle/MerkleSchemaRegistryTest.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,9 @@ void setUp() {
7575
final var hederaConfig = mock(HederaConfig.class);
7676
lenient().when(config.getConfigData(HederaConfig.class)).thenReturn(hederaConfig);
7777
final var merkleDbConfig = mock(MerkleDbConfig.class);
78+
lenient().when(merkleDbConfig.goodAverageBucketEntryCount()).thenReturn(32);
7879
lenient().when(merkleDbConfig.longListChunkSize()).thenReturn(1024);
80+
lenient().when(merkleDbConfig.maxNumOfKeys()).thenReturn(1000L);
7981
lenient().when(config.getConfigData(MerkleDbConfig.class)).thenReturn(merkleDbConfig);
8082
final var virtualMapConfig = mock(VirtualMapConfig.class);
8183
lenient().when(config.getConfigData(VirtualMapConfig.class)).thenReturn(virtualMapConfig);

hedera-node/hedera-consensus-service-impl/src/main/java/com/hedera/node/app/service/consensus/impl/schemas/V0490ConsensusSchema.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public class V0490ConsensusSchema extends Schema {
2525
private static final SemanticVersion VERSION =
2626
SemanticVersion.newBuilder().major(0).minor(49).patch(0).build();
2727

28-
private static final long MAX_TOPICS = 1_000_000_000L;
28+
private static final long MAX_TOPICS = 1_000_000L;
2929

3030
/**
3131
* Constructor for this schema.

hedera-node/hedera-file-service-impl/src/main/java/com/hedera/node/app/service/file/impl/schemas/V0490FileSchema.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,14 @@ public class V0490FileSchema extends Schema {
9292
private static final String DEFAULT_THROTTLES_RESOURCE = "genesis/throttles.json";
9393

9494
/**
95-
* A hint to the database system of the maximum number of files we will store. This MUST NOT BE CHANGED. If it is
96-
* changed, then the database has to be rebuilt.
95+
* A hint to the database system of the expected maximum number of files we will store. This hint
96+
* is used by the database to optimize its indices. If more than this number of files are actually
97+
* stored, the database can handle that just fine.
98+
*
99+
* <p>If this number is changed, it will not have any effect on existing networks. Only new
100+
* deployments will use the updated hint.
97101
*/
98-
private static final int MAX_FILES_HINT = 50_000_000;
102+
private static final int MAX_FILES_HINT = 50_000;
99103
/**
100104
* The version of the schema.
101105
*/

hedera-node/hedera-schedule-service-impl/src/main/java/com/hedera/node/app/service/schedule/impl/schemas/V0490ScheduleSchema.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
* General schema for the schedule service.
1818
*/
1919
public final class V0490ScheduleSchema extends Schema {
20-
private static final long MAX_SCHEDULES_BY_ID_KEY = 50_000_000L;
21-
private static final long MAX_SCHEDULES_BY_EXPIRY_SEC_KEY = 50_000_000L;
22-
private static final long MAX_SCHEDULES_BY_EQUALITY = 50_000_000L;
20+
private static final long MAX_SCHEDULES_BY_ID_KEY = 50_000L;
21+
private static final long MAX_SCHEDULES_BY_EXPIRY_SEC_KEY = 50_000L;
22+
private static final long MAX_SCHEDULES_BY_EQUALITY = 50_000L;
2323
/**
2424
* The version of the schema.
2525
*/

hedera-node/hedera-schedule-service-impl/src/main/java/com/hedera/node/app/service/schedule/impl/schemas/V0570ScheduleSchema.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@
2424
public final class V0570ScheduleSchema extends Schema {
2525
private static final Logger log = LogManager.getLogger(V0570ScheduleSchema.class);
2626

27-
private static final long MAX_SCHEDULED_COUNTS = 50_000_000L;
28-
private static final long MAX_SCHEDULED_ORDERS = 50_000_000L;
29-
private static final long MAX_SCHEDULED_USAGES = 50_000_000L;
30-
private static final long MAX_SCHEDULE_ID_BY_EQUALITY = 50_000_000L;
27+
private static final long MAX_SCHEDULED_COUNTS = 50_000L;
28+
private static final long MAX_SCHEDULED_ORDERS = 50_000L;
29+
private static final long MAX_SCHEDULED_USAGES = 50_000L;
30+
private static final long MAX_SCHEDULE_ID_BY_EQUALITY = 50_000L;
3131
private static final SemanticVersion VERSION =
3232
SemanticVersion.newBuilder().major(0).minor(57).patch(0).build();
3333
/**

hedera-node/hedera-smart-contract-service-impl/src/main/java/com/hedera/node/app/service/contract/impl/schemas/V0490ContractSchema.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
* for both the contract storage and bytecode.
1919
*/
2020
public class V0490ContractSchema extends Schema {
21-
private static final int MAX_BYTECODES = 50_000_000;
22-
private static final int MAX_STORAGE_ENTRIES = 1_000_000_000;
21+
private static final int MAX_BYTECODES = 50_000;
22+
private static final int MAX_STORAGE_ENTRIES = 1_000_000;
2323
private static final SemanticVersion VERSION =
2424
SemanticVersion.newBuilder().major(0).minor(49).patch(0).build();
2525

hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/V0490TokenSchema.java

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,17 @@
2424
* Initial mod-service schema for the token service.
2525
*/
2626
public class V0490TokenSchema extends Schema {
27-
// These need to be big so databases are created at right scale. If they are too small then the on disk hash map
28-
// buckets will be too full which results in very poor performance. Have chosen 10 billion as should give us
29-
// plenty of runway.
30-
private static final long MAX_STAKING_INFOS = 1_000_000L;
31-
private static final long MAX_TOKENS = 1_000_000_000L;
32-
private static final long MAX_ACCOUNTS = 1_000_000_000L;
33-
private static final long MAX_TOKEN_RELS = 1_000_000_000L;
34-
private static final long MAX_MINTABLE_NFTS = 1_000_000_000L;
27+
28+
// Initial virtual map capacity values. Previously these values had to be large enough to avoid
29+
// key hash collisions at the database level, which would result in low performance. With the
30+
// new feature of dynamic hash map resizing in the database, these capacity hints can be kept
31+
// low. These are just hints for initial virtual map sizes. Over time the maps will be able to
32+
// contain more elements, if needed
33+
private static final long MAX_STAKING_INFOS = 1_000L;
34+
private static final long MAX_TOKENS = 1_000_000L;
35+
private static final long MAX_ACCOUNTS = 1_000_000L;
36+
private static final long MAX_TOKEN_RELS = 1_000_000L;
37+
private static final long MAX_MINTABLE_NFTS = 1_000_000L;
3538
private static final SemanticVersion VERSION =
3639
SemanticVersion.newBuilder().major(0).minor(49).patch(0).build();
3740

hedera-node/hedera-token-service-impl/src/main/java/com/hedera/node/app/service/token/impl/schemas/V0530TokenSchema.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
public class V0530TokenSchema extends Schema {
1515
private static final Logger logger = LogManager.getLogger(V0530TokenSchema.class);
16-
private static final long MAX_PENDING_AIRDROPS = 1_000_000_000L;
16+
private static final long MAX_PENDING_AIRDROPS = 1_000_000L;
1717
public static final String AIRDROPS_KEY = "PENDING_AIRDROPS";
1818

1919
private static final SemanticVersion VERSION =

platform-sdk/docs/proposals/scaling-hdhm-buckets/bucket-entries-cleanup.svg

Lines changed: 1 addition & 0 deletions
Loading

platform-sdk/docs/proposals/scaling-hdhm-buckets/bucket-index-after-resize.svg

Lines changed: 1 addition & 0 deletions
Loading
Lines changed: 1 addition & 0 deletions
Loading

0 commit comments

Comments
 (0)