Skip to content

Commit 4716a1f

Browse files
lhotariganesh-ctds
authored andcommitted
[improve][broker] Make maxBatchDeletedIndexToPersist configurable and document other related configs (apache#24392)
(cherry picked from commit d1bca65) (cherry picked from commit dfcc246)
1 parent bda5605 commit 4716a1f

File tree

5 files changed

+72
-18
lines changed

5 files changed

+72
-18
lines changed

conf/broker.conf

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1282,6 +1282,14 @@ managedLedgerCursorMaxEntriesPerLedger=50000
12821282
# Max time before triggering a rollover on a cursor ledger
12831283
managedLedgerCursorRolloverTimeInSeconds=14400
12841284

1285+
# Maximum amount of memory used hold data read from storage (or from the cache).
1286+
# This mechanism prevents the broker to have too many concurrent
1287+
# reads from storage and fall into Out of Memory errors in case
1288+
# of multiple concurrent reads to multiple concurrent consumers.
1289+
# Set 0 in order to disable the feature.
1290+
#
1291+
managedLedgerMaxReadsInFlightSizeInMB=0
1292+
12851293
# Max number of "acknowledgment holes" that are going to be persistently stored.
12861294
# When acknowledging out of order, a consumer will leave holes that are supposed
12871295
# to be quickly filled by acking all the messages. The information of which
@@ -1291,13 +1299,22 @@ managedLedgerCursorRolloverTimeInSeconds=14400
12911299
# crashes.
12921300
managedLedgerMaxUnackedRangesToPersist=10000
12931301

1294-
# Maximum amount of memory used hold data read from storage (or from the cache).
1295-
# This mechanism prevents the broker to have too many concurrent
1296-
# reads from storage and fall into Out of Memory errors in case
1297-
# of multiple concurrent reads to multiple concurrent consumers.
1298-
# Set 0 in order to disable the feature.
1299-
#
1300-
managedLedgerMaxReadsInFlightSizeInMB=0
1302+
# Maximum number of partially acknowledged batch messages per subscription that will have their batch
1303+
# deleted indexes persisted. Batch deleted index state is handled when acknowledgmentAtBatchIndexLevelEnabled=true.
1304+
# When this limit is exceeded, remaining batch message containing the batch deleted indexes will
1305+
# only be tracked in memory. In case of broker restarts or load balancing events, the batch
1306+
# deleted indexes will be cleared while redelivering the messages to consumers.
1307+
managedLedgerMaxBatchDeletedIndexToPersist=10000
1308+
1309+
# When storing acknowledgement state, choose a more compact serialization format that stores
1310+
# individual acknowledgements as a bitmap which is serialized to an array of long values. NOTE: This setting requires
1311+
# managedLedgerUnackedRangesOpenCacheSetEnabled=true to be effective.
1312+
managedLedgerPersistIndividualAckAsLongArray=false
1313+
1314+
# When set to true, a BitSet will be used to track acknowledged messages that come after the "mark delete position"
1315+
# for each subscription. RoaringBitmap is used as a memory efficient BitSet implementation for the acknowledged
1316+
# messages tracking. Unacknowledged ranges are the message ranges excluding the acknowledged messages.
1317+
managedLedgerUnackedRangesOpenCacheSetEnabled=true
13011318

13021319
# Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher
13031320
# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
@@ -1776,9 +1793,6 @@ narExtractionDirectory=
17761793
# Maximum prefetch rounds for ledger reading for offloading
17771794
managedLedgerOffloadPrefetchRounds=1
17781795

1779-
# Use Open Range-Set to cache unacked messages
1780-
managedLedgerUnackedRangesOpenCacheSetEnabled=true
1781-
17821796
# For Amazon S3 ledger offload, AWS region
17831797
s3ManagedLedgerOffloadRegion=
17841798

conf/standalone.conf

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,23 @@ managedLedgerMaxSizePerLedgerMbytes=2048
833833
# crashes.
834834
managedLedgerMaxUnackedRangesToPersist=10000
835835

836+
# Maximum number of partially acknowledged batch messages per subscription that will have their batch
837+
# deleted indexes persisted. Batch deleted index state is handled when acknowledgmentAtBatchIndexLevelEnabled=true.
838+
# When this limit is exceeded, remaining batch message containing the batch deleted indexes will
839+
# only be tracked in memory. In case of broker restarts or load balancing events, the batch
840+
# deleted indexes will be cleared while redelivering the messages to consumers.
841+
managedLedgerMaxBatchDeletedIndexToPersist=10000
842+
843+
# When storing acknowledgement state, choose a more compact serialization format that stores
844+
# individual acknowledgements as a bitmap which is serialized to an array of long values. NOTE: This setting requires
845+
# managedLedgerUnackedRangesOpenCacheSetEnabled=true to be effective.
846+
managedLedgerPersistIndividualAckAsLongArray=false
847+
848+
# When set to true, a BitSet will be used to track acknowledged messages that come after the "mark delete position"
849+
# for each subscription. RoaringBitmap is used as a memory efficient BitSet implementation for the acknowledged
850+
# messages tracking. Unacknowledged ranges are the message ranges excluding the acknowledged messages.
851+
managedLedgerUnackedRangesOpenCacheSetEnabled=true
852+
836853
# Max number of "acknowledgment holes" that can be stored in MetadataStore. If number of unack message range is higher
837854
# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into
838855
# MetadataStore.
@@ -867,9 +884,6 @@ managedLedgerMinimumBacklogEntriesForCaching=1000
867884
# Maximum backlog entry difference to prevent caching entries that can't be reused.
868885
managedLedgerMaxBacklogBetweenCursorsForCaching=1000
869886

870-
# Use Open Range-Set to cache unacked messages
871-
managedLedgerUnackedRangesOpenCacheSetEnabled=true
872-
873887
# Managed ledger prometheus stats latency rollover seconds (default: 60s)
874888
managedLedgerPrometheusStatsLatencyRolloverSeconds=60
875889

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -502,6 +502,16 @@ public int getMaxBatchDeletedIndexToPersist() {
502502
return maxBatchDeletedIndexToPersist;
503503
}
504504

505+
/**
506+
* Set max batch deleted index that will be persisted and recovered.
507+
*
508+
* @param maxBatchDeletedIndexToPersist
509+
* max batch deleted index that will be persisted and recovered.
510+
*/
511+
public void setMaxBatchDeletedIndexToPersist(int maxBatchDeletedIndexToPersist) {
512+
this.maxBatchDeletedIndexToPersist = maxBatchDeletedIndexToPersist;
513+
}
514+
505515
public boolean isPersistentUnackedRangesWithMultipleEntriesEnabled() {
506516
return persistentUnackedRangesWithMultipleEntriesEnabled;
507517
}

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2275,10 +2275,22 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece
22752275
+ " will only be tracked in memory and messages will be redelivered in case of"
22762276
+ " crashes.")
22772277
private int managedLedgerMaxUnackedRangesToPersist = 10000;
2278-
@FieldContext(
2279-
category = CATEGORY_STORAGE_ML,
2280-
doc = "Whether persist cursor ack stats as long arrays, which will compress the data and reduce GC rate")
2278+
2279+
@FieldContext(category = CATEGORY_STORAGE_ML,
2280+
doc = "Maximum number of partially acknowledged batch messages per subscription that will have their batch "
2281+
+ "deleted indexes persisted. Batch deleted index state is handled when "
2282+
+ "acknowledgmentAtBatchIndexLevelEnabled=true.\n\n"
2283+
+ "When this limit is exceeded, remaining batch message containing the batch deleted indexes will "
2284+
+ "only be tracked in memory. In case of broker restarts or load balancing events, the batch "
2285+
+ "deleted indexes will be cleared while redelivering the messages to consumers.")
2286+
private int managedLedgerMaxBatchDeletedIndexToPersist = 10000;
2287+
2288+
@FieldContext(category = CATEGORY_STORAGE_ML,
2289+
doc = "When storing acknowledgement state, choose a more compact serialization format that stores"
2290+
+ " individual acknowledgements as a bitmap which is serialized to an array of long values.\n\n"
2291+
+ "NOTE: This setting requires managedLedgerUnackedRangesOpenCacheSetEnabled=true to be effective.")
22812292
private boolean managedLedgerPersistIndividualAckAsLongArray = false;
2293+
22822294
@FieldContext(
22832295
category = CATEGORY_STORAGE_ML,
22842296
doc = "If enabled, the maximum \"acknowledgment holes\" will not be limited and \"acknowledgment holes\" "
@@ -2301,8 +2313,10 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece
23012313
private int managedLedgerMaxUnackedRangesToPersistInMetadataStore = 1000;
23022314
@FieldContext(
23032315
category = CATEGORY_STORAGE_OFFLOADING,
2304-
doc = "Use Open Range-Set to cache unacked messages (it is memory efficient but it can take more cpu)"
2305-
)
2316+
doc = "When set to true, a BitSet will be used to track acknowledged messages that come after the \"mark "
2317+
+ "delete position\" for each subscription.\n\nRoaringBitmap is used as a memory efficient BitSet "
2318+
+ "implementation for the acknowledged messages tracking. Unacknowledged ranges are the message "
2319+
+ "ranges excluding the acknowledged messages.")
23062320
private boolean managedLedgerUnackedRangesOpenCacheSetEnabled = true;
23072321
@FieldContext(
23082322
dynamic = true,

pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2044,6 +2044,8 @@ public CompletableFuture<ManagedLedgerConfig> getManagedLedgerConfig(@NonNull To
20442044

20452045
managedLedgerConfig
20462046
.setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist());
2047+
managedLedgerConfig.setMaxBatchDeletedIndexToPersist(
2048+
serviceConfig.getManagedLedgerMaxBatchDeletedIndexToPersist());
20472049
managedLedgerConfig
20482050
.setPersistIndividualAckAsLongArray(serviceConfig.isManagedLedgerPersistIndividualAckAsLongArray());
20492051
managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled(

0 commit comments

Comments
 (0)