Skip to content

Commit f12b263

Browse files
Li0kLi0k
authored
feat(storage): monitor avg_key_size and avg_epoch_count (risingwavelabs#8297)
1 parent 953e4b2 commit f12b263

File tree

5 files changed

+86
-32
lines changed

5 files changed

+86
-32
lines changed

‎grafana/risingwave-dashboard.dashboard.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -768,7 +768,7 @@ def section_compaction(outer_panels):
768768
"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size",
769769
[
770770
panels.target(
771-
f"sum by(le, job, instance)(rate({metric('compactor_sstable_avg_key_size_sum')}[$__rate_interval])) / sum by(le, job, instance)(rate({metric('state_store_sstable_avg_key_size_count')}[$__rate_interval]))",
771+
f"sum by(le, job, instance)(rate({metric('compactor_sstable_avg_key_size_sum')}[$__rate_interval])) / sum by(le, job, instance)(rate({metric('compactor_sstable_avg_key_size_count')}[$__rate_interval]))",
772772
"avg_key_size - {{job}} @ {{instance}}",
773773
),
774774
panels.target(
@@ -778,6 +778,17 @@ def section_compaction(outer_panels):
778778
],
779779
),
780780

781+
panels.timeseries_count(
782+
"Hummock Sstable Stat",
783+
"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count",
784+
[
785+
panels.target(
786+
f"sum by(le, job, instance)(rate({metric('compactor_sstable_distinct_epoch_count_sum')}[$__rate_interval])) / sum by(le, job, instance)(rate({metric('compactor_sstable_distinct_epoch_count_count')}[$__rate_interval]))",
787+
"avg_epoch_count - {{job}} @ {{instance}}",
788+
),
789+
],
790+
),
791+
781792
panels.timeseries_latency(
782793
"Hummock Remote Read Duration",
783794
"Total time of operations which read from remote storage when enable prefetch",

‎grafana/risingwave-dashboard.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/storage/src/hummock/sstable/builder.rs

Lines changed: 55 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ pub struct SstableBuilderOutput<WO> {
8383
pub writer_output: WO,
8484
pub avg_key_size: usize,
8585
pub avg_value_size: usize,
86+
pub epoch_count: usize,
8687
}
8788

8889
pub struct SstableBuilder<W: SstableWriter, F: FilterBuilder> {
@@ -119,8 +120,7 @@ pub struct SstableBuilder<W: SstableWriter, F: FilterBuilder> {
119120

120121
filter_builder: F,
121122

122-
min_epoch: u64,
123-
max_epoch: u64,
123+
epoch_set: BTreeSet<u64>,
124124
}
125125

126126
impl<W: SstableWriter> SstableBuilder<W, XorFilterBuilder> {
@@ -168,8 +168,7 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
168168
total_key_count: 0,
169169
table_stats: Default::default(),
170170
last_table_stats: Default::default(),
171-
min_epoch: u64::MAX,
172-
max_epoch: u64::MIN,
171+
epoch_set: BTreeSet::default(),
173172
}
174173
}
175174

@@ -238,6 +237,8 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
238237
self.total_key_count += 1;
239238
self.last_table_stats.total_key_count += 1;
240239

240+
self.epoch_set.insert(full_key.epoch);
241+
241242
if is_new_table && !self.block_builder.is_empty() {
242243
self.build_block().await?;
243244
}
@@ -263,9 +264,6 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
263264
self.raw_key.clear();
264265
self.raw_value.clear();
265266

266-
self.min_epoch = cmp::min(self.min_epoch, full_key.epoch);
267-
self.max_epoch = cmp::max(self.max_epoch, full_key.epoch);
268-
269267
if self.block_builder.approximate_len() >= self.options.block_capacity {
270268
self.build_block().await?;
271269
}
@@ -363,6 +361,48 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
363361
(tombstone_min_epoch, tombstone_max_epoch)
364362
};
365363

364+
let (avg_key_size, avg_value_size) = if self.table_stats.is_empty() {
365+
(0, 0)
366+
} else {
367+
let total_key_count: usize = self
368+
.table_stats
369+
.values()
370+
.map(|s| s.total_key_count as usize)
371+
.sum();
372+
373+
if total_key_count == 0 {
374+
(0, 0)
375+
} else {
376+
let total_key_size: usize = self
377+
.table_stats
378+
.values()
379+
.map(|s| s.total_key_size as usize)
380+
.sum();
381+
382+
let total_value_size: usize = self
383+
.table_stats
384+
.values()
385+
.map(|s| s.total_value_size as usize)
386+
.sum();
387+
388+
(
389+
total_key_size / total_key_count,
390+
total_value_size / total_key_count,
391+
)
392+
}
393+
};
394+
395+
let (min_epoch, max_epoch) = {
396+
if self.epoch_set.is_empty() {
397+
(u64::MAX, u64::MIN)
398+
} else {
399+
(
400+
*self.epoch_set.first().unwrap(),
401+
*self.epoch_set.last().unwrap(),
402+
)
403+
}
404+
};
405+
366406
let sst_info = SstableInfo {
367407
id: self.sstable_id,
368408
key_range: Some(risingwave_pb::hummock::KeyRange {
@@ -377,43 +417,29 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
377417
total_key_count: self.total_key_count,
378418
divide_version: 0,
379419
uncompressed_file_size: uncompressed_file_size + meta.encoded_size() as u64,
380-
min_epoch: cmp::min(self.min_epoch, tombstone_min_epoch),
381-
max_epoch: cmp::max(self.max_epoch, tombstone_max_epoch),
420+
min_epoch: cmp::min(min_epoch, tombstone_min_epoch),
421+
max_epoch: cmp::max(max_epoch, tombstone_max_epoch),
382422
};
383423
tracing::trace!(
384-
"meta_size {} bloom_filter_size {} add_key_counts {} stale_key_count {} min_epoch {} max_epoch {}",
424+
"meta_size {} bloom_filter_size {} add_key_counts {} stale_key_count {} min_epoch {} max_epoch {} epoch_count {}",
385425
meta.encoded_size(),
386426
meta.bloom_filter.len(),
387427
self.total_key_count,
388428
self.stale_key_count,
389-
self.min_epoch,
390-
self.max_epoch,
429+
min_epoch,
430+
max_epoch,
431+
self.epoch_set.len()
391432
);
392433
let bloom_filter_size = meta.bloom_filter.len();
393-
let (avg_key_size, avg_value_size) = if self.table_stats.is_empty() {
394-
(0, 0)
395-
} else {
396-
let avg_key_size = self
397-
.table_stats
398-
.values()
399-
.map(|s| s.total_key_size as usize)
400-
.sum::<usize>()
401-
/ self.table_stats.len();
402-
let avg_value_size = self
403-
.table_stats
404-
.values()
405-
.map(|s| s.total_value_size as usize)
406-
.sum::<usize>()
407-
/ self.table_stats.len();
408-
(avg_key_size, avg_value_size)
409-
};
434+
410435
let writer_output = self.writer.finish(meta).await?;
411436
Ok(SstableBuilderOutput::<W::Output> {
412437
sst_info: LocalSstableInfo::with_stats(sst_info, self.table_stats),
413438
bloom_filter_size,
414439
writer_output,
415440
avg_key_size,
416441
avg_value_size,
442+
epoch_count: self.epoch_set.len(),
417443
})
418444
}
419445

‎src/storage/src/hummock/sstable/multi_builder.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,12 @@ where
210210
.sstable_avg_value_size
211211
.observe(builder_output.avg_value_size as _);
212212
}
213+
214+
if builder_output.epoch_count != 0 {
215+
self.compactor_metrics
216+
.sstable_distinct_epoch_count
217+
.observe(builder_output.epoch_count as _);
218+
}
213219
}
214220
self.sst_outputs.push(SplitTableOutput {
215221
upload_join_handle: builder_output.writer_output,

‎src/storage/src/monitor/compactor_metrics.rs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ pub struct CompactorMetrics {
4242
pub sstable_avg_value_size: Histogram,
4343
pub iter_scan_key_counts: GenericCounterVec<AtomicU64>,
4444
pub write_build_l0_bytes: GenericCounter<AtomicU64>,
45+
pub sstable_distinct_epoch_count: Histogram,
4546
}
4647

4748
impl CompactorMetrics {
@@ -171,7 +172,7 @@ impl CompactorMetrics {
171172
let opts = histogram_opts!(
172173
"compactor_sstable_avg_value_size",
173174
"Total bytes gotten from sstable_avg_value_size, for observing sstable_avg_value_size",
174-
exponential_buckets(1.0, 2.0, 25).unwrap() // max 16MB
175+
exponential_buckets(1.0, 2.0, 26).unwrap() // max 32MB
175176
);
176177

177178
let sstable_avg_value_size = register_histogram_with_registry!(opts, registry).unwrap();
@@ -198,6 +199,15 @@ impl CompactorMetrics {
198199
registry
199200
).unwrap();
200201

202+
let opts = histogram_opts!(
203+
"compactor_sstable_distinct_epoch_count",
204+
"Total number gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count",
205+
exponential_buckets(1.0, 2.0, 17).unwrap()
206+
);
207+
208+
let sstable_distinct_epoch_count =
209+
register_histogram_with_registry!(opts, registry).unwrap();
210+
201211
Self {
202212
compaction_upload_sst_counts,
203213
compact_write_bytes,
@@ -219,6 +229,7 @@ impl CompactorMetrics {
219229
sstable_avg_value_size,
220230
iter_scan_key_counts,
221231
write_build_l0_bytes,
232+
sstable_distinct_epoch_count,
222233
}
223234
}
224235

0 commit comments

Comments
 (0)