Skip to content

Commit 2e16c26

Browse files
MrCroxxMrCroxx
authored
feat(stats): introduce block efficiency metrics (#14654)
Signed-off-by: MrCroxx <[email protected]>
1 parent 5436acf commit 2e16c26

File tree

16 files changed

+475
-150
lines changed

16 files changed

+475
-150
lines changed

‎src/ctl/src/common/hummock_service.rs

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,14 @@ use std::sync::Arc;
1717
use std::time::Duration;
1818

1919
use anyhow::{anyhow, bail, Result};
20-
use risingwave_common::config::ObjectStoreConfig;
20+
use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
2121
use risingwave_object_store::object::build_remote_object_store;
2222
use risingwave_rpc_client::MetaClient;
2323
use risingwave_storage::hummock::hummock_meta_client::MonitoredHummockMetaClient;
24-
use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore};
24+
use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore, SstableStoreConfig};
2525
use risingwave_storage::monitor::{
26-
CompactorMetrics, HummockMetrics, HummockStateStoreMetrics, MonitoredStateStore,
27-
MonitoredStorageMetrics, ObjectStoreMetrics,
26+
global_hummock_state_store_metrics, CompactorMetrics, HummockMetrics, HummockStateStoreMetrics,
27+
MonitoredStateStore, MonitoredStorageMetrics, ObjectStoreMetrics,
2828
};
2929
use risingwave_storage::opts::StorageOpts;
3030
use risingwave_storage::{StateStore, StateStoreImpl};
@@ -162,17 +162,20 @@ impl HummockServiceOpts {
162162

163163
let opts = self.get_storage_opts();
164164

165-
Ok(Arc::new(SstableStore::new(
166-
Arc::new(object_store),
167-
opts.data_directory,
168-
opts.block_cache_capacity_mb * (1 << 20),
169-
opts.meta_cache_capacity_mb * (1 << 20),
170-
0,
171-
opts.block_cache_capacity_mb * (1 << 20),
172-
opts.max_prefetch_block_number,
173-
FileCache::none(),
174-
FileCache::none(),
175-
None,
176-
)))
165+
Ok(Arc::new(SstableStore::new(SstableStoreConfig {
166+
store: Arc::new(object_store),
167+
path: opts.data_directory,
168+
block_cache_capacity: opts.block_cache_capacity_mb * (1 << 20),
169+
meta_cache_capacity: opts.meta_cache_capacity_mb * (1 << 20),
170+
high_priority_ratio: 0,
171+
prefetch_buffer_capacity: opts.block_cache_capacity_mb * (1 << 20),
172+
max_prefetch_block_number: opts.max_prefetch_block_number,
173+
data_file_cache: FileCache::none(),
174+
meta_file_cache: FileCache::none(),
175+
recent_filter: None,
176+
state_store_metrics: Arc::new(global_hummock_state_store_metrics(
177+
MetricLevel::Disabled,
178+
)),
179+
})))
177180
}
178181
}

‎src/jni_core/src/hummock_iterator.rs

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use std::sync::Arc;
1717
use bytes::Bytes;
1818
use futures::TryStreamExt;
1919
use risingwave_common::catalog::ColumnDesc;
20-
use risingwave_common::config::ObjectStoreConfig;
20+
use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
2121
use risingwave_common::hash::VirtualNode;
2222
use risingwave_common::row::OwnedRow;
2323
use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAwareSerde;
@@ -33,9 +33,9 @@ use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion;
3333
use risingwave_storage::hummock::store::version::HummockVersionReader;
3434
use risingwave_storage::hummock::store::HummockStorageIterator;
3535
use risingwave_storage::hummock::{
36-
get_committed_read_version_tuple, CachePolicy, FileCache, SstableStore,
36+
get_committed_read_version_tuple, CachePolicy, FileCache, SstableStore, SstableStoreConfig,
3737
};
38-
use risingwave_storage::monitor::HummockStateStoreMetrics;
38+
use risingwave_storage::monitor::{global_hummock_state_store_metrics, HummockStateStoreMetrics};
3939
use risingwave_storage::row_serde::value_serde::ValueRowSerdeNew;
4040
use risingwave_storage::store::{ReadOptions, StateStoreReadIterStream, StreamTypeOfIter};
4141
use rw_futures_util::select_all;
@@ -66,18 +66,21 @@ impl HummockJavaBindingIterator {
6666
)
6767
.await,
6868
);
69-
let sstable_store = Arc::new(SstableStore::new(
70-
object_store,
71-
read_plan.data_dir,
72-
1 << 10,
73-
1 << 10,
74-
0,
75-
1 << 10,
76-
16,
77-
FileCache::none(),
78-
FileCache::none(),
79-
None,
80-
));
69+
let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
70+
store: object_store,
71+
path: read_plan.data_dir,
72+
block_cache_capacity: 1 << 10,
73+
meta_cache_capacity: 1 << 10,
74+
high_priority_ratio: 0,
75+
prefetch_buffer_capacity: 1 << 10,
76+
max_prefetch_block_number: 16,
77+
data_file_cache: FileCache::none(),
78+
meta_file_cache: FileCache::none(),
79+
recent_filter: None,
80+
state_store_metrics: Arc::new(global_hummock_state_store_metrics(
81+
MetricLevel::Disabled,
82+
)),
83+
}));
8184
let reader = HummockVersionReader::new(
8285
sstable_store,
8386
Arc::new(HummockStateStoreMetrics::unused()),

‎src/storage/benches/bench_compactor.rs

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use criterion::async_executor::FuturesExecutor;
1919
use criterion::{criterion_group, criterion_main, Criterion};
2020
use risingwave_common::cache::CachePriority;
2121
use risingwave_common::catalog::TableId;
22+
use risingwave_common::config::MetricLevel;
2223
use risingwave_common::hash::VirtualNode;
2324
use risingwave_hummock_sdk::key::FullKey;
2425
use risingwave_hummock_sdk::key_range::KeyRange;
@@ -41,26 +42,29 @@ use risingwave_storage::hummock::sstable_store::SstableStoreRef;
4142
use risingwave_storage::hummock::value::HummockValue;
4243
use risingwave_storage::hummock::{
4344
CachePolicy, CompactionDeleteRangeIterator, FileCache, SstableBuilder, SstableBuilderOptions,
44-
SstableIterator, SstableStore, SstableWriterOptions, Xor16FilterBuilder,
45+
SstableIterator, SstableStore, SstableStoreConfig, SstableWriterOptions, Xor16FilterBuilder,
46+
};
47+
use risingwave_storage::monitor::{
48+
global_hummock_state_store_metrics, CompactorMetrics, StoreLocalStatistic,
4549
};
46-
use risingwave_storage::monitor::{CompactorMetrics, StoreLocalStatistic};
4750

4851
pub fn mock_sstable_store() -> SstableStoreRef {
4952
let store = InMemObjectStore::new().monitored(Arc::new(ObjectStoreMetrics::unused()));
5053
let store = Arc::new(ObjectStoreImpl::InMem(store));
5154
let path = "test".to_string();
52-
Arc::new(SstableStore::new(
55+
Arc::new(SstableStore::new(SstableStoreConfig {
5356
store,
5457
path,
55-
64 << 20,
56-
128 << 20,
57-
0,
58-
64 << 20,
59-
16,
60-
FileCache::none(),
61-
FileCache::none(),
62-
None,
63-
))
58+
block_cache_capacity: 64 << 20,
59+
meta_cache_capacity: 128 << 20,
60+
high_priority_ratio: 0,
61+
prefetch_buffer_capacity: 64 << 20,
62+
max_prefetch_block_number: 16,
63+
data_file_cache: FileCache::none(),
64+
meta_file_cache: FileCache::none(),
65+
recent_filter: None,
66+
state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
67+
}))
6468
}
6569

6670
pub fn default_writer_opts() -> SstableWriterOptions {

‎src/storage/benches/bench_multi_builder.rs

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,17 @@ use criterion::{criterion_group, criterion_main, Criterion};
2323
use futures::future::try_join_all;
2424
use itertools::Itertools;
2525
use risingwave_common::catalog::TableId;
26-
use risingwave_common::config::ObjectStoreConfig;
26+
use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
2727
use risingwave_hummock_sdk::key::{FullKey, UserKey};
2828
use risingwave_object_store::object::{ObjectStore, ObjectStoreImpl, S3ObjectStore};
2929
use risingwave_storage::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory};
3030
use risingwave_storage::hummock::value::HummockValue;
3131
use risingwave_storage::hummock::{
3232
BatchSstableWriterFactory, CachePolicy, FileCache, HummockResult, MemoryLimiter,
33-
SstableBuilder, SstableBuilderOptions, SstableStore, SstableWriterFactory,
33+
SstableBuilder, SstableBuilderOptions, SstableStore, SstableStoreConfig, SstableWriterFactory,
3434
SstableWriterOptions, StreamingSstableWriterFactory, Xor16FilterBuilder,
3535
};
36-
use risingwave_storage::monitor::ObjectStoreMetrics;
36+
use risingwave_storage::monitor::{global_hummock_state_store_metrics, ObjectStoreMetrics};
3737

3838
const RANGE: Range<u64> = 0..1500000;
3939
const VALUE: &[u8] = &[0; 400];
@@ -141,18 +141,19 @@ fn bench_builder(
141141
.monitored(metrics)
142142
});
143143
let object_store = Arc::new(ObjectStoreImpl::S3(object_store));
144-
let sstable_store = Arc::new(SstableStore::new(
145-
object_store,
146-
"test".to_string(),
147-
64 << 20,
148-
128 << 20,
149-
0,
150-
64 << 20,
151-
16,
152-
FileCache::none(),
153-
FileCache::none(),
154-
None,
155-
));
144+
let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
145+
store: object_store,
146+
path: "test".to_string(),
147+
block_cache_capacity: 64 << 20,
148+
meta_cache_capacity: 128 << 20,
149+
high_priority_ratio: 0,
150+
prefetch_buffer_capacity: 64 << 20,
151+
max_prefetch_block_number: 16,
152+
data_file_cache: FileCache::none(),
153+
meta_file_cache: FileCache::none(),
154+
recent_filter: None,
155+
state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
156+
}));
156157

157158
let mut group = c.benchmark_group("bench_multi_builder");
158159
group

‎src/storage/hummock_test/src/bin/replay/main.rs

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ use risingwave_object_store::object::build_remote_object_store;
4343
use risingwave_storage::filter_key_extractor::{
4444
FakeRemoteTableAccessor, RpcFilterKeyExtractorManager,
4545
};
46-
use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore};
46+
use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore, SstableStoreConfig};
4747
use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics, ObjectStoreMetrics};
4848
use risingwave_storage::opts::StorageOpts;
4949
use serde::{Deserialize, Serialize};
@@ -97,33 +97,32 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result<impl GlobalRepl
9797
&storage_memory_config,
9898
)));
9999

100-
let state_store_stats = Arc::new(HummockStateStoreMetrics::unused());
101-
let object_store_stats = Arc::new(ObjectStoreMetrics::unused());
100+
let state_store_metrics = Arc::new(HummockStateStoreMetrics::unused());
101+
let object_store_metrics = Arc::new(ObjectStoreMetrics::unused());
102102

103103
let compactor_metrics = Arc::new(CompactorMetrics::unused());
104104

105105
let object_store = build_remote_object_store(
106106
&args.object_storage,
107-
object_store_stats,
107+
object_store_metrics,
108108
"Hummock",
109109
ObjectStoreConfig::default(),
110110
)
111111
.await;
112112

113-
let sstable_store = {
114-
Arc::new(SstableStore::new(
115-
Arc::new(object_store),
116-
storage_opts.data_directory.to_string(),
117-
storage_opts.block_cache_capacity_mb * (1 << 20),
118-
storage_opts.meta_cache_capacity_mb * (1 << 20),
119-
storage_opts.high_priority_ratio,
120-
storage_opts.prefetch_buffer_capacity_mb * (1 << 20),
121-
storage_opts.max_prefetch_block_number,
122-
FileCache::none(),
123-
FileCache::none(),
124-
None,
125-
))
126-
};
113+
let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
114+
store: Arc::new(object_store),
115+
path: storage_opts.data_directory.to_string(),
116+
block_cache_capacity: storage_opts.block_cache_capacity_mb * (1 << 20),
117+
meta_cache_capacity: storage_opts.meta_cache_capacity_mb * (1 << 20),
118+
high_priority_ratio: storage_opts.high_priority_ratio,
119+
prefetch_buffer_capacity: storage_opts.prefetch_buffer_capacity_mb * (1 << 20),
120+
max_prefetch_block_number: storage_opts.max_prefetch_block_number,
121+
data_file_cache: FileCache::none(),
122+
meta_file_cache: FileCache::none(),
123+
recent_filter: None,
124+
state_store_metrics: state_store_metrics.clone(),
125+
}));
127126

128127
let (hummock_meta_client, notification_client, notifier) = {
129128
let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
@@ -158,7 +157,7 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result<impl GlobalRepl
158157
hummock_meta_client.clone(),
159158
notification_client,
160159
key_filter_manager,
161-
state_store_stats,
160+
state_store_metrics,
162161
compactor_metrics,
163162
)
164163
.await

‎src/storage/src/hummock/file_cache/store.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ impl Value for CachedBlock {
455455

456456
fn serialized_len(&self) -> usize {
457457
1 /* type */ + match self {
458-
CachedBlock::Loaded { block } => block.raw_data().len(),
458+
CachedBlock::Loaded { block } => block.raw().len(),
459459
CachedBlock::Fetched { bytes, uncompressed_capacity: _ } => 8 + bytes.len(),
460460
}
461461
}
@@ -506,7 +506,7 @@ impl std::io::Read for CachedBlockCursor {
506506
if self.pos < 1 {
507507
self.pos += copy([0], &mut buf);
508508
}
509-
self.pos += copy(&block.raw_data()[self.pos - 1..], &mut buf);
509+
self.pos += copy(&block.raw()[self.pos - 1..], &mut buf);
510510
}
511511
CachedBlock::Fetched {
512512
bytes,
@@ -541,7 +541,7 @@ impl Value for Box<Block> {
541541
type Cursor = BoxBlockCursor;
542542

543543
fn serialized_len(&self) -> usize {
544-
self.raw_data().len()
544+
self.raw().len()
545545
}
546546

547547
fn read(buf: &[u8]) -> CodingResult<Self> {
@@ -571,7 +571,7 @@ impl BoxBlockCursor {
571571
impl std::io::Read for BoxBlockCursor {
572572
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
573573
let pos = self.pos;
574-
self.pos += copy(&self.inner.raw_data()[self.pos..], buf);
574+
self.pos += copy(&self.inner.raw()[self.pos..], buf);
575575
let n = self.pos - pos;
576576
Ok(n)
577577
}
@@ -748,7 +748,7 @@ mod tests {
748748
std::io::copy(&mut cursor, &mut buf).unwrap();
749749
let target = cursor.into_inner();
750750
let block = Box::<Block>::read(&buf[..]).unwrap();
751-
assert_eq!(target.raw_data(), block.raw_data());
751+
assert_eq!(target.raw(), block.raw());
752752
}
753753

754754
{
@@ -779,7 +779,7 @@ mod tests {
779779
CachedBlock::Loaded { block } => block,
780780
CachedBlock::Fetched { .. } => panic!(),
781781
};
782-
assert_eq!(target.raw_data(), block.raw_data());
782+
assert_eq!(target.raw(), block.raw());
783783
}
784784

785785
{

‎src/storage/src/hummock/iterator/test_utils.rs

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use std::sync::Arc;
1818
use bytes::Bytes;
1919
use itertools::Itertools;
2020
use risingwave_common::catalog::TableId;
21+
use risingwave_common::config::MetricLevel;
2122
use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKey};
2223
use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId};
2324
use risingwave_object_store::object::{
@@ -33,9 +34,9 @@ use crate::hummock::test_utils::{
3334
};
3435
use crate::hummock::{
3536
DeleteRangeTombstone, FileCache, HummockValue, SstableBuilderOptions, SstableIterator,
36-
SstableIteratorType, SstableStoreRef, TableHolder,
37+
SstableIteratorType, SstableStoreConfig, SstableStoreRef, TableHolder,
3738
};
38-
use crate::monitor::ObjectStoreMetrics;
39+
use crate::monitor::{global_hummock_state_store_metrics, ObjectStoreMetrics};
3940

4041
/// `assert_eq` two `Vec<u8>` with human-readable format.
4142
#[macro_export]
@@ -59,18 +60,19 @@ pub fn mock_sstable_store() -> SstableStoreRef {
5960

6061
pub fn mock_sstable_store_with_object_store(store: ObjectStoreRef) -> SstableStoreRef {
6162
let path = "test".to_string();
62-
Arc::new(SstableStore::new(
63+
Arc::new(SstableStore::new(SstableStoreConfig {
6364
store,
6465
path,
65-
64 << 20,
66-
64 << 20,
67-
0,
68-
64 << 20,
69-
16,
70-
FileCache::none(),
71-
FileCache::none(),
72-
None,
73-
))
66+
block_cache_capacity: 64 << 20,
67+
meta_cache_capacity: 64 << 20,
68+
high_priority_ratio: 0,
69+
prefetch_buffer_capacity: 64 << 20,
70+
max_prefetch_block_number: 16,
71+
data_file_cache: FileCache::none(),
72+
meta_file_cache: FileCache::none(),
73+
recent_filter: None,
74+
state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
75+
}))
7476
}
7577

7678
pub fn iterator_test_table_key_of(idx: usize) -> Vec<u8> {

0 commit comments

Comments
 (0)