Skip to content

Commit 0446dfe

Browse files
SungJin1212SungJin1212
committed
Add multi-level chunk cache
Signed-off-by: SungJin1212 <[email protected]>
1 parent e449374 commit 0446dfe

File tree

9 files changed

+587
-43
lines changed

9 files changed

+587
-43
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
* [FEATURE] Ruler: Experimental: Add `ruler.frontend-address` to allow query to query frontends instead of ingesters. #6151
88
* [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129
99
* [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245
10+
* [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249
1011
* [ENHANCEMENT] Ingester: Add `blocks-storage.tsdb.wal-compression-type` to support zstd wal compression type. #6232
1112
* [ENHANCEMENT] Query Frontend: Add info field to query response. #6207
1213
* [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188

‎docs/blocks-storage/querier.md

+19-2
Original file line numberDiff line numberDiff line change
@@ -788,8 +788,10 @@ blocks_storage:
788788
[max_backfill_items: <int> | default = 10000]
789789

790790
chunks_cache:
791-
# Backend for chunks cache, if not empty. Supported values: memcached,
792-
# redis, inmemory, and '' (disable).
791+
# The chunks cache backend type. Single or Multiple cache backend can be
792+
# provided. Supported values in single cache: memcached, redis, inmemory,
793+
# and '' (disable). Supported values in multi level cache: a
794+
# comma-separated list of (inmemory, memcached, redis)
793795
# CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
794796
[backend: <string> | default = ""]
795797

@@ -1000,6 +1002,21 @@ blocks_storage:
10001002
# CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent
10011003
[failure_percent: <float> | default = 0.05]
10021004

1005+
multilevel:
1006+
# The maximum number of concurrent asynchronous operations can occur
1007+
# when backfilling cache items.
1008+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency
1009+
[max_async_concurrency: <int> | default = 50]
1010+
1011+
# The maximum number of enqueued asynchronous operations allowed when
1012+
# backfilling cache items.
1013+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size
1014+
[max_async_buffer_size: <int> | default = 10000]
1015+
1016+
# The maximum number of items to backfill per asynchronous operation.
1017+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items
1018+
[max_backfill_items: <int> | default = 10000]
1019+
10031020
# Size of each subrange that bucket object is split into for better
10041021
# caching.
10051022
# CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size

‎docs/blocks-storage/store-gateway.md

+19-2
Original file line numberDiff line numberDiff line change
@@ -903,8 +903,10 @@ blocks_storage:
903903
[max_backfill_items: <int> | default = 10000]
904904

905905
chunks_cache:
906-
# Backend for chunks cache, if not empty. Supported values: memcached,
907-
# redis, inmemory, and '' (disable).
906+
# The chunks cache backend type. Single or Multiple cache backend can be
907+
# provided. Supported values in single cache: memcached, redis, inmemory,
908+
# and '' (disable). Supported values in multi level cache: a
909+
# comma-separated list of (inmemory, memcached, redis)
908910
# CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
909911
[backend: <string> | default = ""]
910912

@@ -1115,6 +1117,21 @@ blocks_storage:
11151117
# CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent
11161118
[failure_percent: <float> | default = 0.05]
11171119

1120+
multilevel:
1121+
# The maximum number of concurrent asynchronous operations can occur
1122+
# when backfilling cache items.
1123+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency
1124+
[max_async_concurrency: <int> | default = 50]
1125+
1126+
# The maximum number of enqueued asynchronous operations allowed when
1127+
# backfilling cache items.
1128+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size
1129+
[max_async_buffer_size: <int> | default = 10000]
1130+
1131+
# The maximum number of items to backfill per asynchronous operation.
1132+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items
1133+
[max_backfill_items: <int> | default = 10000]
1134+
11181135
# Size of each subrange that bucket object is split into for better
11191136
# caching.
11201137
# CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size

‎docs/configuration/config-file-reference.md

+19-2
Original file line numberDiff line numberDiff line change
@@ -1339,8 +1339,10 @@ bucket_store:
13391339
[max_backfill_items: <int> | default = 10000]
13401340

13411341
chunks_cache:
1342-
# Backend for chunks cache, if not empty. Supported values: memcached,
1343-
# redis, inmemory, and '' (disable).
1342+
# The chunks cache backend type. Single or Multiple cache backend can be
1343+
# provided. Supported values in single cache: memcached, redis, inmemory,
1344+
# and '' (disable). Supported values in multi level cache: a comma-separated
1345+
# list of (inmemory, memcached, redis)
13441346
# CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
13451347
[backend: <string> | default = ""]
13461348

@@ -1549,6 +1551,21 @@ bucket_store:
15491551
# CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-async.circuit-breaker.failure-percent
15501552
[failure_percent: <float> | default = 0.05]
15511553

1554+
multilevel:
1555+
# The maximum number of concurrent asynchronous operations can occur when
1556+
# backfilling cache items.
1557+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-concurrency
1558+
[max_async_concurrency: <int> | default = 50]
1559+
1560+
# The maximum number of enqueued asynchronous operations allowed when
1561+
# backfilling cache items.
1562+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-async-buffer-size
1563+
[max_async_buffer_size: <int> | default = 10000]
1564+
1565+
# The maximum number of items to backfill per asynchronous operation.
1566+
# CLI flag: -blocks-storage.bucket-store.chunks-cache.multilevel.max-backfill-items
1567+
[max_backfill_items: <int> | default = 10000]
1568+
15521569
# Size of each subrange that bucket object is split into for better caching.
15531570
# CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size
15541571
[subrange_size: <int> | default = 16000]

‎integration/querier_test.go

+28-2
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
9797
chunkCacheBackend: tsdb.CacheBackendRedis,
9898
bucketIndexEnabled: true,
9999
},
100+
"blocks sharding disabled, in-memory chunk cache": {
101+
blocksShardingStrategy: "",
102+
indexCacheBackend: tsdb.IndexCacheBackendRedis,
103+
chunkCacheBackend: tsdb.CacheBackendInMemory,
104+
bucketIndexEnabled: true,
105+
},
100106
"blocks default sharding, in-memory chunk cache": {
101107
blocksShardingStrategy: "default",
102108
indexCacheBackend: tsdb.IndexCacheBackendRedis,
@@ -110,6 +116,25 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
110116
chunkCacheBackend: tsdb.CacheBackendInMemory,
111117
bucketIndexEnabled: true,
112118
},
119+
"block sharding disabled, multi-level chunk cache": {
120+
blocksShardingStrategy: "",
121+
indexCacheBackend: tsdb.IndexCacheBackendRedis,
122+
chunkCacheBackend: fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
123+
bucketIndexEnabled: true,
124+
},
125+
"block default sharding, multi-level chunk cache": {
126+
blocksShardingStrategy: "default",
127+
indexCacheBackend: tsdb.IndexCacheBackendRedis,
128+
chunkCacheBackend: fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
129+
bucketIndexEnabled: true,
130+
},
131+
"block shuffle sharding, multi-level chunk cache": {
132+
blocksShardingStrategy: "shuffle-sharding",
133+
tenantShardSize: 1,
134+
indexCacheBackend: tsdb.IndexCacheBackendRedis,
135+
chunkCacheBackend: fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
136+
bucketIndexEnabled: true,
137+
},
113138
}
114139

115140
for testName, testCfg := range tests {
@@ -154,9 +179,10 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
154179
if strings.Contains(testCfg.indexCacheBackend, tsdb.IndexCacheBackendRedis) {
155180
flags["-blocks-storage.bucket-store.index-cache.redis.addresses"] = redis.NetworkEndpoint(e2ecache.RedisPort)
156181
}
157-
if testCfg.chunkCacheBackend == tsdb.CacheBackendMemcached {
182+
if strings.Contains(testCfg.chunkCacheBackend, tsdb.CacheBackendMemcached) {
158183
flags["-blocks-storage.bucket-store.chunks-cache.memcached.addresses"] = "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort)
159-
} else if testCfg.chunkCacheBackend == tsdb.CacheBackendRedis {
184+
}
185+
if strings.Contains(testCfg.chunkCacheBackend, tsdb.CacheBackendRedis) {
160186
flags["-blocks-storage.bucket-store.chunks-cache.redis.addresses"] = redis.NetworkEndpoint(e2ecache.RedisPort)
161187
}
162188

‎pkg/storage/tsdb/caching_bucket.go

+89-35
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,15 @@ import (
2121
"github.com/thanos-io/thanos/pkg/cacheutil"
2222
"github.com/thanos-io/thanos/pkg/model"
2323
storecache "github.com/thanos-io/thanos/pkg/store/cache"
24+
25+
"github.com/cortexproject/cortex/pkg/util"
2426
)
2527

2628
var (
29+
supportedChunkCacheBackends = []string{CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis}
30+
2731
errUnsupportedChunkCacheBackend = errors.New("unsupported chunk cache backend")
32+
errDuplicatedChunkCacheBackend = errors.New("duplicated chunk cache backend")
2833
)
2934

3035
const (
@@ -54,23 +59,52 @@ func (cfg *MetadataCacheBackend) Validate() error {
5459
}
5560

5661
type ChunkCacheBackend struct {
57-
Backend string `yaml:"backend"`
58-
InMemory InMemoryChunkCacheConfig `yaml:"inmemory"`
59-
Memcached MemcachedClientConfig `yaml:"memcached"`
60-
Redis RedisClientConfig `yaml:"redis"`
62+
Backend string `yaml:"backend"`
63+
InMemory InMemoryChunkCacheConfig `yaml:"inmemory"`
64+
Memcached MemcachedClientConfig `yaml:"memcached"`
65+
Redis RedisClientConfig `yaml:"redis"`
66+
MultiLevel MultiLevelChunkCacheConfig `yaml:"multilevel"`
6167
}
6268

6369
// Validate the config.
6470
func (cfg *ChunkCacheBackend) Validate() error {
65-
switch cfg.Backend {
66-
case CacheBackendMemcached:
67-
return cfg.Memcached.Validate()
68-
case CacheBackendRedis:
69-
return cfg.Redis.Validate()
70-
case CacheBackendInMemory, "":
71-
default:
72-
return errUnsupportedChunkCacheBackend
71+
if cfg.Backend == "" {
72+
return nil
73+
}
74+
75+
splitBackends := strings.Split(cfg.Backend, ",")
76+
configuredBackends := map[string]struct{}{}
77+
78+
if len(splitBackends) > 1 {
79+
if err := cfg.MultiLevel.Validate(); err != nil {
80+
return err
81+
}
7382
}
83+
84+
for _, backend := range splitBackends {
85+
if !util.StringsContain(supportedChunkCacheBackends, backend) {
86+
return errUnsupportedChunkCacheBackend
87+
}
88+
89+
if _, ok := configuredBackends[backend]; ok {
90+
return errDuplicatedChunkCacheBackend
91+
}
92+
93+
switch backend {
94+
case CacheBackendMemcached:
95+
if err := cfg.Memcached.Validate(); err != nil {
96+
return err
97+
}
98+
case CacheBackendRedis:
99+
if err := cfg.Redis.Validate(); err != nil {
100+
return err
101+
}
102+
case CacheBackendInMemory:
103+
}
104+
105+
configuredBackends[backend] = struct{}{}
106+
}
107+
74108
return nil
75109
}
76110

@@ -84,16 +118,22 @@ type ChunksCacheConfig struct {
84118
}
85119

86120
func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
87-
f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("Backend for chunks cache, if not empty. Supported values: %s, %s, %s, and '' (disable).", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory))
121+
f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("The chunks cache backend type. Single or Multiple cache backend can be provided. "+
122+
"Supported values in single cache: %s, %s, %s, and '' (disable). "+
123+
"Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedChunkCacheBackends, ", ")))
88124

89125
cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.")
90126
cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.")
91127
cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.")
128+
cfg.MultiLevel.RegisterFlagsWithPrefix(f, prefix+"multilevel.")
92129

93130
f.Int64Var(&cfg.SubrangeSize, prefix+"subrange-size", 16000, "Size of each subrange that bucket object is split into for better caching.")
94131
f.IntVar(&cfg.MaxGetRangeRequests, prefix+"max-get-range-requests", 3, "Maximum number of sub-GetRange requests that a single GetRange request can be split into when fetching chunks. Zero or negative value = unlimited number of sub-requests.")
95132
f.DurationVar(&cfg.AttributesTTL, prefix+"attributes-ttl", 168*time.Hour, "TTL for caching object attributes for chunks.")
96133
f.DurationVar(&cfg.SubrangeTTL, prefix+"subrange-ttl", 24*time.Hour, "TTL for caching individual chunks subranges.")
134+
135+
// In the multi level chunk cache, backfill TTL follows subrange TTL
136+
cfg.ChunkCacheBackend.MultiLevel.BackFillTTL = cfg.SubrangeTTL
97137
}
98138

99139
func (cfg *ChunksCacheConfig) Validate() error {
@@ -230,34 +270,48 @@ func createMetadataCache(cacheName string, cacheBackend *MetadataCacheBackend, l
230270
}
231271

232272
func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) {
233-
switch cacheBackend.Backend {
234-
case "":
273+
if cacheBackend.Backend == "" {
235274
// No caching.
236275
return nil, nil
237-
case CacheBackendInMemory:
238-
inMemoryCache, err := cache.NewInMemoryCacheWithConfig(cacheName, logger, reg, cacheBackend.InMemory.toInMemoryChunkCacheConfig())
239-
if err != nil {
240-
return nil, errors.Wrapf(err, "failed to create in-memory chunk cache")
241-
}
242-
return inMemoryCache, nil
243-
case CacheBackendMemcached:
244-
var client cacheutil.MemcachedClient
245-
client, err := cacheutil.NewMemcachedClientWithConfig(logger, cacheName, cacheBackend.Memcached.ToMemcachedClientConfig(), reg)
246-
if err != nil {
247-
return nil, errors.Wrapf(err, "failed to create memcached client")
248-
}
249-
return cache.NewMemcachedCache(cacheName, logger, client, reg), nil
276+
}
250277

251-
case CacheBackendRedis:
252-
redisCache, err := cacheutil.NewRedisClientWithConfig(logger, cacheName, cacheBackend.Redis.ToRedisClientConfig(), reg)
253-
if err != nil {
254-
return nil, errors.Wrapf(err, "failed to create redis client")
278+
splitBackends := strings.Split(cacheBackend.Backend, ",")
279+
var (
280+
caches []cache.Cache
281+
)
282+
283+
for i, backend := range splitBackends {
284+
iReg := reg
285+
286+
// Create the level label if we have more than one cache
287+
if len(splitBackends) > 1 {
288+
iReg = prometheus.WrapRegistererWith(prometheus.Labels{"level": fmt.Sprintf("L%v", i)}, reg)
255289
}
256-
return cache.NewRedisCache(cacheName, logger, redisCache, reg), nil
257290

258-
default:
259-
return nil, errors.Errorf("unsupported cache type for cache %s: %s", cacheName, cacheBackend.Backend)
291+
switch backend {
292+
case CacheBackendInMemory:
293+
inMemoryCache, err := cache.NewInMemoryCacheWithConfig(cacheName, logger, iReg, cacheBackend.InMemory.toInMemoryChunkCacheConfig())
294+
if err != nil {
295+
return nil, errors.Wrapf(err, "failed to create in-memory chunk cache")
296+
}
297+
caches = append(caches, inMemoryCache)
298+
case CacheBackendMemcached:
299+
var client cacheutil.MemcachedClient
300+
client, err := cacheutil.NewMemcachedClientWithConfig(logger, cacheName, cacheBackend.Memcached.ToMemcachedClientConfig(), reg)
301+
if err != nil {
302+
return nil, errors.Wrapf(err, "failed to create memcached client")
303+
}
304+
caches = append(caches, cache.NewMemcachedCache(cacheName, logger, client, iReg))
305+
case CacheBackendRedis:
306+
redisCache, err := cacheutil.NewRedisClientWithConfig(logger, cacheName, cacheBackend.Redis.ToRedisClientConfig(), reg)
307+
if err != nil {
308+
return nil, errors.Wrapf(err, "failed to create redis client")
309+
}
310+
caches = append(caches, cache.NewRedisCache(cacheName, logger, redisCache, iReg))
311+
}
260312
}
313+
314+
return newMultiLevelChunkCache(cacheName, cacheBackend.MultiLevel, reg, caches...), nil
261315
}
262316

263317
type Matchers struct {

0 commit comments

Comments
 (0)