Skip to content

Commit aed289d

Browse files
SayrusSayrus
committed
mixin: add deletion metrics for compactor
Signed-off-by: Mathis Raguin <[email protected]>
1 parent 05fbe15 commit aed289d

File tree

3 files changed

+287
-9
lines changed

3 files changed

+287
-9
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ sse_config:
209209
- [#2579](https://github.com/thanos-io/thanos/pull/2579) Store: Experimental caching bucket can now cache metadata as well. Config has changed from #2532.
210210
- [#2526](https://github.com/thanos-io/thanos/pull/2526) Compact: In case there are no labels left after deduplication via `--deduplication.replica-label`, assign first `replica-label` with value `deduped`.
211211
- [#2621](https://github.com/thanos-io/thanos/pull/2621) Receive: Added flag to configure forward request timeout. Receive write will complete request as soon as quorum of writes succeeds.
212+
- [#3380](https://github.com/thanos-io/thanos/pull/3380) mixin: add deletion metrics for compactor.
212213

213214
### Changed
214215

‎examples/dashboards/compact.json

+252-9
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@
616616
"dashLength": 10,
617617
"dashes": false,
618618
"datasource": "$datasource",
619-
"description": "Shows rate of execution for all meta files from blocks in the bucket into the memory.",
619+
"description": "Shows deletion rate of blocks already marked for deletion.",
620620
"fill": 10,
621621
"id": 8,
622622
"legend": {
@@ -641,6 +641,249 @@
641641
"span": 4,
642642
"stack": true,
643643
"steppedLine": false,
644+
"targets": [
645+
{
646+
"expr": "sum(rate(thanos_compact_blocks_cleaned_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
647+
"format": "time_series",
648+
"intervalFactor": 2,
649+
"legendFormat": "Blocks cleanup {{job}}",
650+
"legendLink": null,
651+
"step": 10
652+
}
653+
],
654+
"thresholds": [ ],
655+
"timeFrom": null,
656+
"timeShift": null,
657+
"title": "Deletion Rate",
658+
"tooltip": {
659+
"shared": false,
660+
"sort": 0,
661+
"value_type": "individual"
662+
},
663+
"type": "graph",
664+
"xaxis": {
665+
"buckets": null,
666+
"mode": "time",
667+
"name": null,
668+
"show": true,
669+
"values": [ ]
670+
},
671+
"yaxes": [
672+
{
673+
"format": "short",
674+
"label": null,
675+
"logBase": 1,
676+
"max": null,
677+
"min": 0,
678+
"show": true
679+
},
680+
{
681+
"format": "short",
682+
"label": null,
683+
"logBase": 1,
684+
"max": null,
685+
"min": null,
686+
"show": false
687+
}
688+
]
689+
},
690+
{
691+
"aliasColors": { },
692+
"bars": false,
693+
"dashLength": 10,
694+
"dashes": false,
695+
"datasource": "$datasource",
696+
"description": "Shows deletion failures rate of blocks already marked for deletion.",
697+
"fill": 1,
698+
"id": 9,
699+
"legend": {
700+
"avg": false,
701+
"current": false,
702+
"max": false,
703+
"min": false,
704+
"show": true,
705+
"total": false,
706+
"values": false
707+
},
708+
"lines": true,
709+
"linewidth": 1,
710+
"links": [ ],
711+
"nullPointMode": "null as zero",
712+
"percentage": false,
713+
"pointradius": 5,
714+
"points": false,
715+
"renderer": "flot",
716+
"seriesOverrides": [ ],
717+
"spaceLength": 10,
718+
"span": 4,
719+
"stack": false,
720+
"steppedLine": false,
721+
"targets": [
722+
{
723+
"expr": "sum(rate(thanos_compact_block_cleanup_failures_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
724+
"format": "time_series",
725+
"intervalFactor": 2,
726+
"legendFormat": "Blocks cleanup failures {{job}}",
727+
"legendLink": null,
728+
"step": 10
729+
}
730+
],
731+
"thresholds": [ ],
732+
"timeFrom": null,
733+
"timeShift": null,
734+
"title": "Deletion Error Rate",
735+
"tooltip": {
736+
"shared": false,
737+
"sort": 0,
738+
"value_type": "individual"
739+
},
740+
"type": "graph",
741+
"xaxis": {
742+
"buckets": null,
743+
"mode": "time",
744+
"name": null,
745+
"show": true,
746+
"values": [ ]
747+
},
748+
"yaxes": [
749+
{
750+
"format": "short",
751+
"label": null,
752+
"logBase": 1,
753+
"max": null,
754+
"min": 0,
755+
"show": true
756+
},
757+
{
758+
"format": "short",
759+
"label": null,
760+
"logBase": 1,
761+
"max": null,
762+
"min": null,
763+
"show": false
764+
}
765+
]
766+
},
767+
{
768+
"aliasColors": { },
769+
"bars": false,
770+
"dashLength": 10,
771+
"dashes": false,
772+
"datasource": "$datasource",
773+
"description": "Shows rate at which blocks are marked for deletion (from GC and retention policy).",
774+
"fill": 1,
775+
"id": 10,
776+
"legend": {
777+
"avg": false,
778+
"current": false,
779+
"max": false,
780+
"min": false,
781+
"show": true,
782+
"total": false,
783+
"values": false
784+
},
785+
"lines": true,
786+
"linewidth": 1,
787+
"links": [ ],
788+
"nullPointMode": "null as zero",
789+
"percentage": false,
790+
"pointradius": 5,
791+
"points": false,
792+
"renderer": "flot",
793+
"seriesOverrides": [ ],
794+
"spaceLength": 10,
795+
"span": 4,
796+
"stack": false,
797+
"steppedLine": false,
798+
"targets": [
799+
{
800+
"expr": "sum(rate(thanos_compact_blocks_marked_for_deletion_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
801+
"format": "time_series",
802+
"intervalFactor": 2,
803+
"legendFormat": "Blocks marked {{job}}",
804+
"legendLink": null,
805+
"step": 10
806+
}
807+
],
808+
"thresholds": [ ],
809+
"timeFrom": null,
810+
"timeShift": null,
811+
"title": "Marking Rate",
812+
"tooltip": {
813+
"shared": false,
814+
"sort": 0,
815+
"value_type": "individual"
816+
},
817+
"type": "graph",
818+
"xaxis": {
819+
"buckets": null,
820+
"mode": "time",
821+
"name": null,
822+
"show": true,
823+
"values": [ ]
824+
},
825+
"yaxes": [
826+
{
827+
"format": "short",
828+
"label": null,
829+
"logBase": 1,
830+
"max": null,
831+
"min": 0,
832+
"show": true
833+
},
834+
{
835+
"format": "short",
836+
"label": null,
837+
"logBase": 1,
838+
"max": null,
839+
"min": null,
840+
"show": false
841+
}
842+
]
843+
}
844+
],
845+
"repeat": null,
846+
"repeatIteration": null,
847+
"repeatRowId": null,
848+
"showTitle": true,
849+
"title": "Blocks deletion",
850+
"titleSize": "h6"
851+
},
852+
{
853+
"collapse": false,
854+
"height": "250px",
855+
"panels": [
856+
{
857+
"aliasColors": { },
858+
"bars": false,
859+
"dashLength": 10,
860+
"dashes": false,
861+
"datasource": "$datasource",
862+
"description": "Shows rate of execution for all meta files from blocks in the bucket into the memory.",
863+
"fill": 10,
864+
"id": 11,
865+
"legend": {
866+
"avg": false,
867+
"current": false,
868+
"max": false,
869+
"min": false,
870+
"show": true,
871+
"total": false,
872+
"values": false
873+
},
874+
"lines": true,
875+
"linewidth": 0,
876+
"links": [ ],
877+
"nullPointMode": "null as zero",
878+
"percentage": false,
879+
"pointradius": 5,
880+
"points": false,
881+
"renderer": "flot",
882+
"seriesOverrides": [ ],
883+
"spaceLength": 10,
884+
"span": 4,
885+
"stack": true,
886+
"steppedLine": false,
644887
"targets": [
645888
{
646889
"expr": "sum(rate(thanos_blocks_meta_syncs_total{namespace=\"$namespace\",job=~\"$job\"}[$interval])) by (job)",
@@ -697,7 +940,7 @@
697940
"datasource": "$datasource",
698941
"description": "Shows ratio of errors compared to the total number of executed meta file sync.",
699942
"fill": 10,
700-
"id": 9,
943+
"id": 12,
701944
"legend": {
702945
"avg": false,
703946
"current": false,
@@ -774,7 +1017,7 @@
7741017
"datasource": "$datasource",
7751018
"description": "Shows how long has it taken to execute meta file sync, in quantiles.",
7761019
"fill": 1,
777-
"id": 10,
1020+
"id": 13,
7781021
"legend": {
7791022
"avg": false,
7801023
"current": false,
@@ -879,7 +1122,7 @@
8791122
"datasource": "$datasource",
8801123
"description": "Shows rate of execution for operations against the bucket.",
8811124
"fill": 10,
882-
"id": 11,
1125+
"id": 14,
8831126
"legend": {
8841127
"avg": false,
8851128
"current": false,
@@ -958,7 +1201,7 @@
9581201
"datasource": "$datasource",
9591202
"description": "Shows ratio of errors compared to the total number of executed operations against the bucket.",
9601203
"fill": 10,
961-
"id": 12,
1204+
"id": 15,
9621205
"legend": {
9631206
"avg": false,
9641207
"current": false,
@@ -1035,7 +1278,7 @@
10351278
"datasource": "$datasource",
10361279
"description": "Shows how long has it taken to execute operations against the bucket, in quantiles.",
10371280
"fill": 1,
1038-
"id": 13,
1281+
"id": 16,
10391282
"legend": {
10401283
"avg": false,
10411284
"current": false,
@@ -1139,7 +1382,7 @@
11391382
"dashes": false,
11401383
"datasource": "$datasource",
11411384
"fill": 1,
1142-
"id": 14,
1385+
"id": 17,
11431386
"legend": {
11441387
"avg": false,
11451388
"current": false,
@@ -1255,7 +1498,7 @@
12551498
"dashes": false,
12561499
"datasource": "$datasource",
12571500
"fill": 1,
1258-
"id": 15,
1501+
"id": 18,
12591502
"legend": {
12601503
"avg": false,
12611504
"current": false,
@@ -1331,7 +1574,7 @@
13311574
"dashes": false,
13321575
"datasource": "$datasource",
13331576
"fill": 1,
1334-
"id": 16,
1577+
"id": 19,
13351578
"legend": {
13361579
"avg": false,
13371580
"current": false,

‎mixin/dashboards/compact.libsonnet

+34
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,40 @@ local g = import '../lib/thanos-grafana-builder/builder.libsonnet';
8080
g.latencyPanel('thanos_compact_garbage_collection_duration_seconds', 'namespace="$namespace",job=~"$job"')
8181
)
8282
)
83+
.addRow(
84+
g.row('Blocks deletion')
85+
.addPanel(
86+
g.panel(
87+
'Deletion Rate',
88+
'Shows deletion rate of blocks already marked for deletion.'
89+
) +
90+
g.queryPanel(
91+
'sum(rate(thanos_compact_blocks_cleaned_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)',
92+
'Blocks cleanup {{job}}'
93+
) +
94+
g.stack
95+
)
96+
.addPanel(
97+
g.panel(
98+
'Deletion Error Rate',
99+
'Shows deletion failures rate of blocks already marked for deletion.'
100+
) +
101+
g.queryPanel(
102+
'sum(rate(thanos_compact_block_cleanup_failures_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)',
103+
'Blocks cleanup failures {{job}}'
104+
)
105+
)
106+
.addPanel(
107+
g.panel(
108+
'Marking Rate',
109+
'Shows rate at which blocks are marked for deletion (from GC and retention policy).'
110+
) +
111+
g.queryPanel(
112+
'sum(rate(thanos_compact_blocks_marked_for_deletion_total{namespace="$namespace",job=~"$job"}[$interval])) by (job)',
113+
'Blocks marked {{job}}'
114+
)
115+
)
116+
)
83117
.addRow(
84118
g.row('Sync Meta')
85119
.addPanel(

0 commit comments

Comments
 (0)