Skip to content
This repository was archived by the owner on Mar 3, 2023. It is now read-only.

Commit 01abf01

Browse files
authored
Add Prometheus metrics rules (#3651)
1 parent a928cd5 commit 01abf01

File tree

12 files changed

+1579
-7
lines changed

12 files changed

+1579
-7
lines changed

heron/config/src/yaml/conf/aurora/metrics_sinks.yaml

+131
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,137 @@ metricscache-sink:
114114
# include-topology-name: true # Include topology name in metric name (default false)
115115
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
116116
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
117+
# rules:
118+
# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
119+
# - pattern: __jvm-(.+)/(.+)
120+
# name: jvm_$1_$2
121+
# attrNameSnakeCase: true
122+
# type: COUNTER
123+
# # "__execute-time-ns/pulsar-prod-4/default": "418764",
124+
# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
125+
# name: $1
126+
# attrNameSnakeCase: true
127+
# type: COUNTER
128+
# labels:
129+
# dest_component: "$2"
130+
# context: "$3"
131+
# # "__execute-time-ns/pulsar-prod-4": "418764",
132+
# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
133+
# name: $1
134+
# attrNameSnakeCase: true
135+
# type: COUNTER
136+
# labels:
137+
# context: "$2"
138+
# # StreamManager
139+
# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
140+
# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
141+
# name: $2
142+
# attrNameSnakeCase: true
143+
# type: COUNTER
144+
# labels:
145+
# dest_component: "$1"
146+
# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
147+
# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
148+
# name: $1_$4
149+
# attrNameSnakeCase: true
150+
# type: COUNTER
151+
# labels:
152+
# dest_container: "$2"
153+
# dest_task: "$3"
154+
# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
155+
# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
156+
# name: $1
157+
# attrNameSnakeCase: true
158+
# type: COUNTER
159+
# labels:
160+
# dest_container: "$2"
161+
# dest_task: "$3"
162+
# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
163+
# - pattern: PulsarSpout/(.+)/(.+)
164+
# name: pulsar_spout_$2
165+
# attrNameSnakeCase: true
166+
# type: COUNTER
167+
# labels:
168+
# component: "$1"
169+
# - pattern: PulsarBolt/(.+)/(.+)
170+
# name: pulsar_bolt_$2
171+
# attrNameSnakeCase: true
172+
# type: COUNTER
173+
# labels:
174+
# component: "$1"
175+
# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
176+
# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
177+
# name: kafka_consumer_$2_$1
178+
# attrNameSnakeCase: true
179+
# type: COUNTER
180+
# labels:
181+
# client_id: "$3"
182+
# node_id: "$4"
183+
# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
184+
# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
185+
# name: kafka_consumer_$2_$1
186+
# attrNameSnakeCase: true
187+
# type: COUNTER
188+
# labels:
189+
# client_id: "$3"
190+
# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
191+
# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
192+
# name: kafka_consumer_$2_$1
193+
# attrNameSnakeCase: true
194+
# type: COUNTER
195+
# labels:
196+
# client_id: "$3"
197+
# topic: "$4"
198+
# partition: "$5"
199+
# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
200+
# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
201+
# name: kafka_consumer_$2_$1
202+
# attrNameSnakeCase: true
203+
# type: COUNTER
204+
# labels:
205+
# client_id: "$3"
206+
# topic: "$4"
207+
# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
208+
# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
209+
# name: kafka_consumer_$2_$1
210+
# attrNameSnakeCase: true
211+
# type: COUNTER
212+
# labels:
213+
# client_id: "$3"
214+
# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
215+
# name: kafka_consumer_$1
216+
# attrNameSnakeCase: true
217+
# type: COUNTER
218+
# labels:
219+
# client_id: "$2"
220+
# node_id: "$3"
221+
# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
222+
# name: kafka_consumer_$1
223+
# attrNameSnakeCase: true
224+
# type: COUNTER
225+
# labels:
226+
# client_id: "$2"
227+
# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
228+
# name: kafka_consumer_$1
229+
# attrNameSnakeCase: true
230+
# type: COUNTER
231+
# labels:
232+
# client_id: "$2"
233+
# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
234+
# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
235+
# name: kafka_offset_$3
236+
# attrNameSnakeCase: true
237+
# type: COUNTER
238+
# labels:
239+
# topic: "$1"
240+
# partition: "$2"
241+
# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
242+
# - pattern: kafkaOffset/(.+)/(.+)
243+
# name: kafka_offset_$2
244+
# attrNameSnakeCase: true
245+
# type: COUNTER
246+
# labels:
247+
# topic: "$1"
117248

118249
### Config for graphite-sink
119250
### Currently the graphite-sink is disabled

heron/config/src/yaml/conf/examples/metrics_sinks.yaml

+131
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,137 @@ metricscache-sink:
114114
# include-topology-name: true # Include topology name in metric name (default false)
115115
# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000)
116116
# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600)
117+
# rules:
118+
# # __jvm-peak-usage/G1-Survivor-Space-committed": "9",
119+
# - pattern: __jvm-(.+)/(.+)
120+
# name: jvm_$1_$2
121+
# attrNameSnakeCase: true
122+
# type: COUNTER
123+
# # "__execute-time-ns/pulsar-prod-4/default": "418764",
124+
# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+)
125+
# name: $1
126+
# attrNameSnakeCase: true
127+
# type: COUNTER
128+
# labels:
129+
# dest_component: "$2"
130+
# context: "$3"
131+
# # "__execute-time-ns/pulsar-prod-4": "418764",
132+
# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)
133+
# name: $1
134+
# attrNameSnakeCase: true
135+
# type: COUNTER
136+
# labels:
137+
# context: "$2"
138+
# # StreamManager
139+
# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039",
140+
# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs)
141+
# name: $2
142+
# attrNameSnakeCase: true
143+
# type: COUNTER
144+
# labels:
145+
# dest_component: "$1"
146+
# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000",
147+
# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+)
148+
# name: $1_$4
149+
# attrNameSnakeCase: true
150+
# type: COUNTER
151+
# labels:
152+
# dest_container: "$2"
153+
# dest_task: "$3"
154+
# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0",
155+
# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+)
156+
# name: $1
157+
# attrNameSnakeCase: true
158+
# type: COUNTER
159+
# labels:
160+
# dest_container: "$2"
161+
# dest_task: "$3"
162+
# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput'
163+
# - pattern: PulsarSpout/(.+)/(.+)
164+
# name: pulsar_spout_$2
165+
# attrNameSnakeCase: true
166+
# type: COUNTER
167+
# labels:
168+
# component: "$1"
169+
# - pattern: PulsarBolt/(.+)/(.+)
170+
# name: pulsar_bolt_$2
171+
# attrNameSnakeCase: true
172+
# type: COUNTER
173+
# labels:
174+
# component: "$1"
175+
# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1"
176+
# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+)
177+
# name: kafka_consumer_$2_$1
178+
# attrNameSnakeCase: true
179+
# type: COUNTER
180+
# labels:
181+
# client_id: "$3"
182+
# node_id: "$4"
183+
# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout"
184+
# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+)
185+
# name: kafka_consumer_$2_$1
186+
# attrNameSnakeCase: true
187+
# type: COUNTER
188+
# labels:
189+
# client_id: "$3"
190+
# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1"
191+
# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+)
192+
# name: kafka_consumer_$2_$1
193+
# attrNameSnakeCase: true
194+
# type: COUNTER
195+
# labels:
196+
# client_id: "$3"
197+
# topic: "$4"
198+
# partition: "$5"
199+
# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api"
200+
# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)
201+
# name: kafka_consumer_$2_$1
202+
# attrNameSnakeCase: true
203+
# type: COUNTER
204+
# labels:
205+
# client_id: "$3"
206+
# topic: "$4"
207+
# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1"
208+
# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+)
209+
# name: kafka_consumer_$2_$1
210+
# attrNameSnakeCase: true
211+
# type: COUNTER
212+
# labels:
213+
# client_id: "$3"
214+
# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+)
215+
# name: kafka_consumer_$1
216+
# attrNameSnakeCase: true
217+
# type: COUNTER
218+
# labels:
219+
# client_id: "$2"
220+
# node_id: "$3"
221+
# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)
222+
# name: kafka_consumer_$1
223+
# attrNameSnakeCase: true
224+
# type: COUNTER
225+
# labels:
226+
# client_id: "$2"
227+
# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+)
228+
# name: kafka_consumer_$1
229+
# attrNameSnakeCase: true
230+
# type: COUNTER
231+
# labels:
232+
# client_id: "$2"
233+
# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag'
234+
# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+)
235+
# name: kafka_offset_$3
236+
# attrNameSnakeCase: true
237+
# type: COUNTER
238+
# labels:
239+
# topic: "$1"
240+
# partition: "$2"
241+
# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag'
242+
# - pattern: kafkaOffset/(.+)/(.+)
243+
# name: kafka_offset_$2
244+
# attrNameSnakeCase: true
245+
# type: COUNTER
246+
# labels:
247+
# topic: "$1"
117248

118249
### Config for graphite-sink
119250
### Currently the graphite-sink is disabled

0 commit comments

Comments
 (0)