Skip to content

Commit 79d84e2

Browse files
srodenhuissrodenhuis
authored
feat: one prometheus (#1724)
1 parent 3c89f34 commit 79d84e2

15 files changed

+169
-272
lines changed

‎charts/team-ns/templates/istio-sidecar.yaml

+4-31
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{{- $v := .Values | merge (dict) }}
22
{{- $ := . }}
3-
{{- $prometheus := dig "managedMonitoring" "prometheus" false $v }}
3+
{{- $alertmanager := dig "managedMonitoring" "alertmanager" false $v }}
44
{{- if not (eq $v.teamId "admin") }}
55
{{- $egressFilteringEnabled := $v | dig "networkPolicy" "egressPublic" true }}
66
{{- if $egressFilteringEnabled }}
@@ -13,46 +13,19 @@ metadata:
1313
spec:
1414
outboundTrafficPolicy:
1515
mode: REGISTRY_ONLY
16-
{{- if $prometheus }}
16+
{{- if $alertmanager }}
1717
---
1818
apiVersion: networking.istio.io/v1beta1
1919
kind: Sidecar
2020
metadata:
21-
name: team-prometheus
21+
name: team-alertmanager
2222
labels: {{- include "team-ns.chart-labels" $ | nindent 4 }}
2323
spec:
2424
outboundTrafficPolicy:
2525
mode: ALLOW_ANY
2626
workloadSelector:
2727
labels:
28-
otomi.io/app: prometheus-team-{{ $v.teamId }}
28+
app.kubernetes.io/instance: po-prometheus
2929
{{- end }}
3030
{{- end }}
31-
{{- end }}
32-
{{- if $prometheus }}
33-
---
34-
apiVersion: monitoring.coreos.com/v1
35-
kind: PodMonitor
36-
metadata:
37-
labels:
38-
{{- include "team-ns.chart-labels" $ | nindent 4 }}
39-
prometheus: team-{{ $v.teamId }}
40-
name: istio-sidecars-team-services
41-
spec:
42-
namespaceSelector:
43-
matchNames: [team-{{ $v.teamId }}]
44-
podMetricsEndpoints:
45-
- path: /stats/prometheus
46-
port: http-envoy-prom
47-
selector:
48-
matchLabels:
49-
security.istio.io/tlsMode: istio
50-
matchExpressions:
51-
- key: app.kubernetes.io/instance
52-
operator: NotIn
53-
values:
54-
- prometheus-{{ $v.teamId }}
55-
- {{ $v.teamId }}-po-prometheus
56-
- {{ $v.teamId }}-po-alertmanager
57-
- tekton-dashboard-{{ $v.teamId }}
5831
{{- end }}

‎charts/team-ns/templates/netpols/default-network-policies.yaml

+1-96
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
{{/* Below merge is a workaround for: https://github.com/helm/helm/issues/9266 */}}
22
{{- $v := .Values | merge (dict) }}
3-
{{- $prometheus := dig "managedMonitoring" "prometheus" false $v }}
4-
{{- $alertmng := dig "managedMonitoring" "alertmanager" false $v }}
53
{{- if (not (dig "networkPolicy" "ingressPrivate" true $v)) }}
64
---
75
# If team network policies are disabled then we whitelist all traffic to prevent undesired blocking while deploying team workloads
@@ -28,6 +26,7 @@ spec:
2826
policyTypes:
2927
- Ingress
3028
---
29+
# Allow traffic from platform services
3130
apiVersion: networking.k8s.io/v1
3231
kind: NetworkPolicy
3332
metadata:
@@ -61,79 +60,6 @@ spec:
6160
- namespaceSelector:
6261
matchLabels:
6362
name: tekton-pipelines
64-
{{- if $alertmng }}
65-
---
66-
# Allow traffic from team's prometheus to team's alertmanager
67-
apiVersion: networking.k8s.io/v1
68-
kind: NetworkPolicy
69-
metadata:
70-
name: default-to-alertmanager
71-
labels: {{- include "team-ns.chart-labels" $ | nindent 4 }}
72-
spec:
73-
ingress:
74-
- from:
75-
- namespaceSelector:
76-
matchLabels:
77-
name: team-{{ $v.teamId }}
78-
podSelector:
79-
matchLabels:
80-
app.kubernetes.io/instance: {{ $v.teamId }}-po-prometheus
81-
podSelector:
82-
matchLabels:
83-
app.kubernetes.io/instance: {{ $v.teamId }}-po-alertmanager
84-
policyTypes:
85-
- Ingress
86-
{{- end }}
87-
{{- if $prometheus }}
88-
---
89-
# Allow traffic from Alertmanager and Grafana to Prometheus
90-
apiVersion: networking.k8s.io/v1
91-
kind: NetworkPolicy
92-
metadata:
93-
name: default-to-prometheus
94-
labels: {{- include "team-ns.chart-labels" $ | nindent 4 }}
95-
spec:
96-
ingress:
97-
- from:
98-
- namespaceSelector:
99-
matchLabels:
100-
name: team-{{ $v.teamId }}
101-
podSelector:
102-
matchLabels:
103-
app.kubernetes.io/instance: {{ $v.teamId }}-po-alertmanager
104-
- namespaceSelector:
105-
matchLabels:
106-
name: team-{{ $v.teamId }}
107-
podSelector:
108-
matchLabels:
109-
app.kubernetes.io/name: {{ $v.teamId }}-po-grafana
110-
podSelector:
111-
matchLabels:
112-
app.kubernetes.io/instance: {{ $v.teamId }}-po-prometheus
113-
policyTypes:
114-
- Ingress
115-
---
116-
# Allow traffic from Prometheus to kube-state-metrics
117-
apiVersion: networking.k8s.io/v1
118-
kind: NetworkPolicy
119-
metadata:
120-
name: default-to-kube-state-metrics
121-
labels: {{- include "team-ns.chart-labels" $ | nindent 4 }}
122-
spec:
123-
ingress:
124-
- from:
125-
- namespaceSelector:
126-
matchLabels:
127-
name: team-{{ $v.teamId }}
128-
podSelector:
129-
matchLabels:
130-
prometheus: team-{{ $v.teamId }}
131-
podSelector:
132-
matchLabels:
133-
app.kubernetes.io/name: kube-state-metrics
134-
policyTypes:
135-
- Ingress
136-
{{- end }}
13763
---
13864
# Allow webhook traffic from gitea to event listeners
13965
apiVersion: networking.k8s.io/v1
@@ -155,25 +81,4 @@ spec:
15581
app.kubernetes.io/managed-by: EventListener
15682
policyTypes:
15783
- Ingress
158-
{{- if $prometheus }}
159-
---
160-
# Allow traffic from Prometheus to all pods for scraping metrics
161-
apiVersion: networking.k8s.io/v1
162-
kind: NetworkPolicy
163-
metadata:
164-
name: default-from-prometheus
165-
labels: {{- include "team-ns.chart-labels" $ | nindent 4 }}
166-
spec:
167-
podSelector: {}
168-
policyTypes:
169-
- Ingress
170-
ingress:
171-
- from:
172-
- namespaceSelector:
173-
matchLabels:
174-
name: team-{{ $v.teamId }}
175-
podSelector:
176-
matchLabels:
177-
app.kubernetes.io/instance: {{ $v.teamId }}-po-prometheus
178-
{{- end }}
17984
{{- end }}

‎charts/team-ns/templates/servicemonitors/service-monitors.yaml

-21
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
{{- $v := .Values | merge (dict) }}
2-
{{- $prometheus := dig "managedMonitoring" "prometheus" false $v }}
32
{{- $alertmng := dig "managedMonitoring" "alertmanager" false $v }}
43
{{- $grafana := dig "managedMonitoring" "grafana" false $v }}
54
{{- if not (eq $v.teamId "admin") }}
@@ -27,26 +26,6 @@ spec:
2726
release: prometheus-{{ $v.teamId }}
2827
{{- end }}
2928
---
30-
{{- if $prometheus }}
31-
apiVersion: monitoring.coreos.com/v1
32-
kind: ServiceMonitor
33-
metadata:
34-
labels:
35-
prometheus: system
36-
name: po-prometheus-team-{{ $v.teamId }}
37-
spec:
38-
endpoints:
39-
- path: /metrics
40-
port: http-web
41-
namespaceSelector:
42-
matchNames:
43-
- team-{{ $v.teamId }}
44-
selector:
45-
matchLabels:
46-
app: {{ $v.teamId }}-po-prometheus
47-
release: prometheus-{{ $v.teamId }}
48-
{{- end }}
49-
---
5029
{{- if $grafana }}
5130
apiVersion: monitoring.coreos.com/v1
5231
kind: ServiceMonitor

‎core.yaml

+1-8
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,7 @@ adminApps:
337337
teamApps:
338338
- name: alertmanager
339339
ownHost: true
340+
path: /#/alerts?silenced=false&inhibited=false&active=true&filter=%7Bnamespace%3D"team-#TEAM#"%7D
340341
ingress:
341342
- svc: po-alertmanager
342343
hasPrefix: true
@@ -357,14 +358,6 @@ teamApps:
357358
- name: loki
358359
useHost: grafana
359360
path: /explore?orgId=1&left=%7B"datasource":"loki","queries":%5B%7B"refId":"A","expr":"","queryType":"range","datasource":%7B"type":"loki","uid":"loki"%7D%7D%5D,"range":%7B"from":"now-1h","to":"now"%7D%7D
360-
- name: prometheus
361-
ownHost: true
362-
ingress:
363-
- svc: po-prometheus
364-
hasPrefix: true
365-
port: 9090
366-
type: public
367-
auth: true
368361
- name: tekton
369362
ownHost: true
370363
ingress:

‎helmfile.d/helmfile-60.teams.yaml

+10-41
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ releases:
4040
values:
4141
- ../values/tekton-dashboard/tekton-dashboard-teams.gotmpl
4242
- name: prometheus-{{ $teamId }}
43-
installed: {{ or ($team | get "managedMonitoring.grafana" false) ($team | get "managedMonitoring.prometheus" false) ($team | get "managedMonitoring.alertmanager" false) }}
43+
installed: {{ or ($team | get "managedMonitoring.grafana" false) ($team | get "managedMonitoring.alertmanager" false) }}
4444
namespace: team-{{ $teamId }}
4545
chart: ../charts/kube-prometheus-stack
4646
labels:
@@ -61,47 +61,17 @@ releases:
6161
annotations:
6262
sidecar.istio.io/inject: "true"
6363
labels:
64-
prometheus: team-{{ $teamId }}
64+
prometheus: system
6565
# to do: load slackTpl and opsgenieTpl only if alerts.receicers = true
66-
config: {{- tpl (readFile "../helmfile.d/snippets/alertmanager.gotmpl") (dict "instance" $team "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 12 }}
66+
config: {{- tpl (readFile "../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $team "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | nindent 12 }}
6767
defaultRules:
68-
appNamespacesTarget: team-{{ $teamId }}
6968
rules:
70-
general: {{ $team | get "managedMonitoring.prometheus" false }}
69+
general: false
7170
commonLabels:
7271
prometheus: team-{{ $teamId }}
7372
prometheus:
74-
enabled: {{ $team | get "managedMonitoring.prometheus" false }}
75-
namespaceOverride: null # team-{{ $teamId }}
76-
prometheusSpec:
77-
podMetadata:
78-
annotations:
79-
traffic.sidecar.istio.io/excludeOutboundPorts: "9093"
80-
labels:
81-
otomi.io/app: prometheus-team-{{ $teamId }}
82-
externalLabels:
83-
cluster: "prometheus-{{ $teamId }}.{{ $domain }}"
84-
externalUrl: "https://prometheus-{{ $teamId }}.{{ $domain }}"
85-
{{- range $selType := list "podMonitor" "probe" "rule" "serviceMonitor" }}
86-
{{ $selType }}NamespaceSelector:
87-
matchExpressions:
88-
- key: name
89-
operator: In
90-
values:
91-
- team-{{ $teamId }}
92-
{{ $selType }}Selector:
93-
matchLabels:
94-
prometheus: team-{{ $teamId }}
95-
{{- end }}
96-
{{- if gt (len .services) 0 }}
97-
additionalScrapeConfigs:
98-
{{- tpl (readFile "../helmfile.d/snippets/blackbox-targets.gotmpl") (dict "teamId" $teamId "namespace" (printf "team-%s" $teamId) "services" $teamServices "domain" $domain) | nindent 12 }}
99-
{{- end }}
100-
{{- if $team | get "managedMonitoring.prometheus" false }}
101-
additionalPrometheusRules:
102-
- name: blackbox
103-
{{- readFile "../values/prometheus-operator/rules/blackbox.yaml" | nindent 12 }}
104-
{{- end }}
73+
enabled: false
74+
prometheusSpec: {}
10575
grafana:
10676
enabled: {{ $team | get "managedMonitoring.grafana" false }}
10777
namespaceOverride: null # team-{{ $teamId }}
@@ -114,8 +84,7 @@ releases:
11484
root_url: https://grafana-{{ $teamId }}.{{ $domain }}
11585
sidecar:
11686
datasources:
117-
defaultDatasourceEnabled: {{ $team | get "managedMonitoring.prometheus" false }}
118-
uid: Prometheus-team
87+
defaultDatasourceEnabled: false
11988
dashboards:
12089
enabled: true
12190
label: release
@@ -124,6 +93,7 @@ releases:
12493
- name: Prometheus-platform
12594
editable: false
12695
uid: prometheus-platform
96+
isDefault: true
12797
type: prometheus
12898
access: proxy
12999
url: http://po-prometheus.monitoring:9090
@@ -186,7 +156,7 @@ releases:
186156
datasourceUid: 'loki'
187157
{{- if has "msteams" ($team | get "alerts.receivers" list) }}
188158
- name: prometheus-msteams-{{ $teamId }}
189-
installed: {{ $team | get "managedMonitoring.prometheus" false }}
159+
installed: {{ $team | get "managedMonitoring.alertmanager" false }}
190160
namespace: team-{{ $teamId }}
191161
chart: ../charts/prometheus-msteams
192162
labels:
@@ -227,7 +197,6 @@ releases:
227197
{{- if $v.apps.trivy.enabled }}
228198
- trivy-teams
229199
{{- end }}
230-
231200
- name: team-ns-{{ $teamId }}
232201
installed: true
233202
namespace: team-{{ $teamId }}
@@ -249,7 +218,7 @@ releases:
249218
teamId: {{ $teamId }}
250219
teamIds: {{- toYaml (keys $v.teamConfig) | nindent 10 }}
251220
- services: {{- concat $coreTeamServices $teamServices | toYaml | nindent 10 }}
252-
{{- if and ($team | get "managedMonitoring.prometheus" false) (gt (len $teamServices) 0) }}
221+
{{- if (gt (len $teamServices) 0) }}
253222
- name: blackbox
254223
svc: prometheus-blackbox-exporter
255224
port: 9115

0 commit comments

Comments
 (0)