Skip to content

Commit a41a0e2

Browse files
branczbrancz
committed
Change kube-controllers targets to separated scheduler and controller-manager
1 parent b9e6d16 commit a41a0e2

13 files changed

+273
-162
lines changed

‎assets/prometheus-k8s/kube-controllers-service.yaml

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎assets/prometheus-k8s/role-binding-specific-namespaces.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,30 @@ items:
6565
- kind: ServiceAccount
6666
name: prometheus-k8s
6767
namespace: openshift-monitoring
68+
- apiVersion: rbac.authorization.k8s.io/v1
69+
kind: RoleBinding
70+
metadata:
71+
name: prometheus-k8s
72+
namespace: openshift-kube-scheduler
73+
roleRef:
74+
apiGroup: rbac.authorization.k8s.io
75+
kind: Role
76+
name: prometheus-k8s
77+
subjects:
78+
- kind: ServiceAccount
79+
name: prometheus-k8s
80+
namespace: openshift-monitoring
81+
- apiVersion: rbac.authorization.k8s.io/v1
82+
kind: RoleBinding
83+
metadata:
84+
name: prometheus-k8s
85+
namespace: openshift-kube-controller-manager
86+
roleRef:
87+
apiGroup: rbac.authorization.k8s.io
88+
kind: Role
89+
name: prometheus-k8s
90+
subjects:
91+
- kind: ServiceAccount
92+
name: prometheus-k8s
93+
namespace: openshift-monitoring
6894
kind: RoleBindingList

‎assets/prometheus-k8s/role-specific-namespaces.yaml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,4 +85,38 @@ items:
8585
- get
8686
- list
8787
- watch
88+
- apiVersion: rbac.authorization.k8s.io/v1
89+
kind: Role
90+
metadata:
91+
name: prometheus-k8s
92+
namespace: openshift-kube-scheduler
93+
rules:
94+
- apiGroups:
95+
- ""
96+
resources:
97+
- nodes
98+
- services
99+
- endpoints
100+
- pods
101+
verbs:
102+
- get
103+
- list
104+
- watch
105+
- apiVersion: rbac.authorization.k8s.io/v1
106+
kind: Role
107+
metadata:
108+
name: prometheus-k8s
109+
namespace: openshift-kube-controller-manager
110+
rules:
111+
- apiGroups:
112+
- ""
113+
resources:
114+
- nodes
115+
- services
116+
- endpoints
117+
- pods
118+
verbs:
119+
- get
120+
- list
121+
- watch
88122
kind: RoleList

‎assets/prometheus-k8s/service-monitor-kube-controller-manager.yaml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: ServiceMonitor
3+
metadata:
4+
labels:
5+
k8s-app: kube-controller-manager
6+
name: kube-controller-manager
7+
namespace: openshift-monitoring
8+
spec:
9+
endpoints:
10+
- interval: 30s
11+
metricRelabelings:
12+
- action: drop
13+
regex: etcd_(debugging|disk|request|server).*
14+
sourceLabels:
15+
- __name__
16+
port: https
17+
relabelings:
18+
- action: replace
19+
regex: (.+)(?::\d+)
20+
replacement: $1:10252
21+
sourceLabels:
22+
- __address__
23+
targetLabel: __address__
24+
jobLabel: null
25+
namespaceSelector:
26+
matchNames:
27+
- openshift-kube-controller-manager
28+
selector: {}

‎assets/prometheus-k8s/service-monitor-kube-controllers.yaml

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎assets/prometheus-k8s/service-monitor-kube-scheduler.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: ServiceMonitor
3+
metadata:
4+
labels:
5+
k8s-app: kube-scheduler
6+
name: kube-scheduler
7+
namespace: openshift-monitoring
8+
spec:
9+
endpoints:
10+
- interval: 30s
11+
port: https
12+
relabelings:
13+
- action: replace
14+
regex: (.+)(?::\d+)
15+
replacement: $1:10251
16+
sourceLabels:
17+
- __address__
18+
targetLabel: __address__
19+
jobLabel: null
20+
namespaceSelector:
21+
matchNames:
22+
- openshift-kube-scheduler
23+
selector: {}

‎hack/build-jsonnet.sh

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,10 @@ done
3939
#
4040
# * CoreDNS is not used in OpenShift
4141
#
42-
# * kube-controller-manager and kube-scheduler don't exist in OpenShift, but
43-
# instead are grouped into the kube-scheduler component, which is separately
44-
# handled in `jsonnet/prometheus.jsonnet`.
4542

4643
rm -rf "assets/prometheus-operator/0alertmanager-custom-resource-definition.yaml"
4744
rm -rf "assets/prometheus-operator/0prometheus-custom-resource-definition.yaml"
4845
rm -rf "assets/prometheus-operator/0prometheusrule-custom-resource-definition.yaml"
4946
rm -rf "assets/prometheus-operator/0servicemonitor-custom-resource-definition.yaml"
5047
rm -rf "assets/prometheus-k8s/service-monitor-core-d-n-s.yaml"
51-
rm -rf "assets/prometheus-k8s/service-monitor-kube-controller-manager.yaml"
52-
rm -rf "assets/prometheus-k8s/service-monitor-kube-scheduler.yaml"
53-
rm -rf "assets/prometheus-k8s/service-monitor-kube-scheduler.yaml"
5448

‎jsonnet/main.jsonnet

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ local kp = (import 'kube-prometheus/kube-prometheus.libsonnet') +
4949
namespaces+: [
5050
'openshift-cluster-version',
5151
'openshift-apiserver',
52+
'openshift-kube-scheduler',
53+
'openshift-kube-controller-manager',
5254
],
5355
},
5456
},

‎jsonnet/prometheus.jsonnet

Lines changed: 88 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -104,59 +104,8 @@ local namespacesRole =
104104
clusterRole+:
105105
clusterRole.withRulesMixin([authenticationRole, authorizationRole, namespacesRole]),
106106

107-
// OpenShift currently has the kube-controller-manager and
108-
// kube-scheduler combined in one component called the
109-
// kube-controllers. This Service and ServiceMonitor enable scraping
110-
// its metrics.
111-
112-
kubeControllersService:
113-
local service = k.core.v1.service;
114-
local servicePort = k.core.v1.service.mixin.spec.portsType;
115-
116-
local kubeControllersPort = servicePort.newNamed('http-metrics', 8444, 8444);
117-
118-
service.new('kube-controllers', {
119-
'openshift.io/component': 'controllers',
120-
'openshift.io/control-plane': 'true',
121-
}, kubeControllersPort) +
122-
service.mixin.metadata.withNamespace('kube-system') +
123-
service.mixin.metadata.withLabels({ 'k8s-app': 'kube-controllers' }) +
124-
service.mixin.spec.withClusterIp('None'),
125-
126-
serviceMonitorKubeControllers:
127-
{
128-
apiVersion: 'monitoring.coreos.com/v1',
129-
kind: 'ServiceMonitor',
130-
metadata: {
131-
labels: {
132-
'k8s-app': 'kube-controllers',
133-
},
134-
name: 'kube-controllers',
135-
},
136-
spec: {
137-
endpoints: [
138-
{
139-
bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
140-
interval: '30s',
141-
port: 'http-metrics',
142-
scheme: 'https',
143-
tlsConfig: {
144-
caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
145-
},
146-
},
147-
],
148-
jobLabel: 'k8s-app',
149-
namespaceSelector: {
150-
matchNames: ['kube-system'],
151-
},
152-
selector: {
153-
matchLabels: {
154-
'k8s-app': 'kube-controllers',
155-
},
156-
},
157-
},
158-
},
159-
107+
// OpenShift has the kube-apiserver as well as an aggregated API called
108+
// OpenShift apiserver, containing all the extended APIs.
160109
serviceMonitorClusterVersionOperator:
161110
{
162111
apiVersion: 'monitoring.coreos.com/v1',
@@ -331,6 +280,92 @@ local namespacesRole =
331280
},
332281
},
333282

283+
// In OpenShift the kube-scheduler runs in its own namespace, and has a TLS
284+
// cert from the serving certs controller.
285+
286+
serviceMonitorKubeScheduler+:
287+
{
288+
spec+: {
289+
jobLabel: null,
290+
namespaceSelector: {
291+
matchNames: [
292+
'openshift-kube-scheduler',
293+
],
294+
},
295+
selector: {},
296+
endpoints:
297+
std.map(
298+
function(a) a {
299+
300+
//TODO(brancz): Once OpenShift is based on Kubernetes 1.12 the
301+
//scheduler will serve metrics on a secure port, then the below
302+
//commented out code is what we will need without the relabel
303+
//configs.
304+
305+
//bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
306+
interval: '30s',
307+
port: 'https',
308+
//scheme: 'https',
309+
//tlsConfig: {
310+
// caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
311+
// serverName: 'scheduler.openshift-kube-scheduler.svc',
312+
//},
313+
relabelings: [{
314+
sourceLabels: ['__address__'],
315+
action: 'replace',
316+
targetLabel: '__address__',
317+
regex: '(.+)(?::\\d+)',
318+
replacement: '$1:10251',
319+
}],
320+
},
321+
super.endpoints,
322+
),
323+
},
324+
},
325+
326+
// In OpenShift the kube-controller-manager runs in its own namespace, and
327+
// has a TLS cert from the serving certs controller.
328+
329+
serviceMonitorKubeControllerManager+:
330+
{
331+
spec+: {
332+
jobLabel: null,
333+
namespaceSelector: {
334+
matchNames: [
335+
'openshift-kube-controller-manager',
336+
],
337+
},
338+
selector: {},
339+
endpoints:
340+
std.map(
341+
function(a) a {
342+
343+
//TODO(brancz): Once OpenShift is based on Kubernetes 1.12 the
344+
//controller-manager will serve metrics on a secure port, then
345+
//the below commented out code is what we will need without the
346+
//relabel configs.
347+
348+
//bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
349+
interval: '30s',
350+
port: 'https',
351+
//scheme: 'https',
352+
//tlsConfig: {
353+
// caFile: '/etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt',
354+
// serverName: 'controller-manager.openshift-kube-controller-manager.svc',
355+
//},
356+
relabelings: [{
357+
sourceLabels: ['__address__'],
358+
action: 'replace',
359+
targetLabel: '__address__',
360+
regex: '(.+)(?::\\d+)',
361+
replacement: '$1:10252',
362+
}],
363+
},
364+
super.endpoints,
365+
),
366+
},
367+
},
368+
334369
// These patches inject the oauth proxy as a sidecar and configures it with
335370
// TLS. Additionally as the Alertmanager is protected with TLS, authN and
336371
// authZ it requires some additonal configuration.

0 commit comments

Comments
 (0)