Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit ce5faab

Browse files
dominicgunndominicgunn
authored
Merge pull request #1895 from kfr2/coredns-local-v0.16.x
[v0.16.x] Allow dnsmasq to be backed by a local copy of CoreDNS
2 parents 4518022 + d9906df commit ce5faab

File tree

5 files changed

+501
-40
lines changed

5 files changed

+501
-40
lines changed

‎builtin/files/cluster.yaml.tmpl

+33-2
Original file line numberDiff line numberDiff line change
@@ -1167,7 +1167,7 @@ kubernetes:
11671167
# Tells Kubernetes to enable the autoscaler rest client (not using heapster) without the requirement to use metrics-server.
11681168
podAutoscalerUseRestClient:
11691169
enabled: false
1170-
1170+
11711171
# controllerManager:
11721172
# resources:
11731173
# requests:
@@ -1329,6 +1329,35 @@ kubeDns:
13291329
# - --neg-ttl=10
13301330
# - --no-ping
13311331

1332+
# Settings for the dnsmasq-node DaemonSet which must be enabled by setting
1333+
# `kubeDns.nodeLocalResolver` to true.
1334+
dnsmasq:
1335+
coreDNSLocal:
1336+
# When enabled, this will run a copy of CoreDNS within each DNS-masq pod and
1337+
# configure the utility to use it for resolution.
1338+
enabled: false
1339+
1340+
# Defines the resource requests/limits for the coredns-local container.
1341+
# cpu and/or memory constraints can be removed by setting the appropriate value(s)
1342+
# to an empty string.
1343+
resources:
1344+
requests:
1345+
cpu: 50m
1346+
memory: 100Mi
1347+
limits:
1348+
cpu: 50m
1349+
memory: 100Mi
1350+
1351+
# The size of dnsmasq's cache.
1352+
cacheSize: 50000
1353+
1354+
# The maximum number of concurrent DNS queries.
1355+
dnsForwardMax: 500
1356+
1357+
# This option gives a default value for time-to-live (in seconds) which dnsmasq
1358+
# uses to cache negative replies even in the absence of an SOA record.
1359+
# negTTL: 60
1360+
13321361
# When enabled, will modify the TTL of the coredns service.
13331362
# ttl: 30
13341363

@@ -1351,6 +1380,8 @@ kubeDns:
13511380
#
13521381
# This configuration is injected into the CoreDNS config map after the root
13531382
# zone (".") and can be used to add configuration for additional zones.
1383+
# If coreDNSLocal has been enabled, this configuration will additionally be injected
1384+
# into its ConfigMap.
13541385
# additionalZoneCoreDNSConfig: |
13551386
# global:53 {
13561387
# errors
@@ -1378,7 +1409,7 @@ kubeProxy:
13781409
# When enabled, a security group rule is included on the generated kube-aws SG to allow ICMP Ping from all traffic (0.0.0.0/0).
13791410
# This is applied to all nodes (worker & control plane) in the cluster.
13801411
openICMP: true
1381-
1412+
13821413
# Addon features
13831414
addons:
13841415
# When enabled, Kubernetes rescheduler is deployed to the cluster controller(s)

‎builtin/files/userdata/cloud-config-controller

+176-16
Original file line numberDiff line numberDiff line change
@@ -1160,9 +1160,14 @@ write_files:
11601160
"${mfdir}/kube-dns-autoscaler-de.yaml" \
11611161
"${mfdir}/kube-dns-de.yaml"
11621162
{{- end }}
1163-
{{ if .KubeDns.NodeLocalResolver -}}
1163+
{{- if .KubeDns.NodeLocalResolver }}
1164+
{{- if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
1165+
deploy "${mfdir}/dnsmasq-node-coredns-local.yaml"
1166+
{{- else }}
1167+
remove "${mfdir}/dnsmasq-node-coredns-local.yaml"
1168+
{{- end }}
11641169
deploy "${mfdir}/dnsmasq-node-ds.yaml"
1165-
{{ end -}}
1170+
{{- end }}
11661171
forceapply "${mfdir}/kube-dns-pdb.yaml"
11671172

11681173
{{ if .Addons.MetricsServer.Enabled -}}
@@ -5384,6 +5389,9 @@ write_files:
53845389
namespace: kube-system
53855390
data:
53865391
Corefile: |
5392+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5393+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5394+
{{- end }}
53875395
.:53 {
53885396
errors
53895397
health
@@ -5406,9 +5414,6 @@ write_files:
54065414
reload
54075415
loadbalance
54085416
}
5409-
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5410-
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5411-
{{- end }}
54125417
{{- else }}
54135418
- path: /srv/kubernetes/manifests/kube-dns-sa.yaml
54145419
content: |
@@ -5474,9 +5479,86 @@ write_files:
54745479
- --v=2
54755480
- --logtostderr
54765481

5482+
- path: /srv/kubernetes/manifests/dnsmasq-node-coredns-local.yaml
5483+
content: |
5484+
apiVersion: v1
5485+
kind: ConfigMap
5486+
metadata:
5487+
name: coredns-local
5488+
namespace: kube-system
5489+
labels:
5490+
application: coredns
5491+
data:
5492+
Corefile: |
5493+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5494+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
5495+
{{- end }}
5496+
5497+
cluster.local:9254 {{ .PodCIDR }}:9254 {{ .ServiceCIDR }}:9254 {
5498+
errors
5499+
kubernetes {
5500+
pods insecure
5501+
}
5502+
cache 30
5503+
log svc.svc.cluster.local.
5504+
prometheus :9153
5505+
}
5506+
5507+
.:9254 {
5508+
errors
5509+
health :9154 # this is global for all servers
5510+
prometheus :9153
5511+
forward . /etc/resolv.conf
5512+
pprof 127.0.0.1:9156
5513+
cache 30
5514+
reload
5515+
}
5516+
54775517
{{ if .KubeDns.NodeLocalResolver }}
54785518
- path: /srv/kubernetes/manifests/dnsmasq-node-ds.yaml
54795519
content: |
5520+
apiVersion: v1
5521+
kind: ServiceAccount
5522+
metadata:
5523+
name: dnsmasq
5524+
namespace: kube-system
5525+
---
5526+
apiVersion: rbac.authorization.k8s.io/v1
5527+
kind: ClusterRole
5528+
metadata:
5529+
name: dnsmasq
5530+
rules:
5531+
- apiGroups: [""]
5532+
resources: ["endpoints", "services", "pods", "namespaces"]
5533+
verbs: ["list", "watch"]
5534+
---
5535+
apiVersion: rbac.authorization.k8s.io/v1
5536+
kind: ClusterRoleBinding
5537+
metadata:
5538+
name: dnsmasq
5539+
roleRef:
5540+
apiGroup: rbac.authorization.k8s.io
5541+
kind: ClusterRole
5542+
name: dnsmasq
5543+
subjects:
5544+
- kind: ServiceAccount
5545+
name: dnsmasq
5546+
namespace: kube-system
5547+
---
5548+
apiVersion: rbac.authorization.k8s.io/v1
5549+
kind: RoleBinding
5550+
metadata:
5551+
name: dnsmasq-privileged-psp
5552+
namespace: kube-system
5553+
roleRef:
5554+
apiGroup: rbac.authorization.k8s.io
5555+
kind: ClusterRole
5556+
name: privileged-psp
5557+
subjects:
5558+
- kind: ServiceAccount
5559+
name: dnsmasq
5560+
namespace: kube-system
5561+
---
54805562
apiVersion: apps/v1
54815563
kind: DaemonSet
54825564
metadata:
@@ -5485,9 +5567,12 @@ write_files:
54855567
labels:
54865568
k8s-app: dnsmasq-node
54875569
spec:
5570+
selector:
5571+
matchLabels:
5572+
k8s-app: dnsmasq-node
54885573
updateStrategy:
54895574
rollingUpdate:
5490-
maxUnavailable: 100%
5575+
maxUnavailable: 10%
54915576
type: RollingUpdate
54925577
selector:
54935578
matchLabels:
@@ -5512,15 +5597,24 @@ write_files:
55125597
configMap:
55135598
name: kube-dns
55145599
optional: true
5600+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5601+
- name: coredns-local-config
5602+
configMap:
5603+
name: coredns-local
5604+
items:
5605+
- key: Corefile
5606+
path: Corefile
5607+
{{ end }}
55155608
containers:
55165609
- name: dnsmasq
55175610
image: {{ .KubeDnsMasqImage.RepoWithTag }}
55185611
livenessProbe:
55195612
httpGet:
55205613
path: /healthcheck/dnsmasq
5521-
port: 10054
5614+
port: 9054
55225615
scheme: HTTP
55235616
initialDelaySeconds: 60
5617+
periodSeconds: 10
55245618
timeoutSeconds: 5
55255619
successThreshold: 1
55265620
failureThreshold: 5
@@ -5531,13 +5625,24 @@ write_files:
55315625
- -restartDnsmasq=true
55325626
- --
55335627
- -k
5534-
- --min-port=1024
5535-
- --cache-size=1000
5628+
- --cache-size={{ .KubeDns.DNSMasq.CacheSize }}
5629+
- --dns-forward-max={{ .KubeDns.DNSMasq.DNSForwardMax }}
5630+
- --log-facility=-
5631+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5632+
- --no-resolv
5633+
- --keep-in-foreground
5634+
- --neg-ttl={{ .KubeDns.DNSMasq.NegTTL }}
5635+
# Send requests to the last server (coredns-local) first and only
5636+
# fallback to the previous one (global coredns) if it's unreachable.
5637+
- --strict-order
5638+
- --server={{.DNSServiceIP}}#53
5639+
- --server=127.0.0.1#9254
5640+
{{ else }}
55365641
- --server=//{{.DNSServiceIP}}
55375642
- --server=/cluster.local/{{.DNSServiceIP}}
55385643
- --server=/in-addr.arpa/{{.DNSServiceIP}}
55395644
- --server=/ip6.arpa/{{.DNSServiceIP}}
5540-
- --log-facility=-
5645+
{{ end }}
55415646
{{- if ne (len .KubeDns.NodeLocalResolverOptions) 0 }}
55425647
{{- range .KubeDns.NodeLocalResolverOptions }}
55435648
- {{.}}
@@ -5553,8 +5658,10 @@ write_files:
55535658
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
55545659
resources:
55555660
requests:
5556-
cpu: 150m
5557-
memory: 20Mi
5661+
ephemeral-storage: 256Mi
5662+
limits:
5663+
cpu: 10m
5664+
memory: 45Mi
55585665
volumeMounts:
55595666
- name: kube-dns-config
55605667
mountPath: /etc/k8s/dns/dnsmasq-nanny
@@ -5563,7 +5670,7 @@ write_files:
55635670
livenessProbe:
55645671
httpGet:
55655672
path: /metrics
5566-
port: 10054
5673+
port: 9054
55675674
scheme: HTTP
55685675
initialDelaySeconds: 60
55695676
timeoutSeconds: 5
@@ -5572,17 +5679,70 @@ write_files:
55725679
args:
55735680
- --v=2
55745681
- --logtostderr
5682+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5683+
- --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
5684+
{{ else }}
55755685
- --probe=dnsmasq,127.0.0.1:53,ec2.amazonaws.com,5,A
5686+
{{ end }}
5687+
- --prometheus-port=9054
55765688
ports:
5577-
- containerPort: 10054
5689+
- containerPort: 9054
55785690
name: metrics
55795691
protocol: TCP
55805692
resources:
55815693
requests:
5582-
memory: 20Mi
5694+
ephemeral-storage: 256Mi
5695+
limits:
5696+
cpu: 100m
5697+
memory: 50Mi
5698+
terminationMessagePath: /dev/termination-log
5699+
terminationMessagePolicy: File
5700+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5701+
- name: coredns
5702+
image: {{ .CoreDnsImage.RepoWithTag }}
5703+
args: ["-conf", "/etc/coredns/Corefile"]
5704+
volumeMounts:
5705+
- name: coredns-local-config
5706+
mountPath: /etc/coredns
5707+
ports:
5708+
- containerPort: 9254
5709+
name: dns
5710+
protocol: UDP
5711+
- containerPort: 9254
5712+
name: dns-tcp
5713+
protocol: TCP
5714+
livenessProbe:
5715+
httpGet:
5716+
path: /health
5717+
port: 9154
5718+
scheme: HTTP
5719+
initialDelaySeconds: 60
5720+
timeoutSeconds: 5
5721+
successThreshold: 1
5722+
failureThreshold: 5
5723+
resources:
5724+
requests:
5725+
ephemeral-storage: 256Mi
5726+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5727+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5728+
{{ end }}
5729+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5730+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5731+
{{ end }}
5732+
{{ if or .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5733+
limits:
5734+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5735+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5736+
{{ end }}
5737+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5738+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5739+
{{ end }}
5740+
{{ end }}
5741+
{{ end }}
55835742
hostNetwork: true
55845743
dnsPolicy: Default
5585-
automountServiceAccountToken: false
5744+
automountServiceAccountToken: true
5745+
serviceAccountName: dnsmasq
55865746
{{ end }}
55875747

55885748
{{- if eq .KubeDns.Provider "coredns" }}

‎pkg/api/cluster.go

+20-2
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,26 @@ func NewDefaultCluster() *Cluster {
184184
IPVSMode: ipvsMode,
185185
},
186186
KubeDns: KubeDns{
187-
Provider: "coredns",
188-
NodeLocalResolver: false,
187+
Provider: "coredns",
188+
NodeLocalResolver: false,
189+
DNSMasq: DNSMasq{
190+
CoreDNSLocal: CoreDNSLocal{
191+
Enabled: false,
192+
ComputeResources: ComputeResources{
193+
Requests: ResourceQuota{
194+
Cpu: "50m",
195+
Memory: "100Mi",
196+
},
197+
Limits: ResourceQuota{
198+
Cpu: "50m",
199+
Memory: "100Mi",
200+
},
201+
},
202+
},
203+
CacheSize: 50000,
204+
DNSForwardMax: 500,
205+
NegTTL: 60,
206+
},
189207
DeployToControllers: false,
190208
AntiAffinityAvailabilityZone: false,
191209
TTL: 30,

0 commit comments

Comments
 (0)