[kubernetes] add support for kubelet authN/authZ

Author: hkaj

checks.d/kubernetes.py

@@ -14,7 +14,6 @@
 import calendar
 
 # 3rd party
-import requests
 import simplejson as json
 
 # project
@@ -88,8 +87,8 @@ def __init__(self, name, init_config, agentConfig, instances=None):
 
         inst = instances[0] if instances is not None else None
         self.kubeutil = KubeUtil(instance=inst)
-        if not self.kubeutil.host:
-            raise Exception('Unable to retrieve Docker hostname and host parameter is not set')
+        if not self.kubeutil.kubelet_api_url:
+            raise Exception('Unable to reach kubelet. Try setting the host parameter.')
 
         self.k8s_namespace_regexp = None
         if inst:
@@ -104,8 +103,8 @@ def _perform_kubelet_checks(self, url):
         service_check_base = NAMESPACE + '.kubelet.check'
         is_ok = True
         try:
-            r = requests.get(url, params={'verbose': True})
-            for line in r.iter_lines():
+            req = self.kubeutil.retrieve_kubelet_url(url)
+            for line in req.iter_lines():
 
                 # avoid noise; this check is expected to fail since we override the container hostname
                 if line.find('hostname') != -1:

conf.d/kubernetes.yaml.example

@@ -1,27 +1,42 @@
 init_config:
 
 instances:
-  # The kubernetes check retrieves metrics from cadvisor running under kubelet.
-  # By default we will assume we're running under docker and will use the address
-  # of the default router to reach the cadvisor api unless the environment variable
-  # KUBERNETES_KUBELET_HOST is found.
+  # The kubernetes check retrieves metrics from cadvisor running under kubelet on each node.
+  # It also queries kubelet for its health and the list of local-running pods.
   #
-  # To override, e.g. in the case of a standalone cadvisor instance, use the following:
+  # By default we assume we're running under docker and
+  # that the kubelet read-only port with no auth is enabled.
+  #
+  # In this case we will use the address of the default router to
+  # reach the kubelet and cadvisor APIs unless the environment variable
+  # KUBERNETES_KUBELET_HOST is found (if your node names can be resolved by pods, we recommend you set this variable to spec.nodeName through the downward API).
+  #
+  # If the read-only endpoint is disabled, the check will query kubelet over HTTPS
+  #
+  # To override this behavior, e.g. in the case of a standalone cadvisor instance, use the following:
   #
   # host: localhost
   # port: 4194
   # method: http
+
+  # cAdvisor port
  - port: 4194
 
+  # cAdvisor host
+  # host: localhost
+
   # Authentication against the apiserver
+  #
   # By default the agent authenticates against the apiserver with its service account bearer token.
   # X509 certificate authentication is also supported.
   # To enable cert auth in place of bearer token auth, provide here the paths to
   # a client cert/key pair.
   # The recommended way to expose these files to the agent is by using Kubernetes Secrets.
+  #
   # apiserver_client_crt: /path/to/client.crt
   # apiserver_client_key: /path/to/client.key
   #
+  #
   # collect_events controls whether the agent should fetch events from the kubernetes API and
   # ingest them in Datadog. To avoid duplicates, only one agent at a time across the entire
   # cluster should have this feature enabled. To enable the feature, set the parameter to `true`.
@@ -48,8 +63,37 @@ instances:
   #
   # use_histogram: false
 
+  # kubelet port. It needs to be set if you are not using a default one (10250 or 10255)
   # kubelet_port: 10255
   #
+  # SSL configuration for querying kubelet.
+  #
+  # Available options are:
+  # - simple SSL validation (with readily available certificates)
+  # - disabling SSL validation
+  # - providing the server certificate
+  # - providing a client cert/key pair
+  #
+  # The default is to try and use the read-only port that doesn't require SSL
+  # And to fall back to the HTTPS API with simple SSL validation.
+  #
+  # Explicitly disabling ssl_verify should be used with caution:
+  # if an attacker sniffs the agent requests they will see the agent's service account bearer token.
+  #
+  # Providing a server cert enables SSL validation.
+  #
+  # It is possible to providing a client cert/key pair for client auth.
+  # The agent also uses its pod bearer token (if available) when querying kubelet and the apiserver.
+  #
+  # The recommended way to pass certificates and keys to the agent is by using
+  # "Secret" Kubernetes object (and to mount them as volumes).
+  #
+  # kubelet_ssl_verify: True
+  # kubelet_cert: /path/to/ca.pem
+  # kubelet_client_crt: /path/to/client.crt
+  # kubelet_client_key: /path/to/client.key
+  #
+  #
   # We can define a whitelist of patterns that permit publishing raw metrics.
   # enabled_rates:
   #   - cpu.*

tests/checks/fixtures/kubernetes/filtered_node_list_1_4.json

@@ -0,0 +1 @@
+{"items": [{"status": {"capacity": {"alpha.kubernetes.io/nvidia-gpu": "0", "pods": "110", "cpu": "2", "memory": "4045004Ki"}, "addresses": [{"type": "LegacyHostIP", "address": "10.0.0.179"}, {"type": "InternalIP", "address": "10.0.0.179"}, {"type": "Hostname", "address": "ip-10-0-0-179"}], "nodeInfo": {"kernelVersion": "4.4.0-53-generic", "kubeletVersion": "v1.5.1", "containerRuntimeVersion": "docker://1.12.1", "machineID": "7b8ef452673e468f83443e29b4773c15", "kubeProxyVersion": "v1.5.1", "bootID": "929724dc-5603-429a-9e73-6c46e9f6f43b", "osImage": "Ubuntu 16.04.1 LTS", "architecture": "amd64", "systemUUID": "EC2501F2-8EE6-DD8B-96DC-0467866A5D86", "operatingSystem": "linux"}, "allocatable": {"alpha.kubernetes.io/nvidia-gpu": "0", "pods": "110", "cpu": "2", "memory": "4045004Ki"}, "daemonEndpoints": {"kubeletEndpoint": {"Port": 10250}}, "images": [{"sizeBytes": 463140901, "names": ["datadog/dev-dd-agent@sha256:829ccd2e8321507ee51929daa2a6347e0a6389483d25463dc241bfccf9199660", "datadog/dev-dd-agent:kubelet-auth"]}, {"sizeBytes": 398248138, "names": ["datadog/docker-dd-agent@sha256:577b876eae85e6d4bf372cfafb10211eb3abfafe16a44b1b94eec957cb9efb08", "datadog/docker-dd-agent:optimize-kv"]}, {"sizeBytes": 334276060, "names": ["datadog/docker-dd-agent@sha256:286632728630a18315913285b002a970429350e220f5fa36e70ff2ce27bbcb9b", "datadog/docker-dd-agent:11.0.5101"]}, {"sizeBytes": 182896153, "names": ["redis@sha256:eed4da4937cb562e9005f3c66eb8c3abc14bb95ad497c03dc89d66bcd172fc7f", "redis:latest"]}, {"sizeBytes": 175577274, "names": ["gcr.io/google_containers/kube-proxy-amd64@sha256:3b82b2e0862b3c0ece915de29a5a53634c9b0a73140340f232533c645decbd4b", "gcr.io/google_containers/kube-proxy-amd64:v1.5.1"]}, {"sizeBytes": 166671224, "names": ["weaveworks/weave-kube@sha256:ebd53291e97326b87ddcfd9a45a779ebaf2ca42fd54489957bac3c808999a057", "weaveworks/weave-kube:1.8.2"]}, {"sizeBytes": 68772084, "names": ["weaveworks/weave-npc@sha256:39fd8697bcc6ccbcfdbb209e4028fb36002db0d97eec9cadd88120a0b18181a4", "weaveworks/weave-npc:1.8.2"]}, {"sizeBytes": 746888, "names": ["gcr.io/google_containers/pause-amd64@sha256:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516", "gcr.io/google_containers/pause-amd64:3.0"]}], "conditions": [{"status": "False", "lastTransitionTime": "2017-01-11T16:25:13Z", "reason": "KubeletHasSufficientDisk", "lastHeartbeatTime": "2017-01-22T22:17:32Z", "message": "kubelet has sufficient disk space available", "type": "OutOfDisk"}, {"status": "False", "lastTransitionTime": "2017-01-11T16:25:13Z", "reason": "KubeletHasSufficientMemory", "lastHeartbeatTime": "2017-01-22T22:17:32Z", "message": "kubelet has sufficient memory available", "type": "MemoryPressure"}, {"status": "False", "lastTransitionTime": "2017-01-11T16:25:13Z", "reason": "KubeletHasNoDiskPressure", "lastHeartbeatTime": "2017-01-22T22:17:32Z", "message": "kubelet has no disk pressure", "type": "DiskPressure"}, {"status": "True", "lastTransitionTime": "2017-01-11T21:32:03Z", "reason": "KubeletReady", "lastHeartbeatTime": "2017-01-22T22:17:32Z", "message": "kubelet is posting ready status. AppArmor enabled", "type": "Ready"}]}, "spec": {"providerID": "aws:////i-0860ce2a36f75f436", "externalID": "ip-10-0-0-179"}, "metadata": {"name": "ip-10-0-0-179", "labels": {"kubernetes.io/hostname": "ip-10-0-0-179", "beta.kubernetes.io/os": "linux", "beta.kubernetes.io/arch": "amd64"}, "resourceVersion": "1360060", "creationTimestamp": "2017-01-11T16:25:13Z", "annotations": {"volumes.kubernetes.io/controller-managed-attach-detach": "true"}, "selfLink": "/api/v1/nodesip-10-0-0-179", "uid": "87a4a4af-d81a-11e6-aee2-121441fd904e"}}], "kind": "NodeList", "apiVersion": "v1", "metadata": {"selfLink": "/api/v1/nodes", "resourceVersion": "1360069"}}