Merge pull request #2 from DataDog/master

Merge from upstream

Author: Igor Polishchuk

checks.d/http_check.py

@@ -1,41 +1,73 @@
 # stdlib
+from datetime import datetime
+import os.path
 import socket
+import ssl
 import time
 from urlparse import urlparse
 
+# 3rd party
+from httplib2 import Http, HttpLib2Error
+import tornado
+
 # project
 from checks.network_checks import NetworkCheck, Status, EventType
+from config import _is_affirmative
 from util import headers as agent_headers
 
-# 3rd party
-from httplib2 import Http, HttpLib2Error
 
-class HTTPCheck(NetworkCheck):
+def get_ca_certs_path():
+    """
+    Get a path to the trusted certificates of the system
+    """
+    CA_CERTS = [
+        '/opt/datadog-agent/embedded/ssl/certs/cacert.pem',
+        os.path.join(os.path.dirname(tornado.__file__), 'ca-certificates.crt'),
+        '/etc/ssl/certs/ca-certificates.crt',
+    ]
+
+    for f in CA_CERTS:
+        if os.path.exists(f):
+            return f
+    return None
+
 
+class HTTPCheck(NetworkCheck):
     SOURCE_TYPE_NAME = 'system'
-    SERVICE_CHECK_PREFIX = 'http_check'
+    SC_STATUS = 'http_check'
+    SC_SSL_CERT = 'http_check.ssl_cert'
+
+    def __init__(self, name, init_config, agentConfig, instances):
+        self.ca_certs = init_config.get('ca_certs', get_ca_certs_path())
+        NetworkCheck.__init__(self, name, init_config, agentConfig, instances)
 
     def _load_conf(self, instance):
         # Fetches the conf
         tags = instance.get('tags', [])
         username = instance.get('username', None)
         password = instance.get('password', None)
         timeout = int(instance.get('timeout', 10))
-        config_headers = instance.get('headers',{})
+        config_headers = instance.get('headers', {})
         headers = agent_headers(self.agentConfig)
         headers.update(config_headers)
         url = instance.get('url', None)
-        response_time = instance.get('collect_response_time', True)
+        response_time = _is_affirmative(instance.get('collect_response_time', True))
         if url is None:
             raise Exception("Bad configuration. You must specify a url")
-        include_content = instance.get('include_content', False)
-        ssl = instance.get('disable_ssl_validation', True)
-        return url, username, password, timeout, include_content, headers, response_time, tags, ssl
+        include_content = _is_affirmative(instance.get('include_content', False))
+        ssl = _is_affirmative(instance.get('disable_ssl_validation', True))
+        ssl_expire = _is_affirmative(instance.get('check_certificate_expiration', True))
+
+        return url, username, password, timeout, include_content, headers, response_time, tags, ssl, ssl_expire
 
     def _check(self, instance):
-        addr, username, password, timeout, include_content, headers, response_time, tags, disable_ssl_validation = self._load_conf(instance)
+        addr, username, password, timeout, include_content, headers, response_time, tags, disable_ssl_validation, ssl_expire = self._load_conf(instance)
         content = ''
         start = time.time()
+
+        service_checks = []
+        resp = None
+
         try:
             self.log.debug("Connecting to %s" % addr)
             if disable_ssl_validation and urlparse(addr)[0] == "https":
@@ -48,47 +80,76 @@ def _check(self, instance):
         except socket.timeout, e:
             length = int((time.time() - start) * 1000)
             self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length))
-            return Status.DOWN, "%s. Connection failed after %s ms" % (str(e), length)
+            service_checks.append((
+                self.SC_STATUS,
+                Status.DOWN,
+                "%s. Connection failed after %s ms" % (str(e), length)
+            ))
 
         except HttpLib2Error, e:
             length = int((time.time() - start) * 1000)
             self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length))
-            return Status.DOWN, "%s. Connection failed after %s ms" % (str(e), length)
+            service_checks.append((
+                self.SC_STATUS,
+                Status.DOWN,
+                "%s. Connection failed after %s ms" % (str(e), length)
+            ))
 
         except socket.error, e:
             length = int((time.time() - start) * 1000)
             self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, repr(e), length))
-            return Status.DOWN, "Socket error: %s. Connection failed after %s ms" % (repr(e), length)
+            service_checks.append((
+                self.SC_STATUS,
+                Status.DOWN,
+                "Socket error: %s. Connection failed after %s ms" % (repr(e), length)
+            ))
 
         except Exception, e:
             length = int((time.time() - start) * 1000)
             self.log.error("Unhandled exception %s. Connection failed after %s ms" % (str(e), length))
             raise
 
-        if response_time:
-           # Stop the timer as early as possible
-           running_time = time.time() - start
-           # Store tags in a temporary list so that we don't modify the global tags data structure
-           tags_list = []
-           tags_list.extend(tags)
-           tags_list.append('url:%s' % addr)
-           self.gauge('network.http.response_time', running_time, tags=tags_list)
-
-        if int(resp.status) >= 400:
-            self.log.info("%s is DOWN, error code: %s" % (addr, str(resp.status)))
-            if not include_content:
-                content = ''
-            return Status.DOWN, (resp.status, resp.reason, content or '')
-
-        self.log.debug("%s is UP" % addr)
-        return Status.UP, "UP"
-
-    def _create_status_event(self, status, msg, instance):
+        # Only report this metric if the site is not down
+        if response_time and not service_checks:
+            # Stop the timer as early as possible
+            running_time = time.time() - start
+            # Store tags in a temporary list so that we don't modify the global tags data structure
+            tags_list = list(tags)
+            tags_list.append('url:%s' % addr)
+            self.gauge('network.http.response_time', running_time, tags=tags_list)
+
+        if not service_checks:
+            if resp is not None and int(resp.status) >= 400:
+                self.log.info("%s is DOWN, error code: %s" % (addr, str(resp.status)))
+                if not include_content:
+                    content = ''
+                service_checks.append((
+                    self.SC_STATUS, Status.DOWN, (resp.status, resp.reason, content or '')
+                ))
+            else:
+                self.log.debug("%s is UP" % addr)
+                service_checks.append((
+                    self.SC_STATUS, Status.UP, "UP"
+                ))
+
+        if ssl_expire and urlparse(addr)[0] == "https":
+            status, msg = self.check_cert_expiration(instance)
+            service_checks.append((
+                self.SC_SSL_CERT, status, msg
+            ))
+
+        return service_checks
+
+    # FIXME: 5.3 drop this function
+    def _create_status_event(self, sc_name, status, msg, instance):
+        # Create only this deprecated event for old check
+        if sc_name != self.SC_STATUS:
+            return
         # Get the instance settings
         url = instance.get('url', None)
         name = instance.get('name', None)
-        nb_failures = self.statuses[name].count(Status.DOWN)
-        nb_tries = len(self.statuses[name])
+        nb_failures = self.statuses[name][sc_name].count(Status.DOWN)
+        nb_tries = len(self.statuses[name][sc_name])
         tags = instance.get('tags', [])
         tags_list = []
         tags_list.extend(tags)
@@ -153,11 +214,13 @@ def _create_status_event(self, status, msg, instance):
              "tags": tags_list
         }
 
-    def report_as_service_check(self, name, status, instance, msg=None):
-        service_check_name = self.normalize(name, self.SERVICE_CHECK_PREFIX)
+    def report_as_service_check(self, sc_name, status, instance, msg=None):
+        instance_name = instance['name']
+        service_check_name = self.normalize(instance_name, sc_name)
         url = instance.get('url', None)
+        sc_tags = ['url:%s' % url]
 
-        if status == Status.DOWN:
+        if sc_name == self.SC_STATUS:
             # format the HTTP response body into the event
             if isinstance(msg, tuple):
                 code, reason, content = msg
@@ -168,12 +231,40 @@ def report_as_service_check(self, name, status, instance, msg=None):
 
                 msg = "%d %s\n\n%s" % (code, reason, content)
                 msg = msg.rstrip()
-        else:
-            msg=None
 
         self.service_check(service_check_name,
                            NetworkCheck.STATUS_TO_SERVICE_CHECK[status],
-                           tags= ['url:%s' % url],
+                           tags=sc_tags,
                            message=msg
                            )
 
+    def check_cert_expiration(self, instance):
+        warning_days = int(instance.get('days_warning', 14))
+        url = instance.get('url')
+
+        o = urlparse(url)
+        host = o.netloc
+
+        port = o.port or 443
+
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect((host, port))
+            ssl_sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED,
+                                           ca_certs=self.ca_certs)
+            cert = ssl_sock.getpeercert()
+
+        except Exception as e:
+            return Status.DOWN, "%s" % (str(e))
+
+        exp_date = datetime.strptime(cert['notAfter'], "%b %d %H:%M:%S %Y %Z")
+        days_left = exp_date - datetime.utcnow()
+
+        if days_left.days < 0:
+            return Status.DOWN, "Expired by {0} days".format(days_left.days)
+
+        elif days_left.days < warning_days:
+            return Status.WARNING, "This cert is almost expired, only {0} days left".format(days_left.days)
+
+        else:
+            return Status.UP, "Days left: {0}".format(days_left.days)

checks.d/tcp_check.py

@@ -93,13 +93,14 @@ def _check(self, instance):
         return Status.UP, "UP"
 
 
-    def _create_status_event(self, status, msg, instance):
+    # FIXME: 5.3 remove that
+    def _create_status_event(self, sc_name, status, msg, instance):
         # Get the instance settings
         host = instance.get('host', None)
         port = instance.get('port', None)
         name = instance.get('name', None)
-        nb_failures = self.statuses[name].count(Status.DOWN)
-        nb_tries = len(self.statuses[name])
+        nb_failures = self.statuses[name][sc_name].count(Status.DOWN)
+        nb_tries = len(self.statuses[name][sc_name])
 
         # Get a custom message that will be displayed in the event
         custom_message = instance.get('message', "")
@@ -151,8 +152,9 @@ def _create_status_event(self, status, msg, instance):
              "event_object": name,
         }
 
-    def report_as_service_check(self, name, status, instance, msg=None):
-        service_check_name = self.normalize(name, self.SERVICE_CHECK_PREFIX)
+    def report_as_service_check(self, sc_name, status, instance, msg=None):
+        instance_name = instance['name']
+        service_check_name = self.normalize(instance_name, self.SERVICE_CHECK_PREFIX)
         host = instance.get('host', None)
         port = instance.get('port', None)
 

checks.d/zk.py

@@ -100,7 +100,7 @@ def _send_command(self, command, host, port, timeout):
             try:
                 # Connect to the zk client port and send the stat command
                 sock.connect((host, port))
-                sock.sendall('stat')
+                sock.sendall(command)
 
                 # Read the response into a StringIO buffer
                 chunk = sock.recv(chunk_size)