Initial Sigma support (#1028)

* Initial work on Sigma support

* Initial work on Sigma support

* WIP

* Initial work on Sigma support

* Initial work on Sigma support

* Copy sigma config into docker container

* squash! Copy sigma config into docker container

* squash! Copy sigma config into docker container

* Update requirements.txt

Co-authored-by: Johan Berggren <[email protected]>

Author: Onager

data/linux/recon_commands.yaml

@@ -0,0 +1,24 @@
+title: Linux reconnaissance commands
+description: Commands that are run by attackers after compromising a system
+logsource:
+  service: shell
+references:
+  - https://github.com/mubix/post-exploitation/wiki/Linux-Post-Exploitation-Command-List
+detection:
+  keywords:
+    - 'uname -a'
+    - 'cat /proc/version'
+    - 'grep pass'
+    - 'getent group'
+    - 'getent passwd'
+    - 'cat /home/*/.ssh/authorized_keys'
+    - 'cat /etc/sudoers'
+    - 'cat /etc/passwd'
+    - 'cat /etc/resolv.conf'
+    - 'ps aux'
+    - 'who -a'
+    - 'hostname -f'
+    - 'netstat -nltupw'
+    - 'cat /proc/net/*'
+timeframe: 30m
+conditions: count > 3

data/linux/reverse_shell.yaml

@@ -0,0 +1,13 @@
+title: Possible reverse shell command
+description: Commands that look like reverse shell invocations
+references:
+    - https://alamot.github.io/reverse_shells/
+logsource:
+  service: shell
+detection:
+  keywords:
+    - '-i >& /dev/tcp/'
+    - 'exec 5<>/dev/tcp/'
+    - 'nc -e /bin/sh'
+    - "socat exec:'bash -li',pty,stderr,setsid,sigint,sane"
+condition: keywords

data/sigma_config.yaml

@@ -0,0 +1,30 @@
+title: Timesketch Sigma config
+order: 20
+backends:
+  - es-dsl
+  - es-qs
+logsources:
+  sshd:
+    service: sshd
+    conditions:
+      data_type: "syslog/sshd"
+  auth:
+    service: auth
+    conditions:
+      data_type: "syslog"
+  apache:
+    product: apache
+    conditions:
+      data_type: "apache:access"
+  vsftp:
+    service: vsftp
+    conditions:
+      data_type: "vsftpd:log"
+  webserver:
+    category: webserver
+    conditions:
+      data_type: "apache:access OR iis:log:line"
+  shell:
+    service: shell
+    conditions:
+      data_type: "shell:zsh:history OR bash:history:command"

docker/Dockerfile

@@ -30,14 +30,16 @@ RUN apt-get update && apt-get -y install plaso-tools nodejs yarn
 RUN pip3 install --upgrade pip
 ADD . /tmp/timesketch
 RUN cd /tmp/timesketch && yarn install && yarn run build
-# Remove pyyaml from requirements.txt to avoid conflits with python-yaml Ubuntu package
+# Remove pyyaml from requirements.txt to avoid conflicts with python-yaml Ubuntu package
 RUN sed -i -e '/pyyaml/d' /tmp/timesketch/requirements.txt
 RUN pip3 install /tmp/timesketch/
 
 # Copy Timesketch config files into /etc/timesketch
 RUN mkdir /etc/timesketch
 RUN cp /tmp/timesketch/data/timesketch.conf /etc/timesketch/
 RUN cp /tmp/timesketch/data/features.yaml /etc/timesketch/
+RUN cp /tmp/timesketch/data/sigma_config.yaml /etc/timesketch/
+
 
 # Copy the entrypoint script into the container
 COPY docker/docker-entrypoint.sh /

requirements.txt

@@ -24,8 +24,9 @@ python_dateutil==2.8.1
 PyYAML==5.3
 redis==3.3.11
 requests==2.21.0
+sigmatools==0.14 ; python_version > '3.4'
 six==1.12.0
 SQLAlchemy==1.3.12
 Werkzeug==0.16.0
 WTForms==2.2.1
-xlrd==1.2.0
+xlrd==1.2.0

setup.py

@@ -59,7 +59,7 @@
     ],
     data_files=[
         ('share/timesketch', glob.glob(
-            os.path.join('data', '*'))),
+            os.path.join('data', '*'), recursive=True)),
         ('share/doc/timesketch', [
             'AUTHORS', 'LICENSE', 'README.md']),
     ],

timesketch/lib/analyzers/__init__.py

@@ -24,6 +24,7 @@
 from timesketch.lib.analyzers import login
 from timesketch.lib.analyzers import phishy_domains
 from timesketch.lib.analyzers import sessionizer
+from timesketch.lib.analyzers import sigma_tagger
 from timesketch.lib.analyzers import similarity_scorer
 from timesketch.lib.analyzers import ssh_sessionizer
 from timesketch.lib.analyzers import gcp_servicekey

timesketch/lib/analyzers/interface.py

@@ -43,6 +43,27 @@ def wrapper(self, *args, **kwargs):
         return func_return
     return wrapper
 
+def get_config_path(file_name):
+    """Returns a path to a configuration file.
+
+    Args:
+        file_name: String that defines the config file name.
+
+    Returns:
+        The path to the configuration file or None if the file cannot be found.
+    """
+    path = os.path.join(os.path.sep, 'etc', 'timesketch', file_name)
+    if os.path.isfile(path):
+        return path
+
+    path = os.path.join(
+        os.path.dirname(__file__), '..', '..', '..', 'data', file_name)
+    path = os.path.abspath(path)
+    if os.path.isfile(path):
+        return path
+
+    return None
+
 
 def get_yaml_config(file_name):
     """Return a dict parsed from a YAML file within the config directory.
@@ -55,12 +76,8 @@ def get_yaml_config(file_name):
         an empty dict if the file is not found or YAML was unable
         to parse it.
     """
-    root_path = os.path.join(os.path.sep, 'etc', 'timesketch')
-    if not os.path.isdir(root_path):
-        return {}
-
-    path = os.path.join(root_path, file_name)
-    if not os.path.isfile(path):
+    path = get_config_path(file_name)
+    if not path:
         return {}
 
     with open(path, 'r') as fh:

timesketch/lib/analyzers/sigma_tagger.py

@@ -0,0 +1,116 @@
+"""Index analyzer plugin for sigma."""
+from __future__ import unicode_literals
+
+import logging
+import os
+
+from sigma.backends import elasticsearch as sigma_elasticsearch
+import sigma.configuration as sigma_configuration
+from sigma.parser import collection as sigma_collection
+
+
+from timesketch.lib.analyzers import interface
+from timesketch.lib.analyzers import manager
+
+
+class SigmaPlugin(interface.BaseSketchAnalyzer):
+    """Index analyzer for Sigma."""
+
+    NAME = 'sigma'
+
+    _CONFIG_FILE = 'sigma_config.yaml'
+
+    # Path to the directory containing the Sigma Rules to run, relative to
+    # this file.
+    _RULES_PATH = ''
+
+
+    def __init__(self, index_name, sketch_id):
+        """Initialize the Index Analyzer.
+
+        Args:
+            index_name: Elasticsearch index name.
+            sketch_id: Sketch ID.
+        """
+        super(SigmaPlugin, self).__init__(index_name, sketch_id)
+        sigma_config_path = interface.get_config_path(self._CONFIG_FILE)
+        logging.debug('[sigma] Loading config from {0!s}'.format(
+            sigma_config_path))
+        with open(sigma_config_path, 'r') as sigma_config_file:
+            sigma_config = sigma_config_file.read()
+        self.sigma_config = sigma_configuration.SigmaConfiguration(sigma_config)
+
+    def run_sigma_rule(self, query, tag_name):
+        """Runs a sigma rule and applies the appropriate tags.
+
+        Args:
+            query: elastic search query for events to tag.
+            tag_name: tag to apply to matching events.
+
+        Returns:
+            int: number of events tagged.
+        """
+        return_fields = []
+        tagged_events = 0
+        events = self.event_stream(
+            query_string=query, return_fields=return_fields)
+        for event in events:
+            event.add_tags([tag_name])
+            event.commit()
+            tagged_events += 1
+        return tagged_events
+
+    def run(self):
+        """Entry point for the analyzer.
+
+        Returns:
+            String with summary of the analyzer result.
+        """
+        sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(
+            self.sigma_config, {})
+        tags_applied = {}
+
+        rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH)
+        for rule_filename in os.listdir(rules_path):
+            tag_name, _ = rule_filename.rsplit('.')
+            tags_applied[tag_name] = 0
+            rule_file_path = os.path.join(rules_path, rule_filename)
+            rule_file_path = os.path.abspath(rule_file_path)
+            logging.info('[sigma] Reading rules from {0!s}'.format(
+                rule_file_path))
+            with open(rule_file_path, 'r') as rule_file:
+                rule_file_content = rule_file.read()
+                parser = sigma_collection.SigmaCollectionParser(
+                    rule_file_content, self.sigma_config, None)
+                try:
+                    results = parser.generate(sigma_backend)
+                except NotImplementedError as exception:
+                    logging.error(
+                        'Error generating rule in file {0:s}: {1!s}'.format(
+                            rule_file_path, exception))
+                    continue
+
+                for result in results:
+                    logging.info(
+                        '[sigma] Generated query {0:s}'.format(result))
+                    number_of_tagged_events = self.run_sigma_rule(
+                        result, tag_name)
+                    tags_applied[tag_name] += number_of_tagged_events
+
+        total_tagged_events = sum(tags_applied.values())
+        output_string = 'Applied {0:d} tags\n'.format(total_tagged_events)
+        for tag_name, number_of_tagged_events in tags_applied.items():
+            output_string += '* {0:s}: {1:d}'.format(
+                tag_name, number_of_tagged_events)
+        return output_string
+
+
+class LinuxRulesSigmaPlugin(SigmaPlugin):
+    """Sigma plugin to run Linux rules."""
+
+    _RULES_PATH = '../../../data/linux'
+
+    NAME = 'sigma_linux'
+
+
+manager.AnalysisManager.register_analyzer(LinuxRulesSigmaPlugin)

timesketch/lib/analyzers/sigma_tagger_test.py

@@ -0,0 +1,28 @@
+"""Tests for SigmaPlugin."""
+from __future__ import unicode_literals
+
+import mock
+
+from timesketch.lib.analyzers import sigma_tagger
+from timesketch.lib.testlib import BaseTest
+from timesketch.lib.testlib import MockDataStore
+
+
+class TestSigmaPlugin(BaseTest):
+    """Tests the functionality of the analyzer."""
+
+    def __init__(self, *args, **kwargs):
+        super(TestSigmaPlugin, self).__init__(*args, **kwargs)
+        self.test_index = 'test_index'
+
+
+    # Mock the Elasticsearch datastore.
+    @mock.patch(
+        'timesketch.lib.analyzers.interface.ElasticsearchDataStore',
+        MockDataStore)
+    def test_analyzer(self):
+        """Test analyzer."""
+        # TODO: Add more tests
+
+        _ = sigma_tagger.LinuxRulesSigmaPlugin(
+            sketch_id=1, index_name=self.test_index)