Merge pull request #4 from hubblestack/4.0

4.0

Author: devagupt

.gitignore

@@ -7,6 +7,10 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+# patch detritus
+*.rej
+*.orig
+
 # C extensions
 *.so
 

.pipeline

@@ -51,7 +51,9 @@ pipeline {
                     else find hubblestack -name "*.py" -print0 | xargs -r0 git diff --name-only "$LHS" "$RHS"
                     fi > relevant-files.txt
                     '''
-                sh '''mkdir -vp tests/unittests/output'''
+                sh ''' mkdir -vp tests/unittests/output
+                       cp relevant-files.txt tests/unittests/output
+                   '''
             }
         }
         stage('lint/test') {
@@ -106,7 +108,7 @@ pipeline {
                 alwaysLinkToLastBuild: false,
                 keepAll: true,
                 reportDir: 'tests/unittests/output',
-                reportFiles: 'pytest.html, coverage/index.html, pylint.html, profile-diagram.svg, bandit.html',
+                reportFiles: 'pytest.html, coverage/index.html, pylint.html, profile-diagram.svg, bandit.html, relevant-files.txt',
                 reportName: "Test Reports"
             ])
         }

openssl-gen-pretend-certs.sh contrib/openssl-gen-pretend-certs.sh

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'3.0.8'
+version = u'4.0.0'
 # The full version, including alpha/beta/rc tags.
-release = u'3.0.8-1'
+release = u'4.0.0-1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

doc/conf.py

@@ -1,3 +1,3 @@
-__version__ = '3.0.8'
+__version__ = '4.0.0'
 
 __buildinfo__ = {'branch': 'BRANCH_NOT_SET', 'last_commit': 'COMMIT_NOT_SET'}

hubblestack/__init__.py

@@ -798,6 +798,9 @@ def refresh_grains(initial=False):
     hubblestack.utils.signing.__opts__ = __opts__
     hubblestack.utils.signing.__salt__ = __salt__
 
+    hubblestack.utils.signing.__opts__ = __opts__
+    hubblestack.utils.signing.__salt__ = __salt__
+
     if not initial and __salt__['config.get']('splunklogging', False):
         hubblestack.log.emit_to_splunk(__grains__, 'INFO', 'hubblestack.grains_report')
 

hubblestack/daemon.py

@@ -105,7 +105,7 @@
 
 log = logging.getLogger(__name__)
 
-S3_CACHE_EXPIRE = 30  # cache for 30 seconds
+S3_CACHE_EXPIRE = 1800  # cache for 30 minutes
 S3_SYNC_ON_UPDATE = True  # sync cache on update rather than jit
 
 
@@ -336,6 +336,7 @@ def _get_s3_key():
         'service_url': None,
         'keyid': None,
         'key': None,
+        'cache_expire': S3_CACHE_EXPIRE,
     }
 
     ret = dict()
@@ -351,14 +352,17 @@ def _get_s3_key():
 
     return ret
 
+
 def _init():
     """
     Connect to S3 and download the metadata for each file in all buckets
     specified and cache the data to disk.
     """
     cache_file = _get_buckets_cache_filename()
-    exp = time.time() - S3_CACHE_EXPIRE
+    cache_expire_time = float(_get_s3_key().get('cache_expire'))
+    exp = time.time() - cache_expire_time
 
+    log.debug('S3 cache expire time is %ds', cache_expire_time)
     # check mtime of the buckets files cache
     metadata = None
     try:
@@ -443,6 +447,9 @@ def __get_s3_meta(bucket, key=s3_key_kwargs['key'], keyid=s3_key_kwargs['keyid']
                                         path_style=s3_key_kwargs['path_style'],
                                         https_enable=s3_key_kwargs['https_enable'],
                                         params={'marker': marker})
+            if not tmp:
+                return None
+
             headers = []
             for header in tmp:
                 if 'Key' in header:

hubblestack/extmods/fileserver/s3fs.py

@@ -73,7 +73,7 @@ def _get_aws_details():
                              headers=aws_token_header, timeout=3, proxies=proxies)
             if r.status_code == requests.codes.ok:
                 aws_extra['cloud_private_hostname'] = r.text
-            for key in aws_extra.keys():
+            for key in list(aws_extra):
                 if not aws_extra[key]:
                     aws_extra.pop(key)
 
@@ -133,7 +133,7 @@ def _get_azure_details():
                 grain_name_mac = "cloud_interface_{0}_mac_address".format(counter)
                 azure_extra[grain_name_mac] = value['macAddress']
 
-            for key in azure_extra:
+            for key in list(azure_extra):
                 if not azure_extra[key]:
                     azure_extra.pop(key)
 

hubblestack/extmods/grains/cloud_details.py

@@ -17,6 +17,7 @@
     HAS_REQUESTS = False  # pylint: disable=W0612
 
 # Import Salt libs
+import os
 import salt.utils.aws
 import salt.utils.files
 import salt.utils.hashutils
@@ -203,6 +204,11 @@ def query(key, keyid, method='GET', params=None, headers=None,
             err_code = 'http-{0}'.format(result.status_code)
             err_msg = err_text
 
+    if os.environ.get('MOCK_SLOW_DOWN'):
+        result.status_code = 503
+        err_code = 'SlowDown'
+        err_msg = 'MOCK_SLOW_DOWN environment variable set. All S3 queries will fail for testing purposes.'
+
     log.debug('S3 Response Status Code: %s', result.status_code)
 
     if method == 'PUT':
@@ -219,7 +225,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             log.debug('Uploaded from %s to %s', local_file, path)
         else:
             log.debug('Created bucket %s', bucket)
-        return
+        return None
 
     if method == 'DELETE':
         if not six.text_type(result.status_code).startswith('2'):
@@ -235,7 +241,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             log.debug('Deleted %s from bucket %s', path, bucket)
         else:
             log.debug('Deleted bucket %s', bucket)
-        return
+        return None
 
     # This can be used to save a binary object to disk
     if local_file and method == 'GET':
@@ -250,6 +256,10 @@ def query(key, keyid, method='GET', params=None, headers=None,
         return 'Saved to local file: {0}'.format(local_file)
 
     if result.status_code < 200 or result.status_code >= 300:
+        if err_code in ['SlowDown', 'ServiceUnavailable', 'RequestTimeTooSkewed',
+                        'RequestTimeout', 'OperationAborted', 'InternalError']:
+            log.error('Failed s3 operation: %s, %s', err_code, err_msg)
+            return None
         raise CommandExecutionError(
             'Failed s3 operation. {0}: {1}'.format(err_code, err_msg))
 
@@ -268,7 +278,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             return ret, requesturl
     else:
         if result.status_code != requests.codes.ok:
-            return
+            return None
         ret = {'headers': []}
         if full_headers:
             ret['headers'] = dict(result.headers)

hubblestack/extmods/utils/s3.py

@@ -114,7 +114,6 @@ def split_certs(fh):
 
         returns a generator, for list, use `list(split_cerst(fh))`
     """
-
     ret = None
     for line in fh.readlines():
         if ret is None:

hubblestack/utils/signing.py

@@ -136,12 +136,22 @@ RUN mkdir -p "$LIBGIT2TEMP" \
  && make \
  && make install
 
-#pyinstaller requirements start
-#must be preceded by libgit2 install
+# use pyenv
+ARG PYENV_VERSION=3.6.10
+ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer
+ENV PYENV_ROOT=/usr/local/pyenv
+ENV PATH=$PYENV_ROOT/bin:$PATH
+RUN umask 022 \
+ && curl -s -S -L "$PYENV_INSTALLER_URL" -o /usr/bin/pyenv-installer \
+ && chmod 0755 /usr/bin/pyenv-installer \
+ && /usr/bin/pyenv-installer \
+ && eval "$(pyenv init -)" \
+ && env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION \
+ && pyenv global $PYENV_VERSION
+
 COPY pyinstaller-requirements.txt /
-#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget.
-RUN wget -c https://bootstrap.pypa.io/get-pip.py \
- && python get-pip.py \
+RUN eval "$(pyenv init -)" \
+ && pip -v install --upgrade pip \
  && pip -v install -r pyinstaller-requirements.txt
 
 #fpm package making requirements start
@@ -151,8 +161,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v3.0.8
-ENV HUBBLE_VERSION=3.0.8
+ARG HUBBLE_CHECKOUT=v4.0.0
+ENV HUBBLE_VERSION=4.0.0
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_URL=https://github.com/hubblestack/hubble
 ENV HUBBLE_DESCRIPTION="Hubble is a modular, open-source, security & compliance auditing framework which is built in python, using SaltStack as a library."
@@ -165,7 +175,7 @@ ENV _INCLUDE_PATH=""
 ENV LD_LIBRARY_PATH=/opt/hubble/lib:/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
  && cd "$HUBBLE_SRC_PATH" \
- && git checkout "$HUBBLE_CHECKOUT" \
+ && git checkout -B hubble-build && git reset --hard "$HUBBLE_CHECKOUT" && git clean -dfx \
  && cp -rf "$HUBBLE_SRC_PATH" /hubble_build \
  && sed -i "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" /hubble_build/hubblestack/__init__.py \
  && sed -i "s/COMMIT_NOT_SET/`git describe`/g" /hubble_build/hubblestack/__init__.py
@@ -174,6 +184,7 @@ VOLUME /data
 WORKDIR /hubble_build
 ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ]
 CMD [ "if [ -f /data/hubble_buildinfo ] ; then echo \"\" >> /hubble_build/hubblestack/__init__.py ; cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py; fi \
+    && eval \"$(pyenv init -)\" \
     && pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} --runtime-hook=pkg/pyinstaller-runtimehooks/pathopthubble.py hubble.py \
     && mkdir -p /var/log/hubble_osquery/backuplogs \
 # hubble default configuration file

pkg/amazonlinux2016.09/Dockerfile

@@ -135,26 +135,34 @@ RUN mkdir -p "$LIBGIT2TEMP" \
  && make \
  && make install
 
-#pyinstaller requirements start
-#must be preceded by libgit2 install
+# use pyenv
+ARG PYENV_VERSION=3.6.10
+ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer
+ENV PYENV_ROOT=/usr/local/pyenv
+ENV PATH=$PYENV_ROOT/bin:$PATH
+RUN umask 022 \
+ && curl -s -S -L "$PYENV_INSTALLER_URL" -o /usr/bin/pyenv-installer \
+ && chmod 0755 /usr/bin/pyenv-installer \
+ && /usr/bin/pyenv-installer \
+ && eval "$(pyenv init -)" \
+ && env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION \
+ && pyenv global $PYENV_VERSION
+
 COPY pyinstaller-requirements.txt /
-#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget.
-RUN wget -c https://bootstrap.pypa.io/get-pip.py \
- && yum -y install centos-release-scl \
- && yum -y install python27 \
- && chmod u+x ./get-pip.py \
- && scl enable python27 "./get-pip.py" \
- && scl enable python27 "pip -v install -r pyinstaller-requirements.txt"
+RUN eval "$(pyenv init -)" \
+ && pip -v install --upgrade pip \
+ && pip -v install -r pyinstaller-requirements.txt
 
 #fpm package making requirements start
-RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
- && scl enable rh-ruby23 "gem install --no-ri --no-rdoc fpm"
+RUN yum install -y centos-release-scl scl-utils
+RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel
+RUN scl enable rh-ruby23 "gem install --no-ri --no-rdoc fpm"
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v3.0.8
-ENV HUBBLE_VERSION=3.0.8
+ARG HUBBLE_CHECKOUT=v4.0.0
+ENV HUBBLE_VERSION=4.0.0
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_URL=https://github.com/hubblestack/hubble
 ENV HUBBLE_DESCRIPTION="Hubble is a modular, open-source, security & compliance auditing framework which is built in python, using SaltStack as a library."
@@ -167,7 +175,7 @@ ENV _INCLUDE_PATH=""
 ENV LD_LIBRARY_PATH=/opt/hubble/lib:/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
  && cd "$HUBBLE_SRC_PATH" \
- && git checkout "$HUBBLE_CHECKOUT" \
+ && git checkout -B hubble-build && git reset --hard "$HUBBLE_CHECKOUT" && git clean -dfx \
  && cp -rf "$HUBBLE_SRC_PATH" /hubble_build \
  && sed -i "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" /hubble_build/hubblestack/__init__.py \
  && sed -i "s/COMMIT_NOT_SET/`git describe`/g" /hubble_build/hubblestack/__init__.py
@@ -176,7 +184,8 @@ VOLUME /data
 WORKDIR /hubble_build
 ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ]
 CMD [ "if [ -f /data/hubble_buildinfo ] ; then echo \"\" >> /hubble_build/hubblestack/__init__.py ; cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py; fi \
-    && scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} --runtime-hook=pkg/pyinstaller-runtimehooks/pathopthubble.py hubble.py' \
+    && eval \"$(pyenv init -)\" \
+    && pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} --runtime-hook=pkg/pyinstaller-runtimehooks/pathopthubble.py hubble.py \
     && mkdir -p /var/log/hubble_osquery/backuplogs \
 # hubble default configuration file
     && cp -rf /hubble_build/conf/hubble /etc/hubble/ \

pkg/centos6/Dockerfile

@@ -134,12 +134,22 @@ RUN mkdir -p "$LIBGIT2TEMP" \
  && make \
  && make install
 
-#pyinstaller requirements start
-#must be preceded by libgit2 install
+# use pyenv
+ARG PYENV_VERSION=3.6.10
+ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer
+ENV PYENV_ROOT=/usr/local/pyenv
+ENV PATH=$PYENV_ROOT/bin:$PATH
+RUN umask 022 \
+ && curl -s -S -L "$PYENV_INSTALLER_URL" -o /usr/bin/pyenv-installer \
+ && chmod 0755 /usr/bin/pyenv-installer \
+ && /usr/bin/pyenv-installer \
+ && eval "$(pyenv init -)" \
+ && env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION \
+ && pyenv global $PYENV_VERSION
+
 COPY pyinstaller-requirements.txt /
-#default python-pip from yum does not like upgrading itself from pip. looking for better options other than wget.
-RUN wget -c https://bootstrap.pypa.io/get-pip.py \
- && python get-pip.py \
+RUN eval "$(pyenv init -)" \
+ && pip -v install --upgrade pip \
  && pip -v install -r pyinstaller-requirements.txt
 
 #fpm package making requirements start
@@ -149,8 +159,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v3.0.8
-ENV HUBBLE_VERSION=3.0.8
+ARG HUBBLE_CHECKOUT=v4.0.0
+ENV HUBBLE_VERSION=4.0.0
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_URL=https://github.com/hubblestack/hubble
 ENV HUBBLE_DESCRIPTION="Hubble is a modular, open-source, security & compliance auditing framework which is built in python, using SaltStack as a library."
@@ -163,7 +173,7 @@ ENV _INCLUDE_PATH=""
 ENV LD_LIBRARY_PATH=/opt/hubble/lib:/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
 RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
  && cd "$HUBBLE_SRC_PATH" \
- && git checkout "$HUBBLE_CHECKOUT" \
+ && git checkout -B hubble-build && git reset --hard "$HUBBLE_CHECKOUT" && git clean -dfx \
  && cp -rf "$HUBBLE_SRC_PATH" /hubble_build \
  && sed -i "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" /hubble_build/hubblestack/__init__.py \
  && sed -i "s/COMMIT_NOT_SET/`git describe`/g" /hubble_build/hubblestack/__init__.py
@@ -172,6 +182,7 @@ VOLUME /data
 WORKDIR /hubble_build
 ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ]
 CMD [ "if [ -f /data/hubble_buildinfo ] ; then echo \"\" >> /hubble_build/hubblestack/__init__.py ; cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py; fi \
+    && eval \"$(pyenv init -)\" \
     && pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} --runtime-hook=pkg/pyinstaller-runtimehooks/pathopthubble.py hubble.py \
     && mkdir -p /var/log/hubble_osquery/backuplogs \
 # hubble default configuration file