Merge pull request #807 from jettero/4.0-pkg-fix

4.0 pkg fix

Author: jettero

.travis.yml

@@ -1,21 +1,31 @@
 
+version: 1.0
+
 language: python
 
 cache: pip
 
 env:
-  - 'PYLINT_ENABLE=1'
+  global:
+    LIBGIT2_VERSION: "0.26.5"
+    LIBGIT2_SRC_URL: https://github.com/libgit2/libgit2/archive/v${LIBGIT2_VERSION}.tar.gz
+    LIBGIT2_DIR_NAME: libgit2-${LIBGIT2_VERSION}
+    LIBGIT2_TAR_NAME: ${LIBGIT2_DIR_NAME}.tar.gz
+    LIBGIT2: ~/libgit2/_install
+    LD_LIBRARY_PATH: ${LIBGIT2}/lib:${LD_LIBRARY_PATH}
 
 python:
-  - "2.7"
-  - "3.6"
-
-matrix:
-  allow_failures:
-    - python: "3.6"
+  - "3.6.10"
+  - "3.7"
+  - "3.8"
 
 install:
-  - pip install -r test-requirements.txt
+  - pip --upgarde pip
+  - ./mk-requires.sh
+  - pip install -r requirements.txt
 
 script:
-  - bash tests/travis-runner.sh
+  - wget -q "$LIBGIT2_SRC_URL" -O $LIBGIT2_TAR_NAME
+  - tar -xzf $LIBGIT2_TAR_NAME
+  - (cd $LIBGIT2_DIR_NAME; cmake . -DMAKE_INSTALL_PREFIX=${LIBGIT2} && make -j 5 install)
+  - pytest tests/unittests

contrib/gen-pretend-certs.py

@@ -49,10 +49,10 @@ def gen_CA(fname='ca-root', cn='ca-root', path_length=0, authority=None, pdir=DE
     private_key = genkey(**args)
     public_key  = private_key.public_key()
 
-    with open(os.path.join(pdir, fname + '.key'), 'w') as fh:
+    with open(os.path.join(pdir, fname + '.key'), 'wb') as fh:
         fh.write( as_pem(private_key) )
 
-    with open(os.path.join(pdir, fname + '.unsigned'), 'w') as fh:
+    with open(os.path.join(pdir, fname + '.unsigned'), 'wb') as fh:
         fh.write( as_pem(public_key) )
 
     ksec_100 = datetime.timedelta(0, 100e3, 0)
@@ -114,7 +114,7 @@ def gen_CA(fname='ca-root', cn='ca-root', path_length=0, authority=None, pdir=DE
 
     certificate = builder.sign(**signing_args)
 
-    with open(os.path.join(pdir, fname + '.crt'), 'w') as fh:
+    with open(os.path.join(pdir, fname + '.crt'), 'wb') as fh:
         fh.write( as_pem(certificate) )
 
     return Authority(private_key, certificate)
@@ -126,10 +126,10 @@ def gen_leaf(authority, fname_template='{}', cn='Certy Cert McCertFace', pdir=DE
     private_name = fname_template.format('private')
     public_name = fname_template.format('public')
 
-    with open(os.path.join(pdir, private_name + '.key'), 'w') as fh:
+    with open(os.path.join(pdir, private_name + '.key'), 'wb') as fh:
         fh.write( as_pem(private_key) )
 
-    with open(os.path.join(pdir, public_name + '.unsigned'), 'w') as fh:
+    with open(os.path.join(pdir, public_name + '.unsigned'), 'wb') as fh:
         fh.write( as_pem(public_key) )
 
     ksec_100 = datetime.timedelta(0, 100e3, 0)
@@ -186,7 +186,7 @@ def gen_leaf(authority, fname_template='{}', cn='Certy Cert McCertFace', pdir=DE
 
     certificate = builder.sign(**signing_args)
 
-    with open(os.path.join(pdir, public_name + '.crt'), 'w') as fh:
+    with open(os.path.join(pdir, public_name + '.crt'), 'wb') as fh:
         fh.write( as_pem(certificate) )
 
     return Authority(private_key, certificate)
@@ -203,9 +203,9 @@ def main(root_cn, int1_cn, int2_cn, **args):
     lf1 = gen_leaf(cn='Certy Cert #1', fname_template='{}-1', authority=ia1, **args)
     lf2 = gen_leaf(cn='Certy Cert #2', fname_template='{}-2', authority=ia2, **args)
 
-    with open(os.path.join(args['pdir'], 'bundle.pem'), 'w') as ofh:
+    with open(os.path.join(args['pdir'], 'bundle.pem'), 'wb') as ofh:
         for i in range(1,3):
-            with open(os.path.join(args['pdir'], 'intermediate-{}.crt'.format(i)), 'r') as ifh:
+            with open(os.path.join(args['pdir'], 'intermediate-{}.crt'.format(i)), 'rb') as ifh:
                 ofh.write(ifh.read())
 
 if __name__ == '__main__':

cp-pyinstaller.bash

@@ -56,3 +56,9 @@ def verify(*targets, **kw):
 
     return dict(HuS.verify_files(targets, mfname=mfname, sfname=sfname,
         public_crt=public_crt, ca_crt=ca_crt))
+
+def enumerate():
+    """ enumerate installed certificates """
+
+    x509 = HuS.X509AwareCertBucket()
+    return [ ' '.join(x.split()[1:]) for x in x509.trusted ]

hubblestack/extmods/modules/signing.py

@@ -230,7 +230,7 @@ def authenticate_cert(self):
             return STATUS.VERIFIED
         return STATUS.UNKNOWN
 
-    def __init__(self, public_crt, ca_crt):
+    def __init__(self, public_crt=None, ca_crt=None):
         try:
             import hubblestack.pre_packaged_certificates as HPPC
             # iff we have hardcoded certs then we're meant to ignore any other
@@ -244,6 +244,11 @@ def __init__(self, public_crt, ca_crt):
         except ImportError:
             pass
 
+        if public_crt is None:
+            public_crt = Options.public_crt
+        if ca_crt is None:
+            ca_crt = Options.ca_crt
+
         untrusted_crt = list()
 
         if isinstance(ca_crt, (list, tuple)):

hubblestack/utils/signing.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+D="$(dirname "$0")"
+F="$D/requirements.txt"
+
+echo "# This file was generated by $0" > "$F"
+echo "# $(date -u)" >> "$F"
+echo >> "$F"
+
+bash "$D/pkg/generate-requirements.sh" "$@" | tee -a "$F"
+
+CMD=( pip install --upgrade -r requirements.txt )
+
+read -ep "issue ${CMD[*]}? [Y/n]" YN
+[[ "$YN" =~ [Nn] ]] || "${CMD[@]}"

mk-requires.sh

@@ -0,0 +1,7 @@
+
+azure==4.0.0
+azure-storage-common==2.1.0
+azure-storage-blob==2.1.0
+
+boto3
+botocore

optional-requirements.txt

@@ -0,0 +1,2 @@
+pyinstaller==3.6
+pyinstaller-hooks

package-requirements.txt

@@ -0,0 +1 @@
+osquery

pkg/.gitignore

@@ -199,7 +199,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=v4.0.0
-ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
+ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_VERSION=4.0.0
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_URL=https://github.com/hubblestack/hubble

pkg/debian7/Dockerfile pkg/abandoned/debian7/Dockerfile

@@ -1,6 +1,8 @@
 pyinstaller==3.3.1
 Tornado>=4.0.0,<5.0.0
-Crypto
+crypto
+pycryptodome
+cryptography
 pyopenssl>=16.2.0
 argparse
 requests>=2.13.0

pkg/debian7/pyinstaller-requirements.txt pkg/abandoned/debian7/pyinstaller-requirements.txt

@@ -6,7 +6,7 @@
 # To run the container:  docker run -it --rm -v `pwd`:/data <image_name>
 # Requires docker 17.05 or higher
 
-# Set this arguement to "local" if you want to build osquery for local code.
+# Set this argument to "local" if you want to build osquery for local code.
 # In that case, osquery folder must exist besides Dockerfile
 ARG OSQUERY_BUILD_ENV=remote
 
@@ -29,7 +29,7 @@ ONBUILD RUN cd / \
  && echo "Fetching osquery from git"
 
 
-#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) --------------
+#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
 FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
 
 
@@ -40,7 +40,6 @@ RUN yum makecache fast && yum -y update
 
 #paths that hubble or hubble parts need in the package
 RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
-
 #osquery build start
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
@@ -74,10 +73,9 @@ RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
 
 #install packages that should be needed for ligbit2 compilation and successful pyinstaller run
 RUN yum -y install  \
-               python27-devel libffi-devel openssl-devel libssh2-devel autoconf automake libtool \
-               libxml2-devel libxslt-devel libjpeg-devel \
-               zlib-devel make cmake python27-setuptools \
-               gcc python-devel python-setuptools wget openssl
+               libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
+               libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
+               make cmake gcc python-devel python-setuptools wget openssl
 
 #libcurl install start
 #install libcurl to avoid depending on host version
@@ -136,27 +134,29 @@ RUN mkdir -p "$LIBGIT2TEMP" \
  && make \
  && make install
 
+#fpm package making requirements start
+RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
+ && gem install --no-ri --no-rdoc fpm
+
+# things we may need to build a python
+RUN yum install -y bzip2-devel
+
 # use pyenv
 ARG PYENV_VERSION=3.6.10
 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer
-ENV PYENV_ROOT=/usr/local/pyenv
+ENV PYENV_ROOT=/opt/hubble/pyenv
 ENV PATH=$PYENV_ROOT/bin:$PATH
+ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
 RUN umask 022 \
  && curl -s -S -L "$PYENV_INSTALLER_URL" -o /usr/bin/pyenv-installer \
  && chmod 0755 /usr/bin/pyenv-installer \
  && /usr/bin/pyenv-installer \
  && eval "$(pyenv init -)" \
- && env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION \
+ && pyenv install $PYENV_VERSION \
  && pyenv global $PYENV_VERSION
 
-COPY pyinstaller-requirements.txt /
 RUN eval "$(pyenv init -)" \
- && pip -v install --upgrade pip \
- && pip -v install -r pyinstaller-requirements.txt
-
-#fpm package making requirements start
-RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
- && gem install --no-ri --no-rdoc fpm
+ && pip -v install --upgrade pip
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
@@ -167,64 +167,30 @@ ENV HUBBLE_ITERATION=1
 ENV HUBBLE_URL=https://github.com/hubblestack/hubble
 ENV HUBBLE_DESCRIPTION="Hubble is a modular, open-source, security & compliance auditing framework which is built in python, using SaltStack as a library."
 ENV HUBBLE_SUMMARY="Profile based on-demand auditing and monitoring tool"
-ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
+ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"
 ENV _BINARY_LOG_LEVEL="INFO"
 ENV _INCLUDE_PATH=""
 ENV LD_LIBRARY_PATH=/opt/hubble/lib:/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
-RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
+
+RUN set -x; git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
  && cd "$HUBBLE_SRC_PATH" \
- && git checkout -B hubble-build && git reset --hard "$HUBBLE_CHECKOUT" && git clean -dfx \
- && cp -rf "$HUBBLE_SRC_PATH" /hubble_build \
- && sed -i "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" /hubble_build/hubblestack/__init__.py \
- && sed -i "s/COMMIT_NOT_SET/`git describe`/g" /hubble_build/hubblestack/__init__.py
+ && git checkout -B hubble-build && git reset --hard "$HUBBLE_CHECKOUT" && git clean -dfx
+
+RUN cp -rf "$HUBBLE_SRC_PATH" /hubble_build \
+ && rm -rf /hubble_build/.git
+
+RUN cp /hubble_build/hubblestack/__init__.py /hubble_build/hubblestack/__init__.orig \
+ && sed -i -e "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" \
+           -e "s/COMMIT_NOT_SET/$(cd $HUBBLE_SRC_PATH; git describe --long --always --tags)/g" \
+           /hubble_build/hubblestack/__init__.py \
+ && cp /hubble_build/hubblestack/__init__.py /hubble_build/hubblestack/__init__.fixed
+
 RUN mkdir /data
 VOLUME /data
+
 WORKDIR /hubble_build
-ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ]
-CMD [ "if [ -f /data/hubble_buildinfo ] ; then echo \"\" >> /hubble_build/hubblestack/__init__.py ; cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py; fi \
-    && eval \"$(pyenv init -)\" \
-    && pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} --runtime-hook=pkg/pyinstaller-runtimehooks/pathopthubble.py hubble.py \
-    && mkdir -p /var/log/hubble_osquery/backuplogs \
-# hubble default configuration file
-    && cp -rf /hubble_build/conf/hubble /etc/hubble/ \
-    && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \
-    && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \
-    && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \
-    # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes
-    && rm -rf /opt/hubble/hubble-libs/librpm* \
-#rpm pkg start
-    && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /opt/hubble /opt/osquery /etc/profile.d/hubble-profile.sh /var/log/hubble_osquery/backuplogs \
-    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
-    && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
-    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \
-    && if [ -f /data/hubble-autostart ] ; then mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/cron.d ; fi \
-    && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \
-    && if [ -f /data/hubble-autostart ] ; then cp /hubble_build/pkg/hubble-autostart /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/cron.d/ ; fi \
-    && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \
-#during container run, if a configuration file exists in a /data copy it over the existing one so it would be
-#possile to optionally include a custom one with the package
-    && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \
-#also bring in anything from a /data/opt/ directory so we can bundle other executables if needed
-    && if [ -d /data/opt ] ; then cp -r /data/opt/* /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/opt/ ; fi \
-    && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
-    && mkdir -p usr/bin \
-#symlink to have hubble binary in path
-    && ln -s /opt/hubble/hubble usr/bin/hubble \
-#fpm start
-    && fpm -s dir -t rpm \
-       -n hubblestack \
-       -v ${HUBBLE_VERSION} \
-       --iteration ${HUBBLE_ITERATION} \
-       --url ${HUBBLE_URL} \
-       --description \"${HUBBLE_DESCRIPTION}\" \
-       --rpm-summary \"${HUBBLE_SUMMARY}\" \
-       --after-install /hubble_build/conf/afterinstall.sh \
-       --after-upgrade /hubble_build/conf/afterupgrade.sh \
-       --before-remove /hubble_build/conf/beforeremove.sh \
-       etc opt usr /var/log/hubble_osquery/backuplogs \
-#edit to change iteration number, if necessary
-    && cp hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm \
-    && openssl dgst -sha256 /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm \
-                          > /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm.sha256" ]
+
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]