flatten (re)pyinstaller

Author: jettero

package-requirements.txt

@@ -0,0 +1,2 @@
+pyinstaller
+pyinstaller-hooks

pkg/.gitignore

@@ -0,0 +1 @@
+osquery

pkg/amazonlinux2016.09/Dockerfile

@@ -6,7 +6,7 @@
 # To run the container:  docker run -it --rm -v `pwd`:/data <image_name>
 # Requires docker 17.05 or higher
 
-# Set this arguement to "local" if you want to build osquery for local code.
+# Set this argument to "local" if you want to build osquery for local code.
 # In that case, osquery folder must exist besides Dockerfile
 ARG OSQUERY_BUILD_ENV=remote
 
@@ -29,7 +29,7 @@ ONBUILD RUN cd / \
  && echo "Fetching osquery from git"
 
 
-#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) --------------
+#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
 FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
 
 
@@ -40,7 +40,6 @@ RUN yum makecache fast && yum -y update
 
 #paths that hubble or hubble parts need in the package
 RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
-
 #osquery build start
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
@@ -74,10 +73,9 @@ RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
 
 #install packages that should be needed for ligbit2 compilation and successful pyinstaller run
 RUN yum -y install  \
-               python27-devel libffi-devel openssl-devel libssh2-devel autoconf automake libtool \
-               libxml2-devel libxslt-devel libjpeg-devel \
-               zlib-devel make cmake python27-setuptools \
-               gcc python-devel python-setuptools wget openssl
+               libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
+               libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
+               make cmake gcc python-devel python-setuptools wget openssl
 
 #libcurl install start
 #install libcurl to avoid depending on host version
@@ -136,27 +134,29 @@ RUN mkdir -p "$LIBGIT2TEMP" \
  && make \
  && make install
 
+#fpm package making requirements start
+RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
+ && gem install --no-ri --no-rdoc fpm
+
+# things we may need to build a python
+RUN yum install -y bzip2-devel
+
 # use pyenv
 ARG PYENV_VERSION=3.6.10
 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer
-ENV PYENV_ROOT=/usr/local/pyenv
+ENV PYENV_ROOT=/opt/hubble/pyenv
 ENV PATH=$PYENV_ROOT/bin:$PATH
+ENV PYTHON_CONFIGURE_OPTS="--enable-shared"
 RUN umask 022 \
  && curl -s -S -L "$PYENV_INSTALLER_URL" -o /usr/bin/pyenv-installer \
  && chmod 0755 /usr/bin/pyenv-installer \
  && /usr/bin/pyenv-installer \
  && eval "$(pyenv init -)" \
- && env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION \
+ && pyenv install $PYENV_VERSION \
  && pyenv global $PYENV_VERSION
 
-COPY pyinstaller-requirements.txt /
 RUN eval "$(pyenv init -)" \
- && pip -v install --upgrade pip \
- && pip -v install -r pyinstaller-requirements.txt
-
-#fpm package making requirements start
-RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
- && gem install --no-ri --no-rdoc fpm
+ && pip -v install --upgrade pip
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
@@ -173,58 +173,21 @@ ENV _HOOK_DIR="./pkg/"
 ENV _BINARY_LOG_LEVEL="INFO"
 ENV _INCLUDE_PATH=""
 ENV LD_LIBRARY_PATH=/opt/hubble/lib:/lib:/lib64:/usr/lib:/usr/lib64:/usr/local/lib:/usr/local/lib64
-RUN git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
+RUN set -x; git clone "$HUBBLE_GIT_URL" "$HUBBLE_SRC_PATH" \
  && cd "$HUBBLE_SRC_PATH" \
  && git checkout -B hubble-build && git reset --hard "$HUBBLE_CHECKOUT" && git clean -dfx \
  && cp -rf "$HUBBLE_SRC_PATH" /hubble_build \
  && sed -i "s/BRANCH_NOT_SET/${HUBBLE_CHECKOUT}/g" /hubble_build/hubblestack/__init__.py \
  && sed -i "s/COMMIT_NOT_SET/`git describe`/g" /hubble_build/hubblestack/__init__.py
+RUN eval "$(pyenv init -)" \
+ && cd "$HUBBLE_SRC_PATH" \
+ && python setup.py egg_info \
+ && pip install --upgrade \
+        -r hubblestack.egg-info/requires.txt \
+        -r optional-requirements.txt \
+        -r package-requirements.txt
 RUN mkdir /data
 VOLUME /data
 WORKDIR /hubble_build
-ENTRYPOINT [ "/bin/bash", "-o", "xtrace", "-c" ]
-CMD [ "if [ -f /data/hubble_buildinfo ] ; then echo \"\" >> /hubble_build/hubblestack/__init__.py ; cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py; fi \
-    && eval \"$(pyenv init -)\" \
-    && pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --additional-hooks-dir=${_HOOK_DIR} --runtime-hook=pkg/pyinstaller-runtimehooks/pathopthubble.py hubble.py \
-    && mkdir -p /var/log/hubble_osquery/backuplogs \
-# hubble default configuration file
-    && cp -rf /hubble_build/conf/hubble /etc/hubble/ \
-    && cp -rf /hubble_build/conf/hubble-profile.sh /etc/profile.d/ \
-    && cp -pr /hubble_build/dist/hubble /opt/hubble/hubble-libs \
-    && ln -s /opt/hubble/hubble-libs/hubble /opt/hubble/hubble \
-    # make sure rpm shared libs are taken out to avoid mismatch between rpm database and shared libs that pyinstaller includes
-    && rm -rf /opt/hubble/hubble-libs/librpm* \
-#rpm pkg start
-    && tar -cPvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz /etc/hubble /opt/hubble /opt/osquery /etc/profile.d/hubble-profile.sh /var/log/hubble_osquery/backuplogs \
-    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
-    && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
-    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \
-    && if [ -f /data/hubble-autostart ] ; then mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/cron.d ; fi \
-    && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \
-    && if [ -f /data/hubble-autostart ] ; then cp /hubble_build/pkg/hubble-autostart /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/cron.d/ ; fi \
-    && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \
-#during container run, if a configuration file exists in a /data copy it over the existing one so it would be
-#possile to optionally include a custom one with the package
-    && if [ -f /data/hubble ] ; then cp /data/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ ; fi \
-#also bring in anything from a /data/opt/ directory so we can bundle other executables if needed
-    && if [ -d /data/opt ] ; then cp -r /data/opt/* /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/opt/ ; fi \
-    && cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
-    && mkdir -p usr/bin \
-#symlink to have hubble binary in path
-    && ln -s /opt/hubble/hubble usr/bin/hubble \
-#fpm start
-    && fpm -s dir -t rpm \
-       -n hubblestack \
-       -v ${HUBBLE_VERSION} \
-       --iteration ${HUBBLE_ITERATION} \
-       --url ${HUBBLE_URL} \
-       --description \"${HUBBLE_DESCRIPTION}\" \
-       --rpm-summary \"${HUBBLE_SUMMARY}\" \
-       --after-install /hubble_build/conf/afterinstall.sh \
-       --after-upgrade /hubble_build/conf/afterupgrade.sh \
-       --before-remove /hubble_build/conf/beforeremove.sh \
-       etc opt usr /var/log/hubble_osquery/backuplogs \
-#edit to change iteration number, if necessary
-    && cp hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm \
-    && openssl dgst -sha256 /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm \
-                          > /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm.sha256" ]
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]

pkg/amazonlinux2016.09/entrypoint.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+
+eval "$(pyenv init -)"
+
+# locate some pyenv things
+pyenv_prefix="$(pyenv prefix)"
+python_binary="$(pyenv which python)"
+while [ -L "$python_binary" ]
+do python_binary="$(readlink -f "$python_binary")"
+done
+
+# if ENTRYPOINT is given a CMD other than nothing
+# abort here and do that other CMD
+if [ $# -gt 0 ]
+then exec "$@"
+fi
+
+# from now on, exit on error (rather than && every little thing)
+PS4=$'-------------=: '
+set -x -e
+
+cp -rf "$HUBBLE_SRC_PATH"/* /hubble_build/
+
+# possibly replace the version file
+if [ -f /data/hubble_buildinfo ]; then
+    echo >> /hubble_build/hubblestack/__init__.py
+    cat /data/hubble_buildinfo >> /hubble_build/hubblestack/__init__.py
+fi 2>/dev/null
+
+
+cd /hubble_build
+
+# we may have preinstalled requirements that may need upgrading
+# pip install . might not upgrade/downgrade the requirements
+python setup.py egg_info
+pip install --upgrade \
+    -r hubblestack.egg-info/requires.txt \
+    -r optional-requirements.txt \
+    -r package-requirements.txt
+
+[ -f ${_HOOK_DIR:-./pkg}/hook-hubblestack.py ] || exit 1
+
+rm -rf build dist /opt/hubble/hubble-libs /hubble_build/hubble.spec
+export LD_LIBRARY_PATH=$pyenv_prefix/lib:/opt/hubble/lib:/opt/hubble-libs
+export LD_RUN_PATH=$LD_LIBRARY_PATH
+pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL:-INFO} \
+    --additional-hooks-dir ${_HOOK_DIR:-./pkg} \
+    --runtime-hook pkg/runtime-hooks.py \
+    ./hubble.py 2>&1 | tee /tmp/pyinstaller.log
+
+cp -pr dist/hubble /opt/hubble/hubble-libs
+
+cat > /opt/hubble/hubble << EOF
+#!/bin/bash
+exec /opt/hubble/hubble-libs/hubble "\$@"
+exit 1
+EOF
+chmod 0755 /opt/hubble/hubble
+
+[ -d /data/last-build.4 ] && rm -rf /data/last-build.4
+[ -d /data/last-build.3 ] && mv -v /data/last-build.3 /data/last-build.4
+[ -d /data/last-build.2 ] && mv -v /data/last-build.2 /data/last-build.3
+[ -d /data/last-build.1 ] && mv -v /data/last-build.1 /data/last-build.2
+cp -va build/ /data/last-build.1
+mv /tmp/pyinstaller.log /data/last-build.1
+cp -va /entrypoint.sh /data/last-build.1
+
+mkdir -p /var/log/hubble_osquery/backuplogs
+
+mkdir -p /usr/lib/systemd/system
+mkdir -p /etc/profile.d
+mkdir -p /etc/hubble
+
+cp -v /hubble_build/pkg/hubble.service /usr/lib/systemd/system/
+cp -v /hubble_build/conf/hubble-profile.sh /etc/profile.d/
+
+if [ -f /data/hubble ]
+then cp -v /data/hubble /etc/hubble/
+else cp -v /hubble_build/conf/hubble /etc/hubble/
+fi
+
+if [ "X$TEST_BINARIES" = X1 ]; then
+    # weakly test the new bin
+    ./dist/hubble/hubble --version
+
+    # does it still work if we call it in its new home?
+    /opt/hubble/hubble-libs/hubble --version
+
+    # how about if it's via non-home location?
+    /opt/hubble/hubble --version
+
+    # lastly, can we actually use salt grains and other lazy loader items?
+    /opt/hubble/hubble-libs/hubble -vvv grains.get hubble_version
+    /opt/hubble/hubble -vvv grains.get hubble_version
+fi
+
+if [ "X$NO_TAR" = X1 ]; then
+    echo "exiting (as requested by NO_TAR=$NO_TAR) without pre-tar-ing package"
+    exit 0
+fi 2>/dev/null
+
+# rpm pkg start
+tar -cSPvvzf /data/hubblestack-${HUBBLE_VERSION}.tar.gz \
+    --exclude opt/hubble/pyenv \
+    /etc/hubble /opt/hubble /opt/osquery \
+    /etc/profile.d/hubble-profile.sh \
+    /usr/lib/systemd/system/hubble.service \
+    /var/log/hubble_osquery/backuplogs \
+    2>&1 | tee /hubble_build/rpm-pkg-start-tar.log
+
+PKG_STRUCT_DIR=/hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}
+mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}
+tar -xSzvvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C $PKG_STRUCT_DIR
+
+# also bring in anything from a /data/opt/ directory so we can bundle other executables if needed
+if [ -d /data/opt ]
+then cp -r /data/opt/* /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/opt/
+fi
+
+# symlink to have hubble binary in path
+cd /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}
+mkdir -p usr/bin
+ln -s /opt/hubble/hubble usr/bin/hubble
+
+if [ "X$NO_FPM" = X1 ]; then
+    echo "exiting (as requested by NO_FPM=$NO_FPM) without building package"
+    exit 0
+fi
+
+# fpm start
+fpm -s dir -t rpm \
+    -n hubblestack \
+    -v ${HUBBLE_VERSION} \
+    --iteration ${HUBBLE_ITERATION} \
+    --url ${HUBBLE_URL} \
+    --description "${HUBBLE_DESCRIPTION}" \
+    --rpm-summary "${HUBBLE_SUMMARY}" \
+    --after-install /hubble_build/conf/afterinstall-systemd.sh \
+    --after-upgrade /hubble_build/conf/afterupgrade-systemd.sh \
+    --before-remove /hubble_build/conf/beforeremove.sh \
+    etc/hubble opt usr /var/log/hubble_osquery/backuplogs
+
+# edit to change iteration number, if necessary
+PKG_BASE_NAME=hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}
+PKG_OUT_EXT=x86_64.rpm
+PKG_FIN_EXT=all609.$PKG_OUT_EXT
+PKG_ONAME="$PKG_BASE_NAME.$PKG_OUT_EXT"
+PKG_FNAME="$PKG_BASE_NAME.$PKG_FIN_EXT"
+
+cp -va "$PKG_ONAME" /data/"$PKG_FNAME"
+openssl dgst -sha256 /data/"$PKG_FNAME" > /data/"$PKG_FNAME".sha256