Skip to content

Commit 6c1a94b

Browse files
jetterojettero
authored
Merge pull request #848 from fossam/osquery_dockerfiles
osquery dockerfiles and augeas folder fix
2 parents ece5637 + c54a685 commit 6c1a94b

File tree

10 files changed

+203
-616
lines changed

10 files changed

+203
-616
lines changed

‎hubblestack/extmods/modules/nebula_osquery.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,12 @@ def _run_osqueryi_query(query, query_sql, timing, verbose):
192192
Run the osqueryi query in query_sql and return the result
193193
"""
194194
max_file_size = 104857600
195+
augeas_lenses = '/opt/osquery/lenses'
195196
query_ret = {'result': True}
196197

197198
# Run the osqueryi query
198-
cmd = [__grains__['osquerybinpath'], '--read_max', max_file_size, '--json', query_sql]
199+
cmd = [__grains__['osquerybinpath'], '--read_max', max_file_size, '--json',
200+
'--augeas_lenses', augeas_lenses, query_sql]
199201

200202
time_start = time.time()
201203
res = __salt__['cmd.run_all'](cmd, timeout=10000)

‎pkg/amazonlinux2016.09/Dockerfile

+18-68
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,26 @@
1-
# This Dockerfile aims to make building Hubble v2 packages easier.
2-
# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile
3-
# 2. docker build -t <image_name> .
4-
# The resulting image is ready to run the pyinstaller on container start and drop hubble<version>-coreos.tar.gz
5-
# in the /data directory. Mount /data volume into a directory on the host to access the package.
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg=HUBBLE_CHECKOUT=<tag or commit> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-al.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
610
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
7-
# Requires docker 17.05 or higher
811

9-
# Set this argument to "local" if you want to build osquery for local code.
10-
# In that case, osquery folder must exist besides Dockerfile
11-
ARG OSQUERY_BUILD_ENV=remote
12-
13-
#--------------- TEMP CONTAINER FOR LOCAL OSQUERY -------------------------
14-
FROM alpine as osquery_local
15-
ONBUILD COPY osquery /osquery
16-
ONBUILD RUN echo "Copying osquery from local folder"
17-
18-
19-
20-
#--------------- TEMP CONTAINER FOR GIT OSQUERY ----------------------------
21-
FROM alpine/git as osquery_remote
22-
#to pin osquery to a different version change the following envirnment variable
23-
ENV OSQUERY_SRC_VERSION=3.3.2
24-
ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
25-
ONBUILD RUN cd / \
26-
&& git clone "$OSQUERY_GIT_URL" \
27-
&& cd osquery/ \
28-
&& git checkout "$OSQUERY_SRC_VERSION" \
29-
&& echo "Fetching osquery from git"
30-
31-
32-
#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
33-
FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
34-
35-
36-
#--------------- ACTUAL DOCKERFILE FOR BUILD CREATION --------------------------
3712
FROM amazonlinux:2016.09
3813

3914
RUN yum makecache fast && yum -y update
4015

4116
#paths that hubble or hubble parts need in the package
4217
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
43-
#osquery build start
44-
#osquery should be built first since requirements for other packages can interfere with osquery dependencies
45-
#to build, osquery scripts want sudo and a user to sudo with.
46-
ENV OSQUERY_BUILD_USER=osquerybuilder
47-
RUN yum -y install git make python ruby sudo which
48-
RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \
49-
&& sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers
50-
COPY --from=osquery_image /osquery /home/"$OSQUERY_BUILD_USER"/osquery
51-
RUN mkdir -p /usr/local/osquery/ \
52-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \
53-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /home/"$OSQUERY_BUILD_USER"/osquery
54-
USER $OSQUERY_BUILD_USER
55-
ENV SKIP_TESTS=1
56-
RUN cd /home/"$OSQUERY_BUILD_USER"/osquery \
57-
&& make sysprep \
58-
#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
59-
&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
60-
&& make deps \
61-
&& make \
62-
&& make strip
63-
USER root
64-
RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /opt/osquery \
65-
&& cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery/hubble_osqueryd \
66-
&& chown -R root. /opt/osquery \
67-
&& chmod -R 500 /opt/osquery/* \
68-
#put augeas lenses into the default directory that we changed earlier
69-
&& mkdir -p /opt/osquery/lenses \
70-
&& cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
71-
&& chmod -R 400 /opt/osquery/lenses/*
72-
RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
7318

7419
#install packages that should be needed for ligbit2 compilation and successful pyinstaller run
75-
RUN yum -y install \
76-
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
77-
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
78-
make cmake gcc python-devel python-setuptools wget openssl
20+
RUN yum -y install git \
21+
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
22+
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
23+
make cmake gcc python-devel python-setuptools wget openssl
7924

8025
#libcurl install start
8126
#install libcurl to avoid depending on host version
@@ -158,6 +103,11 @@ RUN umask 022 \
158103
RUN eval "$(pyenv init -)" \
159104
&& pip -v install --upgrade pip
160105

106+
#extract osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg
107+
ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar
108+
ADD ${OSQUERY_TAR_FILENAME} /opt/osquery/
109+
RUN /opt/osquery/osqueryi --version
110+
161111
#pyinstaller start
162112
#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
163113
#use the following variables to choose the version of hubble

‎pkg/centos6/Dockerfile

+16-66
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,27 @@
1-
# This Dockerfile aims to make building Hubble v2 packages easier.
2-
# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile
3-
# 2. docker build -t <image_name> .
4-
# The resulting image is ready to run the pyinstaller on container start and drop hubble<version>-coreos.tar.gz
5-
# in the /data directory. Mount /data volume into a directory on the host to access the package.
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg=HUBBLE_CHECKOUT=<tag or commit> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-centos6.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
610
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
7-
# Requires docker 17.05 or higher
811

9-
# Set this argument to "local" if you want to build osquery for local code.
10-
# In that case, osquery folder must exist besides Dockerfile
11-
ARG OSQUERY_BUILD_ENV=remote
12-
13-
#--------------- TEMP CONTAINER FOR LOCAL OSQUERY -------------------------
14-
FROM alpine as osquery_local
15-
ONBUILD COPY osquery /osquery
16-
ONBUILD RUN echo "Copying osquery from local folder"
17-
18-
19-
20-
#--------------- TEMP CONTAINER FOR GIT OSQUERY ----------------------------
21-
FROM alpine/git as osquery_remote
22-
#to pin osquery to a different version change the following envirnment variable
23-
ENV OSQUERY_SRC_VERSION=3.3.2
24-
ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
25-
ONBUILD RUN cd / \
26-
&& git clone "$OSQUERY_GIT_URL" \
27-
&& cd osquery/ \
28-
&& git checkout "$OSQUERY_SRC_VERSION" \
29-
&& echo "Fetching osquery from git"
30-
31-
32-
#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
33-
FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
34-
35-
36-
#--------------- ACTUAL DOCKERFILE FOR BUILD CREATION --------------------------
3712
FROM centos:6
3813

3914
RUN yum makecache fast && yum -y update
4015

4116
#paths that hubble or hubble parts need in the package
4217
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
43-
#osquery build start
44-
#osquery should be built first since requirements for other packages can interfere with osquery dependencies
45-
#to build, osquery scripts want sudo and a user to sudo with.
46-
ENV OSQUERY_BUILD_USER=osquerybuilder
47-
RUN yum -y install xz git make ruby sudo which python-argparse
48-
RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \
49-
&& sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers
50-
COPY --from=osquery_image /osquery /home/"$OSQUERY_BUILD_USER"/osquery
51-
RUN mkdir -p /usr/local/osquery/ \
52-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \
53-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /home/"$OSQUERY_BUILD_USER"/osquery
54-
USER $OSQUERY_BUILD_USER
55-
ENV SKIP_TESTS=1
56-
RUN cd /home/"$OSQUERY_BUILD_USER"/osquery \
57-
&& make sysprep \
58-
#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
59-
&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
60-
&& make deps \
61-
&& make \
62-
&& make strip
63-
USER root
64-
RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /opt/osquery \
65-
&& cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery/hubble_osqueryd \
66-
&& chown -R root. /opt/osquery \
67-
&& chmod -R 500 /opt/osquery/* \
68-
#put augeas lenses into the default directory that we changed earlier
69-
&& mkdir -p /opt/osquery/lenses \
70-
&& cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
71-
&& chmod -R 400 /opt/osquery/lenses/*
72-
RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
7318

7419
#install packages that should be needed for ligbit2 compilation and successful pyinstaller run
75-
RUN yum -y install \
20+
RUN yum -y install git \
7621
libffi-devel openssl-devel libxml2-devel libxslt-devel libffi \
7722
libssh2-devel autoconf automake libtool libjpeg-devel zlib-devel \
7823
make cmake gcc wget openssl
79-
24+
8025
#libcurl install start
8126
#install libcurl to avoid depending on host version
8227
#requires autoconf libtool libssh2-devel zlib-devel autoconf
@@ -159,6 +104,11 @@ RUN umask 022 \
159104
RUN eval "$(pyenv init -)" \
160105
&& pip -v install --upgrade pip
161106

107+
#extract osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg
108+
ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar
109+
ADD ${OSQUERY_TAR_FILENAME} /opt/osquery/
110+
RUN /opt/osquery/osqueryi --version
111+
162112
#pyinstaller start
163113
#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
164114
#use the following variables to choose the version of hubble

‎pkg/centos7/Dockerfile

+18-68
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,26 @@
1-
# This Dockerfile aims to make building Hubble v2 packages easier.
2-
# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile
3-
# 2. docker build -t <image_name> .
4-
# The resulting image is ready to run the pyinstaller on container start and drop hubble<version>-coreos.tar.gz
5-
# in the /data directory. Mount /data volume into a directory on the host to access the package.
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg=HUBBLE_CHECKOUT=<tag or commit> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-centos7.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
610
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
7-
# Requires docker 17.05 or higher
811

9-
# Set this argument to "local" if you want to build osquery for local code.
10-
# In that case, osquery folder must exist besides Dockerfile
11-
ARG OSQUERY_BUILD_ENV=remote
12-
13-
#--------------- TEMP CONTAINER FOR LOCAL OSQUERY -------------------------
14-
FROM alpine as osquery_local
15-
ONBUILD COPY osquery /osquery
16-
ONBUILD RUN echo "Copying osquery from local folder"
17-
18-
19-
20-
#--------------- TEMP CONTAINER FOR GIT OSQUERY ----------------------------
21-
FROM alpine/git as osquery_remote
22-
#to pin osquery to a different version change the following envirnment variable
23-
ENV OSQUERY_SRC_VERSION=3.3.2
24-
ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
25-
ONBUILD RUN cd / \
26-
&& git clone "$OSQUERY_GIT_URL" \
27-
&& cd osquery/ \
28-
&& git checkout "$OSQUERY_SRC_VERSION" \
29-
&& echo "Fetching osquery from git"
30-
31-
32-
#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
33-
FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
34-
35-
36-
#--------------- ACTUAL DOCKERFILE FOR BUILD CREATION --------------------------
3712
FROM centos:7
3813

3914
RUN yum makecache fast && yum -y update
4015

4116
#paths that hubble or hubble parts need in the package
4217
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
43-
#osquery build start
44-
#osquery should be built first since requirements for other packages can interfere with osquery dependencies
45-
#to build, osquery scripts want sudo and a user to sudo with.
46-
ENV OSQUERY_BUILD_USER=osquerybuilder
47-
RUN yum -y install git make python ruby sudo which
48-
RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \
49-
&& sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers
50-
COPY --from=osquery_image /osquery /home/"$OSQUERY_BUILD_USER"/osquery
51-
RUN mkdir -p /usr/local/osquery/ \
52-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \
53-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /home/"$OSQUERY_BUILD_USER"/osquery
54-
USER $OSQUERY_BUILD_USER
55-
ENV SKIP_TESTS=1
56-
RUN cd /home/"$OSQUERY_BUILD_USER"/osquery \
57-
&& make sysprep \
58-
#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
59-
&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
60-
&& make deps \
61-
&& make \
62-
&& make strip
63-
USER root
64-
RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /opt/osquery \
65-
&& cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery/hubble_osqueryd \
66-
&& chown -R root. /opt/osquery \
67-
&& chmod -R 500 /opt/osquery/* \
68-
#put augeas lenses into the default directory that we changed earlier
69-
&& mkdir -p /opt/osquery/lenses \
70-
&& cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
71-
&& chmod -R 400 /opt/osquery/lenses/*
72-
RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
7318

7419
#install packages that should be needed for ligbit2 compilation and successful pyinstaller run
75-
RUN yum -y install \
76-
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
77-
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
78-
make cmake gcc python-devel python-setuptools wget openssl
20+
RUN yum -y install git \
21+
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
22+
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
23+
make cmake gcc python-devel python-setuptools wget openssl
7924

8025
#libcurl install start
8126
#install libcurl to avoid depending on host version
@@ -158,6 +103,11 @@ RUN umask 022 \
158103
RUN eval "$(pyenv init -)" \
159104
&& pip -v install --upgrade pip
160105

106+
#extract osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg
107+
ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar
108+
ADD ${OSQUERY_TAR_FILENAME} /opt/osquery/
109+
RUN /opt/osquery/osqueryi --version
110+
161111
#pyinstaller start
162112
#commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
163113
#use the following variables to choose the version of hubble

0 commit comments

Comments
 (0)