Skip to content

Commit 3d6adfa

Browse files
committed
Separate osquery from Hubble build
This aims to shorten and simplify building and including osquery with Hubble. The new Dockerfile is introduced that builds osquery separately. The output file is ready to be included in Hubble build.
1 parent 74b0ea9 commit 3d6adfa

File tree

9 files changed

+228
-611
lines changed

9 files changed

+228
-611
lines changed

pkg/amazonlinux2016.09/Dockerfile

+22-68
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,35 @@
1-
# This Dockerfile aims to make building Hubble v2 packages easier.
2-
# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile
3-
# 2. docker build -t <image_name> .
4-
# The resulting image is ready to run the pyinstaller on container start and drop hubble<version>-coreos.tar.gz
5-
# in the /data directory. Mount /data volume into a directory on the host to access the package.
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg=HUBBLE_CHECKOUT=<tag or commit> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-al.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
610
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
7-
# Requires docker 17.05 or higher
811

9-
# Set this argument to "local" if you want to build osquery for local code.
10-
# In that case, osquery folder must exist besides Dockerfile
11-
ARG OSQUERY_BUILD_ENV=remote
12-
13-
#--------------- TEMP CONTAINER FOR LOCAL OSQUERY -------------------------
14-
FROM alpine as osquery_local
15-
ONBUILD COPY osquery /osquery
16-
ONBUILD RUN echo "Copying osquery from local folder"
17-
18-
19-
20-
#--------------- TEMP CONTAINER FOR GIT OSQUERY ----------------------------
21-
FROM alpine/git as osquery_remote
22-
#to pin osquery to a different version change the following envirnment variable
23-
ENV OSQUERY_SRC_VERSION=3.3.2
24-
ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
25-
ONBUILD RUN cd / \
26-
&& git clone "$OSQUERY_GIT_URL" \
27-
&& cd osquery/ \
28-
&& git checkout "$OSQUERY_SRC_VERSION" \
29-
&& echo "Fetching osquery from git"
30-
31-
32-
#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
33-
FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
34-
35-
36-
#--------------- ACTUAL DOCKERFILE FOR BUILD CREATION --------------------------
3712
FROM amazonlinux:2016.09
3813

3914
RUN yum makecache fast && yum -y update
4015

4116
#paths that hubble or hubble parts need in the package
4217
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
43-
#osquery build start
44-
#osquery should be built first since requirements for other packages can interfere with osquery dependencies
45-
#to build, osquery scripts want sudo and a user to sudo with.
46-
ENV OSQUERY_BUILD_USER=osquerybuilder
47-
RUN yum -y install git make python ruby sudo which
48-
RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \
49-
&& sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers
50-
COPY --from=osquery_image /osquery /home/"$OSQUERY_BUILD_USER"/osquery
51-
RUN mkdir -p /usr/local/osquery/ \
52-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \
53-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /home/"$OSQUERY_BUILD_USER"/osquery
54-
USER $OSQUERY_BUILD_USER
55-
ENV SKIP_TESTS=1
56-
RUN cd /home/"$OSQUERY_BUILD_USER"/osquery \
57-
&& make sysprep \
58-
#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
59-
&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
60-
&& make deps \
61-
&& make \
62-
&& make strip
63-
USER root
64-
RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /opt/osquery \
65-
&& cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery/hubble_osqueryd \
66-
&& chown -R root. /opt/osquery \
67-
&& chmod -R 500 /opt/osquery/* \
68-
#put augeas lenses into the default directory that we changed earlier
69-
&& mkdir -p /opt/osquery/lenses \
70-
&& cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
71-
&& chmod -R 400 /opt/osquery/lenses/*
72-
RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
18+
19+
#copy in and process osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg
20+
ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar
21+
COPY ${OSQUERY_TAR_FILENAME} /opt/osquery/${OSQUERY_TAR_FILENAME}
22+
RUN cd /opt/osquery \
23+
&& tar xf "$OSQUERY_TAR_FILENAME" \
24+
&& rm -f "$OSQUERY_TAR_FILENAME" \
25+
&& cd / \
26+
&& /opt/osquery/osqueryi --version
7327

7428
#install packages that should be needed for ligbit2 compilation and successful pyinstaller run
75-
RUN yum -y install \
76-
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
77-
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
78-
make cmake gcc python-devel python-setuptools wget openssl
29+
RUN yum -y install git \
30+
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
31+
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
32+
make cmake gcc python-devel python-setuptools wget openssl
7933

8034
#libcurl install start
8135
#install libcurl to avoid depending on host version

pkg/centos6/Dockerfile

+20-66
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,36 @@
1-
# This Dockerfile aims to make building Hubble v2 packages easier.
2-
# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile
3-
# 2. docker build -t <image_name> .
4-
# The resulting image is ready to run the pyinstaller on container start and drop hubble<version>-coreos.tar.gz
5-
# in the /data directory. Mount /data volume into a directory on the host to access the package.
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg=HUBBLE_CHECKOUT=<tag or commit> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-centos6.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
610
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
7-
# Requires docker 17.05 or higher
811

9-
# Set this argument to "local" if you want to build osquery for local code.
10-
# In that case, osquery folder must exist besides Dockerfile
11-
ARG OSQUERY_BUILD_ENV=remote
12-
13-
#--------------- TEMP CONTAINER FOR LOCAL OSQUERY -------------------------
14-
FROM alpine as osquery_local
15-
ONBUILD COPY osquery /osquery
16-
ONBUILD RUN echo "Copying osquery from local folder"
17-
18-
19-
20-
#--------------- TEMP CONTAINER FOR GIT OSQUERY ----------------------------
21-
FROM alpine/git as osquery_remote
22-
#to pin osquery to a different version change the following envirnment variable
23-
ENV OSQUERY_SRC_VERSION=3.3.2
24-
ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
25-
ONBUILD RUN cd / \
26-
&& git clone "$OSQUERY_GIT_URL" \
27-
&& cd osquery/ \
28-
&& git checkout "$OSQUERY_SRC_VERSION" \
29-
&& echo "Fetching osquery from git"
30-
31-
32-
#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
33-
FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
34-
35-
36-
#--------------- ACTUAL DOCKERFILE FOR BUILD CREATION --------------------------
3712
FROM centos:6
3813

3914
RUN yum makecache fast && yum -y update
4015

4116
#paths that hubble or hubble parts need in the package
4217
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
43-
#osquery build start
44-
#osquery should be built first since requirements for other packages can interfere with osquery dependencies
45-
#to build, osquery scripts want sudo and a user to sudo with.
46-
ENV OSQUERY_BUILD_USER=osquerybuilder
47-
RUN yum -y install xz git make ruby sudo which python-argparse
48-
RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \
49-
&& sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers
50-
COPY --from=osquery_image /osquery /home/"$OSQUERY_BUILD_USER"/osquery
51-
RUN mkdir -p /usr/local/osquery/ \
52-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \
53-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /home/"$OSQUERY_BUILD_USER"/osquery
54-
USER $OSQUERY_BUILD_USER
55-
ENV SKIP_TESTS=1
56-
RUN cd /home/"$OSQUERY_BUILD_USER"/osquery \
57-
&& make sysprep \
58-
#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
59-
&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
60-
&& make deps \
61-
&& make \
62-
&& make strip
63-
USER root
64-
RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /opt/osquery \
65-
&& cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery/hubble_osqueryd \
66-
&& chown -R root. /opt/osquery \
67-
&& chmod -R 500 /opt/osquery/* \
68-
#put augeas lenses into the default directory that we changed earlier
69-
&& mkdir -p /opt/osquery/lenses \
70-
&& cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
71-
&& chmod -R 400 /opt/osquery/lenses/*
72-
RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
18+
19+
#copy in and process osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg
20+
ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar
21+
COPY ${OSQUERY_TAR_FILENAME} /opt/osquery/${OSQUERY_TAR_FILENAME}
22+
RUN cd /opt/osquery \
23+
&& tar xf "$OSQUERY_TAR_FILENAME" \
24+
&& rm -f "$OSQUERY_TAR_FILENAME" \
25+
&& cd / \
26+
&& /opt/osquery/osqueryi --version
7327

7428
#install packages that should be needed for ligbit2 compilation and successful pyinstaller run
75-
RUN yum -y install \
29+
RUN yum -y install git \
7630
libffi-devel openssl-devel libxml2-devel libxslt-devel libffi \
7731
libssh2-devel autoconf automake libtool libjpeg-devel zlib-devel \
7832
make cmake gcc wget openssl
79-
33+
8034
#libcurl install start
8135
#install libcurl to avoid depending on host version
8236
#requires autoconf libtool libssh2-devel zlib-devel autoconf

pkg/centos7/Dockerfile

+22-68
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,35 @@
1-
# This Dockerfile aims to make building Hubble v2 packages easier.
2-
# To build an image: 1. copy pkg/scripts/pyinstaller-requirements.txt to directory with this Dockerfile
3-
# 2. docker build -t <image_name> .
4-
# The resulting image is ready to run the pyinstaller on container start and drop hubble<version>-coreos.tar.gz
5-
# in the /data directory. Mount /data volume into a directory on the host to access the package.
1+
# This Dockerfile aims to make building Hubble v4 packages easier.
2+
# Starting with version 4 building osquery is removed from individual Dockerfiles to its own.
3+
# osquery needs to be built once. Resulting tar file can be used in hubblev4 Dockerfiles.
4+
# Before building hubble, build osquery using a Dockerfile in pkg/osquery/ directory.
5+
# To build this image: 1. copy previously built osquery_4hubble.tar to directory with this Dockerfile
6+
# 2. docker build -t <image_name> --build-arg=HUBBLE_CHECKOUT=<tag or commit> .
7+
# The resulting image is ready to build and run pyinstaller on container start that should
8+
# create hubble<version>-centos7.tar.gz in the /data directory inside the container.
9+
# Mount /data volume into a directory on the host to access the package.
610
# To run the container: docker run -it --rm -v `pwd`:/data <image_name>
7-
# Requires docker 17.05 or higher
811

9-
# Set this argument to "local" if you want to build osquery for local code.
10-
# In that case, osquery folder must exist besides Dockerfile
11-
ARG OSQUERY_BUILD_ENV=remote
12-
13-
#--------------- TEMP CONTAINER FOR LOCAL OSQUERY -------------------------
14-
FROM alpine as osquery_local
15-
ONBUILD COPY osquery /osquery
16-
ONBUILD RUN echo "Copying osquery from local folder"
17-
18-
19-
20-
#--------------- TEMP CONTAINER FOR GIT OSQUERY ----------------------------
21-
FROM alpine/git as osquery_remote
22-
#to pin osquery to a different version change the following envirnment variable
23-
ENV OSQUERY_SRC_VERSION=3.3.2
24-
ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
25-
ONBUILD RUN cd / \
26-
&& git clone "$OSQUERY_GIT_URL" \
27-
&& cd osquery/ \
28-
&& git checkout "$OSQUERY_SRC_VERSION" \
29-
&& echo "Fetching osquery from git"
30-
31-
32-
#--------------- TEMP CONTAINER FOR OSQUERY ( BASED ON ARGUMENT ) ---------------
33-
FROM osquery_"$OSQUERY_BUILD_ENV" as osquery_image
34-
35-
36-
#--------------- ACTUAL DOCKERFILE FOR BUILD CREATION --------------------------
3712
FROM centos:7
3813

3914
RUN yum makecache fast && yum -y update
4015

4116
#paths that hubble or hubble parts need in the package
4217
RUN mkdir -p /etc/hubble/hubble.d /opt/hubble /opt/osquery /var/log/hubble_osquery/backuplogs
43-
#osquery build start
44-
#osquery should be built first since requirements for other packages can interfere with osquery dependencies
45-
#to build, osquery scripts want sudo and a user to sudo with.
46-
ENV OSQUERY_BUILD_USER=osquerybuilder
47-
RUN yum -y install git make python ruby sudo which
48-
RUN useradd --shell /bin/bash --create-home --user-group --groups wheel "$OSQUERY_BUILD_USER" \
49-
&& sed -i '0,/^#\ %wheel/s/^#\ %wheel.*/%wheel\ ALL=\(ALL\)\ NOPASSWD:\ ALL/' /etc/sudoers
50-
COPY --from=osquery_image /osquery /home/"$OSQUERY_BUILD_USER"/osquery
51-
RUN mkdir -p /usr/local/osquery/ \
52-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /usr/local/osquery/ \
53-
&& chown "$OSQUERY_BUILD_USER":"$OSQUERY_BUILD_USER" -R /home/"$OSQUERY_BUILD_USER"/osquery
54-
USER $OSQUERY_BUILD_USER
55-
ENV SKIP_TESTS=1
56-
RUN cd /home/"$OSQUERY_BUILD_USER"/osquery \
57-
&& make sysprep \
58-
#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
59-
&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
60-
&& make deps \
61-
&& make \
62-
&& make strip
63-
USER root
64-
RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /opt/osquery \
65-
&& cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery/hubble_osqueryd \
66-
&& chown -R root. /opt/osquery \
67-
&& chmod -R 500 /opt/osquery/* \
68-
#put augeas lenses into the default directory that we changed earlier
69-
&& mkdir -p /opt/osquery/lenses \
70-
&& cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
71-
&& chmod -R 400 /opt/osquery/lenses/*
72-
RUN ls -lahR /opt/osquery/ && /opt/osquery/osqueryi --version
18+
19+
#copy in and process osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg
20+
ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar
21+
COPY ${OSQUERY_TAR_FILENAME} /opt/osquery/${OSQUERY_TAR_FILENAME}
22+
RUN cd /opt/osquery \
23+
&& tar xf "$OSQUERY_TAR_FILENAME" \
24+
&& rm -f "$OSQUERY_TAR_FILENAME" \
25+
&& cd / \
26+
&& /opt/osquery/osqueryi --version
7327

7428
#install packages that should be needed for ligbit2 compilation and successful pyinstaller run
75-
RUN yum -y install \
76-
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
77-
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
78-
make cmake gcc python-devel python-setuptools wget openssl
29+
RUN yum -y install git \
30+
libffi-devel openssl-devel libffi libssh2-devel autoconf automake libtool \
31+
libxml2-devel libxslt-devel libjpeg-devel zlib-devel \
32+
make cmake gcc python-devel python-setuptools wget openssl
7933

8034
#libcurl install start
8135
#install libcurl to avoid depending on host version

0 commit comments

Comments
 (0)