tooling: Add authentication and extra parameters to Dockerfiles

Author: cassava

dist/docker/Dockerfile.archlinux

@@ -1,6 +1,6 @@
 # Dockerfile.archlinux
 #
-# See Dockerfile.ubuntu for documentation of each of the lines.
+# See Dockerfile.focal for documentation of each of the lines.
 ARG BUILD_FROM=archlinux:latest
 ARG DEPLOY_FROM=${BUILD_FROM}
 
@@ -25,6 +25,8 @@ RUN pip3 install --upgrade pip && \
 # Install and Setup Conan
 ARG CONAN_REMOTE=https://conan.bintray.com
 ARG CONAN_REMOTE_VERIFY_SSL=true
+ARG CONAN_LOGIN_USERNAME
+ARG CONAN_PASSWORD
 RUN conan profile new --detect default && \
     conan profile update settings.compiler.libcxx=libstdc++11 default && \
     conan config set general.request_timeout=360 && \
@@ -36,12 +38,19 @@ WORKDIR /cloe
 COPY . /cloe
 ARG WITH_VTD=0
 ARG PACKAGE_TARGET=package-select
-RUN make export-vendor export && \
+RUN export CONAN_NON_INTERACTIVE=yes && \
+    if [ ${CONAN_LOGIN_USERNAME} != "" ]; then \
+        CONAN_LOGIN_USERNAME="${CONAN_LOGIN_USERNAME}" CONAN_PASSWORD="${CONAN_PASSWORD}" conan user --remote=default -p || exit 1; \
+    fi && \
+    make export-vendor export && \
     make WITH_VTD=${WITH_VTD} ${PACKAGE_TARGET} && \
-    export LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 && \
-    conan remove \* -b -f
+    conan remove \* -b -f && \
+    conan user --clean
 
-RUN make WITH_VTD=${WITH_VTD} smoketest && \
+ARG VI_LIC_SERVER
+RUN export LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 && \
+    export VI_LIC_SERVER="${VI_LIC_SERVER}" && \
+    make WITH_VTD=${WITH_VTD} smoketest && \
     make WITH_VTD=${WITH_VTD} INSTALL_DIR="/deploy" deploy
 
 # Create Deploy Image

dist/docker/Dockerfile.bionic

@@ -1,18 +1,12 @@
-# Dockerfile.ubuntu
+# Dockerfile.bionic
 #
-# This file acts partly as a Docker recipe for building Cloe on Ubuntu.
-# It is currently only tested with Ubuntu 18.04 and 20.04.
-#
-# If you are behind a proxy, make sure to pass in the respective HTTP_PROXY,
-# HTTPS_PROXY, and NO_PROXY variables.
+# See Dockerfile.focal for documentation of each of the lines.
 ARG BUILD_FROM=ubuntu:18.04
 ARG DEPLOY_FROM=${BUILD_FROM}
 
 FROM ${BUILD_FROM} AS build
 
 # Install System Packages
-#
-# These packages are required for building and testing Cloe.
 COPY Makefile.setup /cloe/Makefile.setup
 RUN apt-get update && \
     apt-get install -y make locales && \
@@ -31,58 +25,37 @@ RUN pip3 install --upgrade pip && \
         install-python-deps
 
 # Install and Setup Conan
-#
-# You may not want to use the default Conan remote (conan-center), so we use
-# whatever is stored in the build arguments CONAN_REMOTE. Currently, only
-# anonymous access is possible in this Dockerfile.
 ARG CONAN_REMOTE=https://conan.bintray.com
 ARG CONAN_REMOTE_VERIFY_SSL=true
+ARG CONAN_LOGIN_USERNAME
+ARG CONAN_PASSWORD
 RUN conan profile new --detect default && \
     conan profile update settings.compiler.libcxx=libstdc++11 default && \
     conan config set general.request_timeout=360 && \
     conan remote clean && \
     conan remote add default ${CONAN_REMOTE} ${CONAN_REMOTE_VERIFY_SSL}
 
 # Build and Install Cloe
-#
-# All common processes are made easy to apply by writing target recipes in the
-# Makefile at the root of the repository. This also acts as a form of
-# documentation.
 WORKDIR /cloe
 COPY . /cloe
 ARG WITH_VTD=0
 ARG PACKAGE_TARGET=package-select
-RUN \
-    # Export our own Conan recipes, in case they are not available in the
-    # CONAN_REMOTE specified above.
+RUN export CONAN_NON_INTERACTIVE=yes && \
+    if [ ${CONAN_LOGIN_USERNAME} != "" ]; then \
+        CONAN_LOGIN_USERNAME="${CONAN_LOGIN_USERNAME}" CONAN_PASSWORD="${CONAN_PASSWORD}" conan user --remote=default -p || exit 1; \
+    fi && \
     make export-vendor export && \
-    # Build all the packages, except for vtd, because that currently requires
-    # dependencies we don't have in the Docker container.
-    # You can specify more than one package here, see the Makefile for more
-    # information on the WITH_*, PACKAGES, NOBUILD_PKGS, and BUILD_PKGS variables.
     make WITH_VTD=${WITH_VTD} ${PACKAGE_TARGET} && \
-    # In the above commands, Conan downloads and creates packages into its
-    # local cache. We don't need the build directories, since we have deployed
-    # Cloe, so we should clean up to keep the Docker image down.
-    conan remove \* -b -f
+    conan remove \* -b -f && \
+    conan user --clean
 
-RUN \
-    # Run smoketests.
-    export LC_ALL=C.UTF-8 LANG=C.UTF-8 && \
+ARG VI_LIC_SERVER
+RUN export LC_ALL=C.UTF-8 LANG=C.UTF-8 && \
+    export VI_LIC_SERVER="${VI_LIC_SERVER}" && \
     make WITH_VTD=${WITH_VTD} smoketest && \
-    # Deploy all the Cloe packages to INSTALL_DIR, which is /usr/local by
-    # default. This will also populate BUILD_DIR, so that should be removed
-    # afterwards to prevent this image from getting too big.
     make WITH_VTD=${WITH_VTD} INSTALL_DIR="/deploy" deploy
 
 # Create Deploy Image
-#
-# Unfortunately, there are still some issues with cloe-engine not finding
-# libcloe-runtime.so, so we need to explicitly set LD_LIBRARY_PATH.
-#
-# This image can now be used to work with stack files. You should mount in
-# whatever directory contains your stack files for best results, otherwise
-# the engine might try to read files that it doesn't have in its own filesystem.
 FROM ${DEPLOY_FROM}
 COPY --from=build /deploy /usr/local/
 ENV LD_LIBRARY_PATH=/usr/local/lib

dist/docker/Dockerfile.focal

@@ -1,7 +1,6 @@
-# Dockerfile.ubuntu
+# Dockerfile.focal
 #
 # This file acts partly as a Docker recipe for building Cloe on Ubuntu.
-# It is currently only tested with Ubuntu 18.04 and 20.04.
 #
 # If you are behind a proxy, make sure to pass in the respective HTTP_PROXY,
 # HTTPS_PROXY, and NO_PROXY variables.
@@ -37,6 +36,8 @@ RUN pip3 install --upgrade pip && \
 # anonymous access is possible in this Dockerfile.
 ARG CONAN_REMOTE=https://conan.bintray.com
 ARG CONAN_REMOTE_VERIFY_SSL=true
+ARG CONAN_LOGIN_USERNAME
+ARG CONAN_PASSWORD
 RUN conan profile new --detect default && \
     conan profile update settings.compiler.libcxx=libstdc++11 default && \
     conan config set general.request_timeout=360 && \
@@ -53,6 +54,10 @@ COPY . /cloe
 ARG WITH_VTD=0
 ARG PACKAGE_TARGET=package-select
 RUN \
+    export CONAN_NON_INTERACTIVE=yes && \
+    if [ ${CONAN_LOGIN_USERNAME} != "" ]; then \
+        CONAN_LOGIN_USERNAME="${CONAN_LOGIN_USERNAME}" CONAN_PASSWORD="${CONAN_PASSWORD}" conan user --remote=default -p || exit 1; \
+    fi && \
     # Export our own Conan recipes, in case they are not available in the
     # CONAN_REMOTE specified above.
     make export-vendor export && \
@@ -64,11 +69,14 @@ RUN \
     # In the above commands, Conan downloads and creates packages into its
     # local cache. We don't need the build directories, since we have deployed
     # Cloe, so we should clean up to keep the Docker image down.
-    conan remove \* -b -f
+    conan remove \* -b -f && \
+    conan user --clean
 
+ARG VI_LIC_SERVER
 RUN \
     # Run smoketests.
     export LC_ALL=C.UTF-8 LANG=C.UTF-8 && \
+    export VI_LIC_SERVER="${VI_LIC_SERVER}" && \
     make WITH_VTD=${WITH_VTD} smoketest && \
     # Deploy all the Cloe packages to INSTALL_DIR, which is /usr/local by
     # default. This will also populate BUILD_DIR, so that should be removed

dist/docker/Dockerfile.xenial

@@ -1,10 +1,6 @@
-# Dockerfile.ubuntu
+# Dockerfile.xenial
 #
-# This file acts partly as a Docker recipe for building Cloe on Ubuntu.
-# It is currently only tested with Ubuntu 16.04.
-#
-# If you are behind a proxy, make sure to pass in the respective HTTP_PROXY,
-# HTTPS_PROXY, and NO_PROXY variables.
+# See Dockerfile.focal for documentation of each of the lines.
 ARG BUILD_FROM=ubuntu:16.04
 ARG DEPLOY_FROM=${BUILD_FROM}
 
@@ -13,7 +9,6 @@ FROM ${BUILD_FROM} AS build
 # Install Newer CMake and GCC Packages
 RUN apt-get update && \
     apt-get install --no-install-recommends -y apt-transport-https ca-certificates gnupg software-properties-common wget && \
-    # Add cmake repository:
     wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg && \
     apt-add-repository "deb https://apt.kitware.com/ubuntu/ xenial main" && \
     add-apt-repository "ppa:ubuntu-toolchain-r/test" && \
@@ -58,8 +53,6 @@ RUN . /etc/profile.d/pyenv.sh && \
     pyenv global ${PYTHON_VERSION}
 
 # Install System Packages
-#
-# These packages are required for building and testing Cloe.
 COPY Makefile.setup /cloe/Makefile.setup
 RUN apt-get update && \
     apt-get install -y make locales && \
@@ -79,12 +72,10 @@ RUN . /etc/profile.d/pyenv.sh && \
         install-python-deps
 
 # Install and Setup Conan
-#
-# You may not want to use the default Conan remote (conan-center), so we use
-# whatever is stored in the build arguments CONAN_REMOTE. Currently, only
-# anonymous access is possible in this Dockerfile.
 ARG CONAN_REMOTE=https://conan.bintray.com
 ARG CONAN_REMOTE_VERIFY_SSL=true
+ARG CONAN_LOGIN_USERNAME
+ARG CONAN_PASSWORD
 RUN . /etc/profile.d/pyenv.sh && \
     conan profile new --detect default && \
     conan profile update settings.compiler.libcxx=libstdc++11 default && \
@@ -93,45 +84,28 @@ RUN . /etc/profile.d/pyenv.sh && \
     conan remote add default ${CONAN_REMOTE} ${CONAN_REMOTE_VERIFY_SSL}
 
 # Build and Install Cloe
-#
-# All common processes are made easy to apply by writing target recipes in the
-# Makefile at the root of the repository. This also acts as a form of
-# documentation.
 WORKDIR /cloe
 COPY . /cloe
 ARG WITH_VTD=0
 ARG PACKAGE_TARGET=package-select
 RUN . /etc/profile.d/pyenv.sh && \
-    # Export our own Conan recipes, in case they are not available in the
-    # CONAN_REMOTE specified above.
+    export CONAN_NON_INTERACTIVE=yes && \
+    if [ ${CONAN_LOGIN_USERNAME} != "" ]; then \
+        CONAN_LOGIN_USERNAME="${CONAN_LOGIN_USERNAME}" CONAN_PASSWORD="${CONAN_PASSWORD}" conan user --remote=default -p || exit 1; \
+    fi && \
     make export-vendor export && \
-    # Build all the packages, except for vtd, because that currently requires
-    # dependencies we don't have in the Docker container.
-    # You can specify more than one package here, see the Makefile for more
-    # information on the WITH_*, PACKAGES, NOBUILD_PKGS, and BUILD_PKGS variables.
     make WITH_VTD=${WITH_VTD} ${PACKAGE_TARGET} && \
-    # In the above commands, Conan downloads and creates packages into its
-    # local cache. We don't need the build directories, since we have deployed
-    # Cloe, so we should clean up to keep the Docker image down.
-    conan remove \* -b -f
+    conan remove \* -b -f && \
+    conan user --clean
 
+ARG VI_LIC_SERVER
 RUN . /etc/profile.d/pyenv.sh && \
-    # Run smoketests.
     export LC_ALL=C.UTF-8 LANG=C.UTF-8 && \
+    export VI_LIC_SERVER="${VI_LIC_SERVER}" && \
     make WITH_VTD=${WITH_VTD} smoketest && \
-    # Deploy all the Cloe packages to INSTALL_DIR, which is /usr/local by
-    # default. This will also populate BUILD_DIR, so that should be removed
-    # afterwards to prevent this image from getting too big.
     make WITH_VTD=${WITH_VTD} INSTALL_DIR="/deploy" deploy
 
 # Create Deploy Image
-#
-# Unfortunately, there are still some issues with cloe-engine not finding
-# libcloe-runtime.so, so we need to explicitly set LD_LIBRARY_PATH.
-#
-# This image can now be used to work with stack files. You should mount in
-# whatever directory contains your stack files for best results, otherwise
-# the engine might try to read files that it doesn't have in its own filesystem.
 FROM ${DEPLOY_FROM}
 COPY --from=build /deploy /usr/local/
 ENV LD_LIBRARY_PATH=/usr/local/lib

dist/docker/Makefile

@@ -7,10 +7,10 @@ CLOE_VERSION := $(shell cat ${CLOE_ROOT}/VERSION)
 DOCKER_NETWORK := \
 	$(shell \
 		if [ -n $$https_proxy ]; then \
+			echo " --network=host"; \
 			echo " --build-arg https_proxy=\"$$https_proxy\""; \
 			echo " --build-arg http_proxy=\"$$http_proxy\""; \
 			echo " --build-arg no_proxy=\"$$no_proxy\""; \
-			echo " --network=host"; \
 		fi \
 	)
 
@@ -19,20 +19,39 @@ IMAGE_VERSION := ${CLOE_VERSION}
 DOCKER_IMAGE := ${IMAGE_BASE}:${IMAGE_VERSION}
 DOCKER_CONTEXT := ${CLOE_ROOT}
 
-DOCKER_BUILD_ARGS += ${DOCKER_NETWORK}
+# Default build arguments
+PACKAGE_TARGET := package-select
+BUILD_TYPE := Release
+WITH_VTD := 0
+VI_LIC_SERVER :=
 
 # Read extra build arguments from the .env file.
 # The .env file is in .gitignore so the user can write their overrides there.
 # This is currently primarily used so that the user can set their preferred
 # Conan remote with the variables CONAN_REMOTE and CONAN_REMOTE_VERIFY_SSL.
 -include .env
 
+DOCKER_BUILD_ARGS += ${DOCKER_NETWORK}
+
 ifneq (${CONAN_REMOTE},)
-DOCKER_BUILD_ARGS += \
-	--build-arg CONAN_REMOTE="${CONAN_REMOTE}" \
-	--build-arg CONAN_REMOTE_VERIFY_SSL="${CONAN_REMOTE_VERIFY_SSL}"
+DOCKER_BUILD_ARGS += --build-arg CONAN_REMOTE="${CONAN_REMOTE}"
+DOCKER_BUILD_ARGS += --build-arg CONAN_REMOTE_VERIFY_SSL="${CONAN_REMOTE_VERIFY_SSL}"
 endif
-DOCKER_ARGS = $(shell echo "${DOCKER_BUILD_ARGS}" | sed -r 's/\s+/ /g')
+
+ifneq (${CONAN_LOGIN_USERNAME},)
+DOCKER_BUILD_ARGS += --build-arg CONAN_LOGIN_USERNAME="${CONAN_LOGIN_USERNAME}"
+DOCKER_BUILD_ARGS += --build-arg CONAN_PASSWORD="${CONAN_PASSWORD}"
+endif
+
+DOCKER_BUILD_ARGS += --build-arg PACKAGE_TARGET=${PACKAGE_TARGET}
+
+ifeq (${WITH_VTD},1)
+DOCKER_IMAGE := ${DOCKER_IMAGE}-vtd
+DOCKER_BUILD_ARGS += --build-arg WITH_VTD=${WITH_VTD}
+DOCKER_BUILD_ARGS += --build-arg VI_LIC_SERVER="${VI_LIC_SERVER}"
+endif
+
+DOCKER_ARGS := ${DOCKER_BUILD_ARGS}
 
 .PHONY: help
 .DEFAULT: help

dist/docker/README.md

@@ -0,0 +1,58 @@
+Cloe Docker Images
+==================
+
+This directory contains Dockerfiles to build Cloe Conan packages in various
+environments. We use a Makefile to automate the build process as much as
+possible. Run `make help` to see a list of the available targets.
+
+```console
+$ make help
+Usage: make <target>
+
+Available targets:
+  all           to build all stable docker images
+  ubuntu-20.04  to build the Ubuntu 20.04 image
+  ubuntu-18.04  to build the Ubuntu 18.04 image
+  ubuntu-16.04  to build the Ubuntu 18.04 image
+  archlinux     to build the Archlinux image
+  alpine        to build the Alpine image
+
+Configuration:
+  IMAGE_BASE:     cloe/cloe-engine
+  IMAGE_VERSION:  0.18.0-nightly
+  CONAN_REMOTE:   https://artifactory.example.com/artifactory/api/conan/cloe
+  DOCKER_CONTEXT: ../..
+  DOCKER_ARGS:     --network=host
+                   --build-arg https_proxy=http://127.0.0.1:3128/
+                   --build-arg http_proxy=http://127.0.0.1:3128/
+                   --build-arg no_proxy=localhost,127.0.0.1,127.*,172.*,192.168.*,10.*
+                   --build-arg CONAN_REMOTE=https://artifactory.example.com/artifactory/api/conan/cloe
+                   --build-arg CONAN_REMOTE_VERIFY_SSL=True
+                   --build-arg CONAN_LOGIN_USERNAME=cloebuilder
+                   --build-arg CONAN_PASSWORD=secret
+                   --build-arg WITH_VTD=0
+                   --build-arg VI_LIC_SERVER=license-server.example.com
+                   --build-arg PACKAGE_TARGET=package-select
+```
+
+The output has been slightly formatted to make the Docker build arguments a
+little more obvious. The following build arguments are available:
+
+- `CONAN_REMOTE`: optional string; defaults to conan-center URL.
+- `CONAN_REMOTE_VERIFY_SSL`: optional, one of `True` and `False`; defaults to `True`.
+- `CONAN_LOGIN_USERNAME`: optional username for authenticating with remote.
+- `CONAN_PASSWORD`: optional password, required if username specified.
+- `WITH_VTD`: optional, one of `0` and `1`; defaults to `0`.
+- `VI_LIC_SERVER`: optional hostname of VTD license server.
+- `PACKAGE_TARGET`: optional, one of `package`, `package-select`, and `package-all`;
+  defaults to `package-select`.
+
+You can specify these arguments on the command line when calling make, or you
+can specify them in a `.env` file, which is exempt from version control.
+It is recommended to specify the Conan build arguments in the `.env` file, and
+the other arguments on the command line. Because of the way the .env file is
+read by Make, it is important to not use any quotation marks around the values.
+
+Because Docker images may be built in environments that have a proxy running,
+the Makefile will automatically add the proxy variables if they are are
+detected in the host environment.