Skip to content

Commit 7383d56

Browse files
cmeister2cmeister2
authored
Test the gssapi / libssh2 build using a Docker container (#6)
* Initial commits for stage 3 - add a gssapi / libssh2 image * Add travis build parallelization and speed up installs * Clean up images at the end of the build * Test compiling the gssapi / libssh2 build using an image. * Ensure pull requests can work
1 parent 2d80501 commit 7383d56

20 files changed

+479
-60
lines changed

‎.travis.yml

+49-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,51 @@
11
language: minimal
2+
services:
3+
- docker
4+
jobs:
5+
include:
6+
# Pull requests don't have access to encrypted variables, so they cannot
7+
# benefit from the multistage support.
28

3-
script:
4-
# Run all stages in one stage at the moment. Docker images are not shared
5-
# between stages.
6-
- ./stage1.sh
7-
- ./stage2.sh
9+
# Stage 1: Build the base image
10+
- stage: stage1_base_image
11+
script: ./build_scripts/stage1.sh
12+
if: type != pull_request
13+
14+
# Stage 2: GCC and Clang
15+
- stage: stage2_gcc_clang
16+
script: ./build_scripts/stage2.sh gcc8
17+
if: type != pull_request
18+
- script: ./build_scripts/stage2.sh clang7
19+
if: type != pull_request
20+
21+
# Stage 3: Build remaining images
22+
- stage: stage3_build_images
23+
script: ./build_scripts/stage3.sh gssapi_libssh2
24+
if: type != pull_request
25+
26+
# Stage 4: Test building curl
27+
- stage: stage4_test_images
28+
script: ./build_scripts/stage4.sh gssapi_libssh2
29+
if: type != pull_request
30+
31+
# Cleanup: remove any temporary images
32+
- stage: remove_images
33+
script: ./build_scripts/cleanup.sh
34+
if: tag IS blank AND type != pull_request
35+
36+
# -----
37+
38+
# For pull requests we still want to do testing, but we have to do it in
39+
# serial mode.
40+
- stage: all_stages_pull_request
41+
script:
42+
- ./build_scripts/stage1.sh
43+
- ./build_scripts/stage2.sh
44+
- ./build_scripts/stage3.sh
45+
- ./build_scripts/stage4.sh
46+
if: type = pull_request
47+
48+
env:
49+
global:
50+
- secure: hZHhrJmUroDWlcXMs6oH9ckWpbKOd3/9z6EEceBmRy/X1J21N5vMCnnjeQT1uDRbN3IO/Ir56eCNQ8tGXYptH50/yxywQ4QtUk+vR/VrqUuI+m69lKG0WjDP+SzOCHaf17lbrXtyBPRsVZe+W34uNt0bMMcRpF3IG0WiNQFnrugwXtaYDyZPEJb8sb4s95lhHzsjfAaJQpUNncL7Tk4neg3ItZ0oLvjdBptMJqDOPrXtEFwBr8OMQ0WxkP7hdeDdj1xvTvvb1xmlI69tRO9w5xkAhLPD9YA7mcizCjnEPuLGFowiyafLF3MfQapII2opYaFyhoj1Z6gCUNNhi9ueDMOkSjq4TDgJ0SIGIHFJLf+9VMhubPFCMySGFjtI/unziSQjAnYOczM+crFPU3a4VxHti9289jTmiu9YmFxivb09PGo3OrWemHERySy3wtcKxbpruljk1F0gsFKrRFH4ovLLq4Oz0zfNyWnSZIdVVUc5TQKtJNlBQkLA1IAaSC6okIV+gTrdWHhPb9Fuxj4gb+qX0hNdLTwc7RDhPUF2ua/CnOi3JXOrE7Y9AZvN/9d6zlq+cfh4yYbOEEcpJELHSGhulaGgz1PQ3iSMbBh5L1c0CgNydbcoHRzVzdhZYNbm3C9t46pcb/EFjKH0eEDxjD/3CB9egLdXjnRoiMmr1/o=
51+
- secure: UVw8wRAzc1YcbGnFeAVNohOKZ2vMtuaKJC53MobN/4HXCQ4xRST2XaUl6CkdS2MyXXrrvcMC9jER5lIJntwKYc8aCtEeSoqQP6EjWfODBPjg0IJll63uZaYSm/iLdavhjA/ZQN5NN5tIlOx5T9C47rBDmRJdB+L+pobDrMaGt35qZNfI7ibugN/ecfUfJkzldVEWr5wh732zuBjLdpXTzx/mbNZyolfOPXubysGs8LJI2aqU9VB68HlbbvZXq7oUzvYz6t0L2DornD3DazsQXFemuMqAqB/17eEpO4HqtzDO+6o0WRtjctLXFhvGX8sXYlMH24GuZvEemQeUlWpDPAKS9BiUNpG/21ShA0wCK+2BuiZ8rNBngZjRK2YLVgIi+2582knf3HxXsDlEKuuVdJRY6dVYekTXSNIR95I/WtypoXWHM1L5HhHFfecVgGu5yIPMgbLK3h0uV8G4E/Og6IY/OWBi0ZWKgmK8xLbKVf/MsK6Xa1uBxJvf3709WzyzxY0mVJTTIEgHCcCWsF9ixJDwJ6C7yPibJfwE/66q7XbFLFuahUMYGtcpreI8OIx7dNzdGRnAVzafygFGdzfPrgzN9zPmV72T1y0uqAJpWznRShWco4yneLxddvymrQ/Qrn77QFKbLbzM+8VBKnKnBw3wm88WbYdg1a7y8Jm8kns=

‎Dockerfile

+3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ FROM $BASE_IMAGE
55
# The docker build process is noninteractive
66
ARG DEBIAN_FRONTEND=noninteractive
77

8+
# The container user is root.
9+
ENV USER=root
10+
811
# Run the install process using the provided script (chosen using --build-arg)
912
ARG DOCKER_SCRIPT
1013
COPY docker_scripts/ /scripts/

‎build_scripts/build_and_push.sh

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
# Import the common functionality.
6+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
7+
8+
# The first argument is the tag. This should match a repository in Docker and
9+
# is the "name" of the image.
10+
TAG=${1}
11+
12+
# The second argument is the base image. This is what the image is built on top
13+
# of.
14+
BASE_IMAGE=${2}
15+
16+
# The third argument is the build script. This is run inside Docker to install
17+
# the new files.
18+
DOCKER_SCRIPT=${3}
19+
20+
# Use the common script functionality to build the image.
21+
build_image_versioned ${TAG} ${BASE_IMAGE} ${DOCKER_SCRIPT}
22+
23+
# On Travis, push the newly generated image if possible.
24+
if [[ -n "${TRAVIS:-}" ]]
25+
then
26+
push_image_versioned ${TAG}
27+
fi

‎build_scripts/cleanup.sh

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
# Import the common functionality.
6+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
7+
8+
if [[ -n "${TRAVIS:-}" ]]
9+
then
10+
if [[ -z "${TRAVIS_TAG:-}" ]]
11+
then
12+
# Install requests.
13+
sudo apt-get update
14+
sudo apt-get -y install python3 python3-pip
15+
sudo pip3 install requests
16+
17+
# Remove all the images.
18+
remove_image_versioned base_image
19+
remove_image_versioned base_gcc8
20+
remove_image_versioned base_clang7
21+
remove_image_versioned gssapi_libssh2
22+
fi
23+
fi

‎build_scripts/common.sh

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/usr/bin/env bash
2+
3+
# Calculate the current build locations
4+
export SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
5+
export ROOTDIR=${SCRIPTDIR}/..
6+
7+
# If running on travis and processing a tag, work with curlbuildimages.
8+
# If running on travis and processing anything else, work with
9+
# curlbuildimagestemp.
10+
# If running locally, work with the curlbuildimagestemp namespace.
11+
if [[ -n "${TRAVIS:-}" ]]
12+
then
13+
14+
# Check to see if we can log into Docker. Pull requests cannot log into Docker
15+
if [[ -n "${DOCKER_USER:-}" ]]
16+
then
17+
# Log into Docker.
18+
echo "${DOCKER_PASS}" | docker login -u "${DOCKER_USER}" --password-stdin
19+
fi
20+
21+
if [[ -n "${TRAVIS_TAG:-}" ]]
22+
then
23+
# Processing a tagged build. Get the version from the version file.
24+
export VERSION=$(cat VERSION)
25+
26+
# Use the main repository.
27+
export DOCKER_REPO=curlbuildimages
28+
else
29+
# Processing an untagged build. The version is the build ID, which is
30+
# shared between stages.
31+
export VERSION=${TRAVIS_BUILD_ID}
32+
33+
# Use the temporary repository.
34+
export DOCKER_REPO=curlbuildimagestemp
35+
fi
36+
else
37+
# Get the version from the version file.
38+
export VERSION=$(cat VERSION)
39+
40+
# Use the temporary repository.
41+
export DOCKER_REPO=curlbuildimagestemp
42+
fi
43+
44+
45+
# Simple function which uses docker to build images.
46+
build_image()
47+
{
48+
TAG=$1
49+
BASE_IMAGE=$2
50+
DOCKER_SCRIPT=$3
51+
52+
echo "Building ${TAG}:${VERSION} from ${BASE_IMAGE}"
53+
54+
docker build --build-arg BASE_IMAGE=${BASE_IMAGE} \
55+
--build-arg DOCKER_SCRIPT=${DOCKER_SCRIPT} \
56+
-t ${DOCKER_REPO}/${TAG}:${VERSION} \
57+
.
58+
}
59+
60+
# Simple wrapper function which adds the REPOSITORY and VERSION to the given
61+
# BASE_IMAGE, then builds.
62+
build_image_versioned()
63+
{
64+
TAG=$1
65+
BASE_IMAGE=$2
66+
DOCKER_SCRIPT=$3
67+
68+
build_image ${TAG} ${DOCKER_REPO}/${BASE_IMAGE}:${VERSION} ${DOCKER_SCRIPT}
69+
}
70+
71+
72+
# Simple function which adds the REPOSITORY and VERSION to the given TAG,
73+
# then pushes to Docker.
74+
push_image_versioned()
75+
{
76+
if [[ -n "${DOCKER_USER:-}" ]]
77+
then
78+
# The Docker user environment variable exists, so pushing should work.
79+
TAG=$1
80+
docker push ${DOCKER_REPO}/${TAG}:${VERSION}
81+
else
82+
echo "Would have pushed to ${DOCKER_REPO}/${TAG}:${VERSION} but "
83+
echo "DOCKER_USER is not set."
84+
fi
85+
}
86+
87+
88+
# Function to remove an image tag from the Docker hub
89+
remove_image_versioned()
90+
{
91+
python3 ${ROOTDIR}/docker_hub_scripts/remove_tag.py \
92+
--username ${DOCKER_USER} \
93+
--password ${DOCKER_PASS} \
94+
--repository ${1} \
95+
--tag ${VERSION}
96+
}

‎build_scripts/download_stable_curl.sh

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
pushd /tmp
4+
5+
# Download an archive of a stable version of curl.
6+
wget https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz
7+
8+
# Untar curl
9+
tar xvf curl-7.65.1.tar.gz
10+
11+
# Rename the folder to curl
12+
mv curl-7.65.1 curl
13+
14+
popd

‎build_scripts/run_normal.sh

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
# Import the common functionality.
6+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
7+
8+
# Get the inputs to this script.
9+
IMAGE_NAME=${1}
10+
CURL_DIR=${2}
11+
12+
# Shift the arguments along so we can pass $@ to the docker container.
13+
shift 2
14+
15+
# Derive the full name of the image.
16+
FULL_IMAGE_NAME=${DOCKER_REPO}/${IMAGE_NAME}:${VERSION}
17+
18+
# If we need to run with special environment variables, check now.
19+
ENV_CHECKSRC=
20+
21+
if [[ -n ${CHECKSRC:-} ]]
22+
then
23+
ENV_CHECKSRC="-e CHECKSRC=1"
24+
fi
25+
26+
# Finally, just run the docker test.
27+
docker run --rm -it $ENV_CHECKSRC -v ${CURL_DIR}:/opt/curl ${FULL_IMAGE_NAME} /scripts/test_normal.sh $@

‎stage1.sh ‎build_scripts/stage1.sh

+9-1
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,18 @@
22

33
set -euxo pipefail
44

5-
. common.sh
5+
# Import the common functionality.
6+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
67

78
# Build stage 1. Stage 1 consists of the root base image with some common tools installed.
89
echo "Building stage 1 (version ${VERSION})"
910

1011
# Build the root base image from Xenial
1112
build_image base_image ubuntu:xenial-20190515 install_base_tools.sh
13+
14+
15+
# On Travis, push the images.
16+
if [[ -n "${TRAVIS:-}" ]]
17+
then
18+
push_image_versioned base_image
19+
fi

‎build_scripts/stage2.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
# This script encompasses all the builds that should happen in stage 2 of
4+
# the Travis build.
5+
6+
set -euxo pipefail
7+
8+
# Import the common functionality.
9+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
10+
11+
# If a build name is passed then just build that build.
12+
REQUESTED=${1:-}
13+
14+
# Build stage 2. Stage 2 produces images with gcc-8 and clang-7 installed.
15+
16+
if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "gcc8" ]]
17+
then
18+
# Build gcc-8. Depends on `base_image` from stage1
19+
${SCRIPTDIR}/build_and_push.sh base_gcc8 base_image install_gcc8.sh
20+
fi
21+
22+
if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "clang7" ]]
23+
then
24+
# Build clang-7. Depends on `base_image` from stage1
25+
${SCRIPTDIR}/build_and_push.sh base_clang7 base_image install_clang7.sh
26+
fi

‎build_scripts/stage3.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
3+
# This script encompasses all the builds that should happen in stage 3 of
4+
# the Travis build.
5+
6+
set -euxo pipefail
7+
8+
# Import the common functionality.
9+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
10+
11+
# If a build name is passed then just build that build.
12+
REQUESTED=${1:-}
13+
14+
# Build stage 3. Stage 3 produces all the relevant test images.
15+
if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "gssapi_libssh2" ]]
16+
then
17+
# Build for test: gcc-8 (with-gssapi, with-libssh2) CHECKSRC=1
18+
${SCRIPTDIR}/build_and_push.sh gssapi_libssh2 base_gcc8 create_gssapi_libssh2.sh
19+
fi

‎build_scripts/stage4.sh

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
3+
# This script encompasses all the tests that should happen in stage 4 of
4+
# the Travis build.
5+
6+
set -euxo pipefail
7+
8+
# Import the common functionality.
9+
. $(dirname "${BASH_SOURCE[0]}")/common.sh
10+
11+
# If a test name is passed then just run that test.
12+
REQUESTED=${1:-}
13+
14+
# Download a temporary version of curl if required.
15+
if [[ ! -d /tmp/curl ]]
16+
then
17+
${SCRIPTDIR}/download_stable_curl.sh
18+
fi
19+
20+
if [[ -z ${REQUESTED} ]] || [[ ${REQUESTED} == "gssapi_libssh2" ]]
21+
then
22+
# Run test: gcc-8 (with-gssapi, with-libssh2) CHECKSRC=1
23+
CHECKSRC=1 ${SCRIPTDIR}/run_normal.sh gssapi_libssh2 /tmp/curl --with-gssapi --with-libssh2
24+
fi

‎common.sh

-34
This file was deleted.

0 commit comments

Comments
 (0)