Skip to content

Commit c1def9b

Browse files
author
Nosheen Adil
committed
add jenkins files
1 parent 39a6140 commit c1def9b

File tree

7 files changed

+257
-10
lines changed

7 files changed

+257
-10
lines changed

‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@ clean:
115115
@rm -rf build
116116
@rm -rf docs/build
117117

118+
setup:
119+
./scripts/setup -v --src ${REMOTE_DIR} --dst ${REMOTE_LOCAL_DIR} --remote-copy
120+
118121
ifeq ($(DETECTED_OS), mac)
119122
env:
120123
CONDA_SUBDIR=osx-64 conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \
@@ -150,7 +153,7 @@ endif
150153

151154
rebuild: clean build
152155

153-
test: build
156+
test:
154157
$(VENV_ACTIVATE) $(VENV_NAME) && \
155158
pip install --force-reinstall '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)[test]' && \
156159
cd build/Release && \

‎jenkins/development/linux/Jenkinsfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// The Jenkinsfile for the MacOS Development Builds pipeline.
2+
3+
node("idps-linux-node") {
4+
checkout scm
5+
pipelineModule = load "jenkins/pipeline.groovy"
6+
pipelineModule.run("linux")
7+
}

‎jenkins/development/mac/Jenkinsfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// The Jenkinsfile for the MacOS Development Builds pipeline.
2+
3+
node("idps-mac-node") {
4+
checkout scm
5+
pipelineModule = load "jenkins/pipeline.groovy"
6+
pipelineModule.run("mac")
7+
}

‎jenkins/development/windows/Jenkinsfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// The Jenkinsfile for the MacOS Development Builds pipeline.
2+
3+
node("idps-windows-node") {
4+
checkout scm
5+
pipelineModule = load "jenkins/pipeline.groovy"
6+
pipelineModule.run("windows")
7+
}

‎jenkins/pipeline.groovy

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/// Common functions for build pipelines
2+
3+
def run_command(command, os) {
4+
// Run a command on a particular OS
5+
// On Windows, the command needs to be run as a Bat script
6+
// which calls the git bash shell. This was the best way to
7+
// run Unix commands on Windows using Jenkins
8+
if (os == "linux" || os == "mac") {
9+
sh command
10+
}
11+
else {
12+
bat ("sh -c \"${command}\"")
13+
}
14+
}
15+
16+
def run(os, deploy = false) {
17+
stage("Setup") {
18+
run_command("make setup REMOTE_DIR=${IDPS_REMOTE_EXT_DIR} REMOTE_LOCAL_DIR=${IDPS_REMOTE_EXT_COPY_DIR}", os)
19+
}
20+
21+
stage("Build") {
22+
run_command("make build THIRD_PARTY_DIR=${IDPS_REMOTE_EXT_COPY_DIR}", os)
23+
}
24+
25+
stage("Test") {
26+
run_command("make test TEST_DATA_DIR=${IDPS_REMOTE_EXT_COPY_DIR}/test_data_structured", os)
27+
}
28+
}
29+
30+
return this

‎scripts/check-git-clean.sh

-9
This file was deleted.

‎scripts/setup

+202
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
#!/bin/bash
2+
3+
usage()
4+
{
5+
echo "Usage $0 [-v | --verbose] [-l | --large-data]"
6+
echo " -v | --verbose: print debug output"
7+
echo " --src: specify the location of the directory to copy from"
8+
echo " --dst: specify the location of the directory to copy to, otherwise use default value (third_party)"
9+
echo " --remote-copy: copy data from src to dst as if it's a local copy of remote data (i.e., keep dir structure of src in dst)"
10+
echo " this option is useful for improving download speeds on build servers by keep a local copy of remote data"
11+
echo ""
12+
echo "Downloads test data and third party libraries from a remote source external directory to this repository."
13+
}
14+
15+
# Synchronizes a directory recursively while maintaining attributes.
16+
#
17+
# Arguments
18+
# ---------
19+
# VERBOSE : {false, true}
20+
# If true, print verbose messages.
21+
# SRC : str
22+
# The source directory.
23+
# DEST : str
24+
# The destination directory.
25+
syncdir()
26+
{
27+
local VERBOSE=$1
28+
local SRC=$2
29+
local DEST=$3
30+
31+
if [[ ! -d ${DEST} ]]; then
32+
MKDIR_COMMAND="mkdir -p"
33+
if [[ ${VERBOSE} == true ]]; then
34+
MKDIR_COMMAND="${MKDIR_COMMAND} -v"
35+
fi
36+
${MKDIR_COMMAND} ${DEST}
37+
fi
38+
39+
# Some notes on options.
40+
#
41+
# -p: retain permissions
42+
# -u: update destination
43+
# -r: recurse directories
44+
# -l: copy symlinks as symlinks
45+
# -P: progress updates and partial downloads
46+
# -h: human readable
47+
SYNC_COMMAND="rsync -purlPh"
48+
if [[ ${VERBOSE} == true ]]; then
49+
SYNC_COMMAND="${SYNC_COMMAND} -v"
50+
fi
51+
52+
if [[ ${VERBOSE} == true ]]; then
53+
echo ${SYNC_COMMAND} ${SRC} ${DEST}
54+
fi
55+
${SYNC_COMMAND} "${SRC}" "${DEST}"
56+
}
57+
58+
# Synchronizes a version of a library for the given OS.
59+
#
60+
# Arguments
61+
# ---------
62+
# NAME : str
63+
# The top level directory name of the library.
64+
# VERSION : str
65+
# The version number sub-directory.
66+
# REMOTE_EXT_DIR : str
67+
# The directory containing the source libraries.
68+
# LOCAL_EXT_DIR : str
69+
# The destination directory into which to copy the library.
70+
# OS_SUB_DIR : str
71+
# The OS sub-directory name.
72+
# VERBOSE : {false, true}
73+
# If true, print verbose messages.
74+
synclib()
75+
{
76+
local NAME=$1
77+
local VERSION=$2
78+
local REMOTE_EXT_DIR=$3
79+
local LOCAL_EXT_DIR=$4
80+
local OS_SUB_DIR=$5
81+
local VERBOSE=$6
82+
83+
SRC_DIR=${REMOTE_EXT_DIR}/${NAME}/${VERSION}/${OS_SUB_DIR}
84+
DEST_DIR=${LOCAL_EXT_DIR}/${NAME}/${VERSION}
85+
86+
syncdir ${VERBOSE} ${SRC_DIR} ${DEST_DIR}
87+
}
88+
89+
source scripts/utils.sh
90+
91+
# Fixed/default settings
92+
OS_SUB_DIR=linux
93+
if isMac; then
94+
OS_SUB_DIR=osx
95+
elif isLinux; then
96+
if isArm; then
97+
OS_SUB_DIR=linux-arm
98+
else
99+
OS_SUB_DIR=linux
100+
fi
101+
elif isWin; then
102+
OS_SUB_DIR=win
103+
fi
104+
105+
106+
LOCAL_EXT_DIR=third_party
107+
108+
# Parse other settings from arguments
109+
VERBOSE=false
110+
IS_REMOTE_COPY=false
111+
112+
while [[ $# > 0 ]]; do
113+
key="$1"
114+
case $key in
115+
# verbose output
116+
-v|--verbose)
117+
VERBOSE=true
118+
;;
119+
# specify src directory
120+
--src)
121+
shift
122+
REMOTE_EXT_DIR=$1
123+
;;
124+
# specify dst directory
125+
--dst)
126+
shift
127+
LOCAL_EXT_DIR=$1
128+
;;
129+
# force dst to be a copy of src
130+
--remote-copy)
131+
IS_REMOTE_COPY=true
132+
;;
133+
*)
134+
# unknown option
135+
usage
136+
exit 1
137+
;;
138+
esac
139+
shift # past argument or value
140+
done
141+
142+
if [[ ${VERBOSE} == true ]]; then
143+
echo "Using VERBOSE = ${VERBOSE}"
144+
echo "Using REMOTE_EXT_DIR = ${REMOTE_EXT_DIR}"
145+
echo "Using LOCAL_EXT_DIR = ${LOCAL_EXT_DIR}"
146+
echo "Using OS_SUB_DIR = ${OS_SUB_DIR}"
147+
echo "Using REMOTE_TEST_DATA_DIR = ${REMOTE_TEST_DATA_DIR}"
148+
echo "Using IS_REMOTE_COPY = ${IS_REMOTE_COPY}"
149+
fi
150+
151+
# Check that remote directory exists
152+
if [[ ! -d ${REMOTE_EXT_DIR} ]]; then
153+
echo "The remote source/external directory does not exist: ${REMOTE_EXT_DIR}. Did you forget to mount it?"
154+
exit 1
155+
fi
156+
157+
# Check that rsync is available
158+
which rsync > /dev/null
159+
exitCode=$?
160+
if [[ ${exitCode} -ne 0 ]]; then
161+
echo "ERROR: Could not find rsync. Install it manually according to the instructions in the README."
162+
exit ${exitCode}
163+
fi
164+
165+
# Copy libraries
166+
synclib "catch" "1.4.0" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} "." ${VERBOSE}
167+
synclib "ffmpeg" "3.1.4" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
168+
synclib "json_modern_C++" "2.0.1_isx" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} "." ${VERBOSE}
169+
synclib "libtiff" "4.0.8.isx" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
170+
synclib "boost" "1.72.0" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
171+
172+
if isUbuntu; then
173+
synclib "hdf5" "1.10" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} "linux-ubuntu" ${VERBOSE}
174+
if [[ ${IS_REMOTE_COPY} == false ]]; then
175+
mv "${LOCAL_EXT_DIR}/hdf5/1.10/linux-ubuntu" "${LOCAL_EXT_DIR}/hdf5/1.10/linux"
176+
fi
177+
else
178+
synclib "hdf5" "1.10" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
179+
fi;
180+
181+
if isArm; then
182+
synclib "Qt" "5.8" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
183+
synclib "OpenCV" "3.2.0.cuda" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
184+
synclib "OpenBLAS" "0.2.20" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
185+
else
186+
# Change to 5.12 when ready - see IDPS-87
187+
synclib "Qt" "5.8" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
188+
synclib "OpenCV" "3.2.0.no_mkl" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
189+
190+
if isLinux; then
191+
synclib "Qt" "5.5" ${REMOTE_EXT_DIR} ${LOCAL_EXT_DIR} ${OS_SUB_DIR} ${VERBOSE}
192+
fi
193+
fi
194+
195+
# Copy test data
196+
REMOTE_TEST_DATA_DIR=test_data_structured
197+
LOCAL_TEST_DATA_DIR=test_data
198+
if [[ ${IS_REMOTE_COPY} == true ]]; then
199+
LOCAL_TEST_DATA_DIR="${LOCAL_EXT_DIR}/test_data_structured"
200+
fi
201+
# Data for unit tests
202+
syncdir ${VERBOSE} ${REMOTE_EXT_DIR}/${REMOTE_TEST_DATA_DIR}/unit_test ${LOCAL_TEST_DATA_DIR}

0 commit comments

Comments
 (0)