Skip to content

Commit acb136d

Browse files
llxiallxia
committed
Update dev.external workflow for CRIU
Signed-off-by: Lan Xia <[email protected]>
1 parent 4c168e5 commit acb136d

File tree

4 files changed

+132
-50
lines changed

4 files changed

+132
-50
lines changed

‎buildenv/jenkins/openjdk_tests

+111
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,60 @@ timestamps{
162162
}
163163
parallel testJobs
164164
}
165+
} else if (!JOB_NAME.contains("_imagePull") && !JOB_NAME.contains("_imageUpload") && params.TARGET == "dev.external" ) {
166+
// trigger xxx_imageUpload test job, then xxx_imagePull test job
167+
def commonLabel = "sw.tool.podman&&sw.tool.container.criu"
168+
if (params.LABEL_ADDITION) {
169+
commonLabel += "&&${params.LABEL_ADDITION}"
170+
}
171+
def imageUploadMap = [
172+
'x86-64_linux' : [
173+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.broadwell"],
174+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.amd"],
175+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.broadwell"],
176+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.amd"],
177+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.skylake"]
178+
]
179+
]
180+
def imagePullMap = [
181+
'x86-64_linux' : [
182+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22"],
183+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8"]
184+
]
185+
]
186+
if (params.PLATFORM && imageUploadMap[params.PLATFORM] != null && imagePullMap[params.PLATFORM] != null) {
187+
def imageUploadJobName = "${JOB_NAME}_imageUpload"
188+
def imagePullJobName = "${JOB_NAME}_imagePull"
189+
generateJobViaAutoGen(imageUploadJobName)
190+
generateJobViaAutoGen(imagePullJobName)
191+
192+
def imageUploadJobs = [:]
193+
def target = "testList TESTLIST=disabled.criu_pingPerf_testCreateRestoreImageAndPushToRegistry,disabled.criu-portable-checkpoint_test,disabled.criu-ubi-portable-checkpoint_test"
194+
echo "Trigger ${imageUploadJobName} job ..."
195+
for (int i = 0; i < imageUploadMap[params.PLATFORM].size(); i++) {
196+
def labelAddition = imageUploadMap[params.PLATFORM][i].LABEL_ADDITION
197+
def newParams = ["LABEL_ADDITION" : labelAddition, "TARGET" : "${target}", "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
198+
def childParams = changeParam(params, newParams)
199+
imageUploadJobs["${imageUploadJobName}_${i}]"] = {
200+
build job: imageUploadJobName, parameters: childParams
201+
}
202+
}
203+
parallel imageUploadJobs
204+
205+
def imagePullJobs = [:]
206+
echo "Trigger ${imagePullJobName} job ..."
207+
for (int i = 0; i < imagePullMap[params.PLATFORM].size(); i++) {
208+
def labelAddition = imagePullMap[params.PLATFORM][i].LABEL_ADDITION
209+
def newParams = ["LABEL_ADDITION" : labelAddition, "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
210+
def childParams = changeParam(params, newParams)
211+
imagePullJobs["${imagePullJobName}_${i}]"] = {
212+
build job: imagePullJobName, parameters: childParams
213+
}
214+
}
215+
parallel imagePullJobs
216+
} else {
217+
assert false : "Cannot find key PLATFORM: ${params.PLATFORM} in imageGenMap: ${imageGenMap} and/or imagePullMap: ${imagePullMap}."
218+
}
165219
} else {
166220
if (PLATFORM_MAP.containsKey(params.PLATFORM)) {
167221
env.EXTRA_OPTIONS = params.EXTRA_OPTIONS ? params.EXTRA_OPTIONS : ""
@@ -466,3 +520,60 @@ def forceCleanWS() {
466520
}
467521
}
468522

523+
def changeParam(paramsList, newParams) {
524+
def childParams = []
525+
// loop through all the params and change the parameters if needed
526+
paramsList.each { param ->
527+
def value = param.value.toString()
528+
newParams.each { newParam ->
529+
530+
if (param.key == newParam.key) {
531+
value = newParam.value.toString()
532+
}
533+
}
534+
if (value == "true" || value == "false") {
535+
childParams << booleanParam(name: param.key, value: value.toBoolean())
536+
} else {
537+
childParams << string(name: param.key, value: value)
538+
}
539+
}
540+
return childParams
541+
}
542+
543+
def generateJobViaAutoGen(testJobName) {
544+
def JobHelper = library(identifier: 'openjdk-jenkins-helper@master').JobHelper
545+
if (params.GENERATE_JOBS || !JobHelper.jobIsRunnable(testJobName as String)) {
546+
def jobParams = []
547+
jobParams << string(name: 'TEST_JOB_NAME', value: testJobName)
548+
jobParams << string(name: 'ARCH_OS_LIST', value: params.PLATFORM)
549+
jobParams << booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: false)
550+
551+
def jdk_impl = params.JDK_IMPL ?: ""
552+
def jdk_version = params.JDK_VERSION ?: ""
553+
if (testJobName.startsWith("Test_openjdk")) {
554+
def tokens = testJobName.split('_');
555+
def level = ""
556+
def group = ""
557+
if (tokens.size() > 3) {
558+
if (!jdk_version) {
559+
jdk_version = tokens[1]
560+
}
561+
if (!jdk_impl) {
562+
jdk_impl = tokens[2]
563+
}
564+
if (tokens[3].contains(".")) {
565+
level = tokens[3].split("\\.")[0]
566+
group = tokens[3].split("\\.")[1]
567+
if (level && group) {
568+
jobParams << string(name: 'LEVELS', value: level)
569+
jobParams << string(name: 'GROUPS', value: group)
570+
}
571+
}
572+
}
573+
}
574+
jobParams << string(name: 'JDK_IMPL', value: jdk_impl)
575+
jobParams << string(name: 'JDK_VERSIONS', value: jdk_version)
576+
577+
build job: 'Test_Job_Auto_Gen', parameters: jobParams, propagate: true
578+
}
579+
}

‎external/criu/pingPerf.sh

+10-23
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ pingPerfZipPath=""
1818
testJDKPath=""
1919
jdkVersion=""
2020
restoreImage="ol-instanton-test-pingperf-restore"
21-
job_name=""
22-
build_number=""
21+
job_name=$JOB_NAME
22+
build_number=$BUILD_NUMBER
2323
docker_registry_dir=""
2424
docker_os="ubi"
2525
node_label_current_os=""
@@ -185,37 +185,24 @@ pushImage() {
185185

186186
restore_ready_checkpoint_image_folder="${DOCKER_REGISTRY_URL}/${job_name}/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${node_label_current_os}-${node_label_micro_architecture}"
187187
tagged_restore_ready_checkpoint_image_num="${restore_ready_checkpoint_image_folder}:${build_number}"
188-
tagged_restore_ready_checkpoint_image_latest="${restore_ready_checkpoint_image_folder}:latest"
189188

190189
# Push a docker image with build_num for records
191190
echo "tag $restoreImage $tagged_restore_ready_checkpoint_image_num"
192191
sudo podman tag $restoreImage $tagged_restore_ready_checkpoint_image_num
193192
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_num}"
194193
sudo podman push $tagged_restore_ready_checkpoint_image_num
195194

196-
if [[ "$job_name" == *"criu_image_upload"* ]]; then
197-
# Push another copy as the nightly default latest
198-
echo "tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest"
199-
sudo podman tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest
200-
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_latest} to docker registry"
201-
sudo podman push $tagged_restore_ready_checkpoint_image_latest
202-
fi
203-
204195
dockerRegistryLogout
205196
}
206197

207198
getImageNameList() {
208-
if [ ! -z "${docker_registry_dir}" ]; then
209-
echo "Testing image from specified docker_registry_dir: $docker_registry_dir"
210-
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/${docker_registry_dir}")
211-
else
212-
echo "Testing images from nightly builds"
213-
image_os_combo_list=($CRIU_XLINUX_COMBO_LIST)
214-
for image_os_combo in ${image_os_combo_list[@]}
215-
do
216-
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/criu_image_upload/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${image_os_combo}:latest")
217-
done
218-
fi
199+
200+
echo "Testing images from job_name:build_number : ${job_name}:${build_number}"
201+
image_os_combo_list=($CRIU_XLINUX_COMBO_LIST)
202+
for image_os_combo in ${image_os_combo_list[@]}
203+
do
204+
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/${job_name}/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${image_os_combo}:${build_number}")
205+
done
219206
}
220207

221208
pullImageUnprivilegedRestore() {
@@ -291,9 +278,9 @@ setup() {
291278
IFS=':' read -r -a dir_array <<< "$docker_registry_dir"
292279
job_name=${dir_array[0]}
293280
build_number=${dir_array[1]}
294-
echo "job_name: $job_name"
295281
echo "build_number: $build_number"
296282
fi
283+
echo "job_name: $job_name"
297284

298285
node_label_micro_architecture=""
299286
node_label_current_os=""

‎external/criuSettings.mk

-1
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,3 @@
1414

1515
export CRIU_XLINUX_COMBO_LIST=sw.os.ubuntu.22-hw.arch.x86.broadwell sw.os.ubuntu.22-hw.arch.x86.amd sw.os.rhel.8-hw.arch.x86.broadwell sw.os.rhel.8-hw.arch.x86.amd sw.os.rhel.8-hw.arch.x86.skylake
1616
# not available: sw.os.ubuntu.22-hw.arch.x86.skylake
17-
export CRIU_DEFAULT_IMAGE_JOB_NAME=criu_image_upload

‎external/external.sh

+11-26
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ test=derby
2727
testtarget=""
2828
platform="linux_x86-64"
2929
portable="false"
30-
job_name=""
31-
build_number=""
30+
job_name=$JOB_NAME
31+
build_number=$BUILD_NUMBER
3232
node_name=""
3333
node_labels=""
3434
node_label_micro_architecture=""
@@ -293,22 +293,12 @@ if [ $command_type == "run" ]; then
293293

294294
restore_ready_checkpoint_image_folder="${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}"
295295
tagged_restore_ready_checkpoint_image_num="${restore_ready_checkpoint_image_folder}:${build_number}"
296-
tagged_restore_ready_checkpoint_image_latest="${restore_ready_checkpoint_image_folder}:latest"
297296

298297
# Push a docker image with build_num for records
299298
echo "tagged_restore_ready_checkpoint_image_num is $tagged_restore_ready_checkpoint_image_num"
300299
$container_commit --change='ENTRYPOINT ["/bin/bash", "/test_restore.sh"]' $test-test $tagged_restore_ready_checkpoint_image_num
301300
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_num}"
302301
$container_push $tagged_restore_ready_checkpoint_image_num
303-
304-
# Push another copy as the nightly default latest
305-
echo "Change Tag from build_number to latest"
306-
$container_tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest
307-
308-
if [[ "$job_name" == *"$criu_default_image_job_name"* ]]; then
309-
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_latest} to docker registry"
310-
$container_push $tagged_restore_ready_checkpoint_image_latest
311-
fi
312302
$container_logout $docker_registry_url
313303
else
314304
echo "Docker Registry is not available on this Jenkins"
@@ -338,18 +328,14 @@ if [ $command_type == "load" ]; then
338328
fi
339329

340330
restore_docker_image_name_list=()
341-
if [ ! -z "${docker_registry_dir}" ]; then
342-
echo "Testing image from specified DOCKER_REGISTRY_DIR"
343-
restore_docker_image_name_list+=("${docker_registry_url}/${docker_registry_dir}")
344-
else
345-
echo "Testing images from nightly builds"
346-
image_os_micro_architecture_list=($criu_combo_os_microarch_list)
347-
for image_os_micro_architecture in ${image_os_micro_architecture_list[@]}
348-
do
349-
restore_docker_image_name_list+=("${docker_registry_url}/$criu_default_image_job_name/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${image_os_micro_architecture}:latest")
350-
done
351-
fi
352-
331+
332+
echo "Testing images from nightly builds"
333+
image_os_micro_architecture_list=($criu_combo_os_microarch_list)
334+
for image_os_micro_architecture in ${image_os_micro_architecture_list[@]}
335+
do
336+
restore_docker_image_name_list+=("${docker_registry_url}/$job_name/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${image_os_micro_architecture}:${build_number}")
337+
done
338+
353339
echo "The host machine OS is ${node_label_current_os}, and micro-architecture is ${node_label_micro_architecture}"
354340
for restore_docker_image_name in ${restore_docker_image_name_list[@]}
355341
do
@@ -390,6 +376,5 @@ if [ $command_type == "clean" ]; then
390376
fi
391377
$container_rm -f $test-test; $container_rmi -f adoptopenjdk-$test-test:${JDK_VERSION}-$package-$docker_os-${JDK_IMPL}-$build_type
392378
$container_rm -f restore-test
393-
$container_rmi -f ${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:latest
394-
$container_rmi -f ${docker_registry_url}/${criu_default_image_job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:latest
379+
$container_rmi -f ${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:${build_number}
395380
fi

0 commit comments

Comments
 (0)