Skip to content

Commit 68bfe96

Browse files
llxiallxia
committed
Update dev.external workflow for CRIU
Signed-off-by: Lan Xia <[email protected]>
1 parent 48ed774 commit 68bfe96

File tree

4 files changed

+129
-50
lines changed

4 files changed

+129
-50
lines changed

‎buildenv/jenkins/openjdk_tests

+108
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,57 @@ timestamps{
162162
}
163163
parallel testJobs
164164
}
165+
} else if (!JOB_NAME.contains("_imagePull") && !JOB_NAME.contains("_imageUpload") && params.TARGET == "dev.external" ) {
166+
// trigger xxx_imageUpload test job, then xxx_imagePull test job
167+
def commonLabel = "sw.tool.podman&&sw.tool.container.criu"
168+
def imageUploadMap = [
169+
'x86-64_linux' : [
170+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.broadwell"],
171+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22&&hw.arch.x86.amd"],
172+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.broadwell"],
173+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.amd"],
174+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8&&hw.arch.x86.skylake"]
175+
]
176+
]
177+
def imagePullMap = [
178+
'x86-64_linux' : [
179+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.ubuntu.22"],
180+
['LABEL_ADDITION' : "${commonLabel}&&sw.os.rhel.8"]
181+
]
182+
]
183+
if (params.PLATFORM && imageUploadMap[params.PLATFORM] != null && imagePullMap[params.PLATFORM] != null) {
184+
def imageUploadJobName = "${JOB_NAME}_imageUpload"
185+
def imagePullJobName = "${JOB_NAME}_imagePull"
186+
generateJobViaAutoGen(imageUploadJobName)
187+
generateJobViaAutoGen(imagePullJobName)
188+
189+
def imageUploadJobs = [:]
190+
def target = "testList TESTLIST=disabled.criu_pingPerf_testCreateRestoreImageAndPushToRegistry,disabled.criu-portable-checkpoint_test,disabled.criu-ubi-portable-checkpoint_test"
191+
echo "Trigger ${imageUploadJobName} job ..."
192+
for (int i = 0; i < imageUploadMap[params.PLATFORM].size(); i++) {
193+
def labelAddition = imageUploadMap[params.PLATFORM][i].LABEL_ADDITION
194+
def newParams = ["LABEL_ADDITION" : labelAddition, "TARGET" : "${target}", "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
195+
def childParams = changeParam(params, newParams)
196+
imageUploadJobs["${imageUploadJobName}_${i}]"] = {
197+
build job: imageUploadJobName, parameters: childParams
198+
}
199+
}
200+
parallel imageUploadJobs
201+
202+
def imagePullJobs = [:]
203+
echo "Trigger ${imagePullJobName} job ..."
204+
for (int i = 0; i < imagePullMap[params.PLATFORM].size(); i++) {
205+
def labelAddition = imagePullMap[params.PLATFORM][i].LABEL_ADDITION
206+
def newParams = ["LABEL_ADDITION" : labelAddition, "DOCKER_REGISTRY_DIR" : "${JOB_NAME}:${BUILD_NUMBER}"]
207+
def childParams = changeParam(params, newParams)
208+
imagePullJobs["${imagePullJobName}_${i}]"] = {
209+
build job: imagePullJobName, parameters: childParams
210+
}
211+
}
212+
parallel imagePullJobs
213+
} else {
214+
assert false : "Cannot find key PLATFORM: ${params.PLATFORM} in imageGenMap: ${imageGenMap} and/or imagePullMap: ${imagePullMap}."
215+
}
165216
} else {
166217
if (PLATFORM_MAP.containsKey(params.PLATFORM)) {
167218
env.EXTRA_OPTIONS = params.EXTRA_OPTIONS ? params.EXTRA_OPTIONS : ""
@@ -466,3 +517,60 @@ def forceCleanWS() {
466517
}
467518
}
468519

520+
def changeParam(paramsList, newParams) {
521+
def childParams = []
522+
// loop through all the params and change the parameters if needed
523+
paramsList.each { param ->
524+
def value = param.value.toString()
525+
newParams.each { newParam ->
526+
527+
if (param.key == newParam.key) {
528+
value = newParam.value.toString()
529+
}
530+
}
531+
if (value == "true" || value == "false") {
532+
childParams << booleanParam(name: param.key, value: value.toBoolean())
533+
} else {
534+
childParams << string(name: param.key, value: value)
535+
}
536+
}
537+
return childParams
538+
}
539+
540+
def generateJobViaAutoGen(testJobName) {
541+
def JobHelper = library(identifier: 'openjdk-jenkins-helper@master').JobHelper
542+
if (params.GENERATE_JOBS || !JobHelper.jobIsRunnable(testJobName as String)) {
543+
def jobParams = []
544+
jobParams << string(name: 'TEST_JOB_NAME', value: testJobName)
545+
jobParams << string(name: 'ARCH_OS_LIST', value: params.PLATFORM)
546+
jobParams << booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: false)
547+
548+
def jdk_impl = params.JDK_IMPL ?: ""
549+
def jdk_version = params.JDK_VERSION ?: ""
550+
if (testJobName.startsWith("Test_openjdk")) {
551+
def tokens = testJobName.split('_');
552+
def level = ""
553+
def group = ""
554+
if (tokens.size() > 3) {
555+
if (!jdk_version) {
556+
jdk_version = tokens[1]
557+
}
558+
if (!jdk_impl) {
559+
jdk_impl = tokens[2]
560+
}
561+
if (tokens[3].contains(".")) {
562+
level = tokens[3].split("\\.")[0]
563+
group = tokens[3].split("\\.")[1]
564+
if (level && group) {
565+
jobParams << string(name: 'LEVELS', value: level)
566+
jobParams << string(name: 'GROUPS', value: group)
567+
}
568+
}
569+
}
570+
}
571+
jobParams << string(name: 'JDK_IMPL', value: jdk_impl)
572+
jobParams << string(name: 'JDK_VERSIONS', value: jdk_version)
573+
574+
build job: 'Test_Job_Auto_Gen', parameters: jobParams, propagate: true
575+
}
576+
}

‎external/criu/pingPerf.sh

+10-23
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ pingPerfZipPath=""
1818
testJDKPath=""
1919
jdkVersion=""
2020
restoreImage="ol-instanton-test-pingperf-restore"
21-
job_name=""
22-
build_number=""
21+
job_name=$JOB_NAME
22+
build_number=$BUILD_NUMBER
2323
docker_registry_dir=""
2424
docker_os="ubi"
2525
node_label_current_os=""
@@ -185,37 +185,24 @@ pushImage() {
185185

186186
restore_ready_checkpoint_image_folder="${DOCKER_REGISTRY_URL}/${job_name}/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${node_label_current_os}-${node_label_micro_architecture}"
187187
tagged_restore_ready_checkpoint_image_num="${restore_ready_checkpoint_image_folder}:${build_number}"
188-
tagged_restore_ready_checkpoint_image_latest="${restore_ready_checkpoint_image_folder}:latest"
189188

190189
# Push a docker image with build_num for records
191190
echo "tag $restoreImage $tagged_restore_ready_checkpoint_image_num"
192191
sudo podman tag $restoreImage $tagged_restore_ready_checkpoint_image_num
193192
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_num}"
194193
sudo podman push $tagged_restore_ready_checkpoint_image_num
195194

196-
if [[ "$job_name" == *"criu_image_upload"* ]]; then
197-
# Push another copy as the nightly default latest
198-
echo "tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest"
199-
sudo podman tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest
200-
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_latest} to docker registry"
201-
sudo podman push $tagged_restore_ready_checkpoint_image_latest
202-
fi
203-
204195
dockerRegistryLogout
205196
}
206197

207198
getImageNameList() {
208-
if [ ! -z "${docker_registry_dir}" ]; then
209-
echo "Testing image from specified docker_registry_dir: $docker_registry_dir"
210-
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/${docker_registry_dir}")
211-
else
212-
echo "Testing images from nightly builds"
213-
image_os_combo_list=($CRIU_XLINUX_COMBO_LIST)
214-
for image_os_combo in ${image_os_combo_list[@]}
215-
do
216-
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/criu_image_upload/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${image_os_combo}:latest")
217-
done
218-
fi
199+
200+
echo "Testing images from job_name:build_number : ${job_name}:${build_number}"
201+
image_os_combo_list=($CRIU_XLINUX_COMBO_LIST)
202+
for image_os_combo in ${image_os_combo_list[@]}
203+
do
204+
restore_docker_image_name_list+=("${DOCKER_REGISTRY_URL}/${job_name}/pingperf_${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${PLATFORM}-${image_os_combo}:${build_number}")
205+
done
219206
}
220207

221208
pullImageUnprivilegedRestore() {
@@ -291,9 +278,9 @@ setup() {
291278
IFS=':' read -r -a dir_array <<< "$docker_registry_dir"
292279
job_name=${dir_array[0]}
293280
build_number=${dir_array[1]}
294-
echo "job_name: $job_name"
295281
echo "build_number: $build_number"
296282
fi
283+
echo "job_name: $job_name"
297284

298285
node_label_micro_architecture=""
299286
node_label_current_os=""

‎external/criuSettings.mk

-1
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,3 @@
1414

1515
export CRIU_XLINUX_COMBO_LIST=sw.os.ubuntu.22-hw.arch.x86.broadwell sw.os.ubuntu.22-hw.arch.x86.amd sw.os.rhel.8-hw.arch.x86.broadwell sw.os.rhel.8-hw.arch.x86.amd sw.os.rhel.8-hw.arch.x86.skylake
1616
# not available: sw.os.ubuntu.22-hw.arch.x86.skylake
17-
export CRIU_DEFAULT_IMAGE_JOB_NAME=criu_image_upload

‎external/external.sh

+11-26
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ test=derby
2727
testtarget=""
2828
platform="linux_x86-64"
2929
portable="false"
30-
job_name=""
31-
build_number=""
30+
job_name=$JOB_NAME
31+
build_number=$BUILD_NUMBER
3232
node_name=""
3333
node_labels=""
3434
node_label_micro_architecture=""
@@ -293,22 +293,12 @@ if [ $command_type == "run" ]; then
293293

294294
restore_ready_checkpoint_image_folder="${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}"
295295
tagged_restore_ready_checkpoint_image_num="${restore_ready_checkpoint_image_folder}:${build_number}"
296-
tagged_restore_ready_checkpoint_image_latest="${restore_ready_checkpoint_image_folder}:latest"
297296

298297
# Push a docker image with build_num for records
299298
echo "tagged_restore_ready_checkpoint_image_num is $tagged_restore_ready_checkpoint_image_num"
300299
$container_commit --change='ENTRYPOINT ["/bin/bash", "/test_restore.sh"]' $test-test $tagged_restore_ready_checkpoint_image_num
301300
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_num}"
302301
$container_push $tagged_restore_ready_checkpoint_image_num
303-
304-
# Push another copy as the nightly default latest
305-
echo "Change Tag from build_number to latest"
306-
$container_tag $tagged_restore_ready_checkpoint_image_num $tagged_restore_ready_checkpoint_image_latest
307-
308-
if [[ "$job_name" == *"$criu_default_image_job_name"* ]]; then
309-
echo "Pushing docker image ${tagged_restore_ready_checkpoint_image_latest} to docker registry"
310-
$container_push $tagged_restore_ready_checkpoint_image_latest
311-
fi
312302
$container_logout $docker_registry_url
313303
else
314304
echo "Docker Registry is not available on this Jenkins"
@@ -338,18 +328,14 @@ if [ $command_type == "load" ]; then
338328
fi
339329

340330
restore_docker_image_name_list=()
341-
if [ ! -z "${docker_registry_dir}" ]; then
342-
echo "Testing image from specified DOCKER_REGISTRY_DIR"
343-
restore_docker_image_name_list+=("${docker_registry_url}/${docker_registry_dir}")
344-
else
345-
echo "Testing images from nightly builds"
346-
image_os_micro_architecture_list=($criu_combo_os_microarch_list)
347-
for image_os_micro_architecture in ${image_os_micro_architecture_list[@]}
348-
do
349-
restore_docker_image_name_list+=("${docker_registry_url}/$criu_default_image_job_name/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${image_os_micro_architecture}:latest")
350-
done
351-
fi
352-
331+
332+
echo "Testing images from nightly builds"
333+
image_os_micro_architecture_list=($criu_combo_os_microarch_list)
334+
for image_os_micro_architecture in ${image_os_micro_architecture_list[@]}
335+
do
336+
restore_docker_image_name_list+=("${docker_registry_url}/$job_name/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${image_os_micro_architecture}:${build_number}")
337+
done
338+
353339
echo "The host machine OS is ${node_label_current_os}, and micro-architecture is ${node_label_micro_architecture}"
354340
for restore_docker_image_name in ${restore_docker_image_name_list[@]}
355341
do
@@ -390,6 +376,5 @@ if [ $command_type == "clean" ]; then
390376
fi
391377
$container_rm -f $test-test; $container_rmi -f adoptopenjdk-$test-test:${JDK_VERSION}-$package-$docker_os-${JDK_IMPL}-$build_type
392378
$container_rm -f restore-test
393-
$container_rmi -f ${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:latest
394-
$container_rmi -f ${docker_registry_url}/${criu_default_image_job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:latest
379+
$container_rmi -f ${docker_registry_url}/${job_name}/${JDK_VERSION}-${JDK_IMPL}-${docker_os}-${platform}-${node_label_current_os}-${node_label_micro_architecture}:${build_number}
395380
fi

0 commit comments

Comments
 (0)