Skip to content

Commit 9688cc8

Browse files
jiridanekjiridanek
committed
RHOAIENG-19043: chore(tests): fix 2024b ci builds after repo restructuring
1 parent cea9eb4 commit 9688cc8

File tree

4 files changed

+39
-27
lines changed

4 files changed

+39
-27
lines changed

‎Makefile

+13-12
Original file line numberDiff line numberDiff line change
@@ -62,23 +62,24 @@ else
6262
CONTAINER_ENGINE := podman
6363
endif
6464

65-
# Build function for the notebok image:
65+
# Build function for the notebook image:
6666
# ARG 1: Image tag name.
67-
# ARG 2: Path of image context we want to build.
68-
# ARG 3: Base image tag name (optional).
67+
# ARG 2: Path of container engine build context.
68+
# ARG 3: Path of image context we want to build.
69+
# ARG 4: Base image tag name (optional).
6970
define build_image
7071
$(eval IMAGE_NAME := $(IMAGE_REGISTRY):$(1)-$(IMAGE_TAG))
7172
$(info # Building $(IMAGE_NAME) image...)
7273
$(if $(3),
73-
$(eval BASE_IMAGE_NAME := $(IMAGE_REGISTRY):$(3)-$(IMAGE_TAG))
74+
$(eval BASE_IMAGE_NAME := $(IMAGE_REGISTRY):$(4)-$(IMAGE_TAG))
7475
$(eval BUILD_ARGS := --build-arg BASE_IMAGE=$(BASE_IMAGE_NAME)),
7576
$(eval BUILD_ARGS :=)
7677
)
77-
$(ROOT_DIR)/scripts/sandbox.py --dockerfile '$(2)/Dockerfile' -- \
78-
$(CONTAINER_ENGINE) build $(CONTAINER_BUILD_CACHE_ARGS) --tag $(IMAGE_NAME) --file '$(2)/Dockerfile' $(BUILD_ARGS) {}\;
78+
$(ROOT_DIR)/scripts/sandbox.py --dockerfile '$(2)/$(3)/Dockerfile' --build-context '$(2)' -- \
79+
$(CONTAINER_ENGINE) build $(CONTAINER_BUILD_CACHE_ARGS) --tag $(IMAGE_NAME) --file '$(2)/$(3)/Dockerfile' $(BUILD_ARGS) {}\;
7980
endef
8081

81-
# Push function for the notebok image:
82+
# Push function for the notebook image:
8283
# ARG 1: Path of image context we want to build.
8384
define push_image
8485
$(eval IMAGE_NAME := $(IMAGE_REGISTRY):$(subst /,-,$(1))-$(IMAGE_TAG))
@@ -94,10 +95,10 @@ endef
9495
# BUILD_DEPENDENT_IMAGES: only build images that were explicitly given as a goal on command line
9596
# PUSH_IMAGES: allows skipping podman push
9697
define image
97-
$(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...)
98+
$(info #*# Image build directory: <$(RELEASE)/$(2)> #(MACHINE-PARSED LINE)#*#...)
9899

99100
$(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))),
100-
$(call build_image,$(1),$(2),$(3))
101+
$(call build_image,$(1),$(RELEASE),$(2),$(3))
101102
102103
$(if $(PUSH_IMAGES:no=),
103104
$(call push_image,$(1))
@@ -269,7 +270,7 @@ endif
269270
deploy9-%: bin/kubectl bin/yq
270271
$(eval TARGET := $(shell echo $* | sed 's/-ubi9-python.*//'))
271272
$(eval PYTHON_VERSION := $(shell echo $* | sed 's/.*-python-//'))
272-
$(eval NOTEBOOK_DIR := $(subst -,/,$(subst cuda-,,$(TARGET)))/ubi9-python-$(PYTHON_VERSION)/kustomize/base)
273+
$(eval NOTEBOOK_DIR := $(RELEASE)/$(subst -,/,$(subst cuda-,,$(TARGET)))/ubi9-python-$(PYTHON_VERSION)/kustomize/base)
273274
ifndef NOTEBOOK_TAG
274275
$(eval NOTEBOOK_TAG := $*-$(IMAGE_TAG))
275276
endif
@@ -290,7 +291,7 @@ undeploy9-%: bin/kubectl
290291
deploy-c9s-%: bin/kubectl bin/yq
291292
$(eval TARGET := $(shell echo $* | sed 's/-c9s-python.*//'))
292293
$(eval PYTHON_VERSION := $(shell echo $* | sed 's/.*-python-//'))
293-
$(eval NOTEBOOK_DIR := $(subst -,/,$(subst cuda-,,$(TARGET)))/c9s-python-$(PYTHON_VERSION)/kustomize/base)
294+
$(eval NOTEBOOK_DIR := $(RELEASE)/$(subst -,/,$(subst cuda-,,$(TARGET)))/c9s-python-$(PYTHON_VERSION)/kustomize/base)
294295
ifndef NOTEBOOK_TAG
295296
$(eval NOTEBOOK_TAG := $*-$(IMAGE_TAG))
296297
endif
@@ -311,7 +312,7 @@ undeploy-c9s-%: bin/kubectl
311312
.PHONY: test
312313
test-%: bin/kubectl
313314
$(info # Running tests for $* notebook...)
314-
@./scripts/test_jupyter_with_papermill.sh $*
315+
@./scripts/test_jupyter_with_papermill.sh $(RELEASE) $*
315316

316317
# Validate that runtime image meets minimum criteria
317318
# This validation is created from subset of https://github.com/elyra-ai/elyra/blob/9c417d2adc9d9f972de5f98fd37f6945e0357ab9/Makefile#L325

‎ci/cached-builds/gen_gha_matrix_jobs.py

100644100755
+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env python3
2+
13
import argparse
24
import itertools
35
import json

‎scripts/sandbox.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@
1919

2020
class Args(argparse.Namespace):
2121
dockerfile: pathlib.Path
22+
build_context: pathlib.Path
2223
remaining: list[str]
2324

2425

2526
def main() -> int:
2627
p = argparse.ArgumentParser(allow_abbrev=False)
2728
p.add_argument("--dockerfile", type=pathlib.Path, required=True)
29+
p.add_argument("--build-context", type=pathlib.Path, required=True)
2830
p.add_argument('remaining', nargs=argparse.REMAINDER)
2931

3032
args = cast(Args, p.parse_args())
@@ -42,12 +44,12 @@ def main() -> int:
4244
subprocess.check_call([MAKE, "bin/buildinputs"], cwd=ROOT_DIR)
4345
stdout = subprocess.check_output([ROOT_DIR / "bin/buildinputs", str(args.dockerfile)],
4446
text=True, cwd=ROOT_DIR)
45-
prereqs = [pathlib.Path(file) for file in json.loads(stdout)] if stdout != "\n" else []
47+
prereqs = [args.build_context / pathlib.Path(file) for file in json.loads(stdout)] if stdout != "\n" else []
4648
print(f"{prereqs=}")
4749

4850
with tempfile.TemporaryDirectory(delete=True) as tmpdir:
4951
setup_sandbox(prereqs, pathlib.Path(tmpdir))
50-
command = [arg if arg != "{};" else tmpdir for arg in args.remaining[1:]]
52+
command = [arg if arg != "{};" else tmpdir / args.build_context for arg in args.remaining[1:]]
5153
print(f"running {command=}")
5254
try:
5355
subprocess.check_call(command)

‎scripts/test_jupyter_with_papermill.sh

+20-13
Original file line numberDiff line numberDiff line change
@@ -99,18 +99,20 @@ function _get_accelerator_flavor()
9999
# Returns the absolute path of notebook resources in the notebooks/ repo based on the notebook name
100100
#
101101
# Arguments:
102-
# $1 : Name of the notebook identifier
103-
# $2 : [optional] Subdirectory to append to computed absolute path
102+
# $1 : Release to test (2024a, 2024n, ...)
103+
# $2 : Name of the notebook identifier
104+
# $3 : [optional] Subdirectory to append to computed absolute path
104105
# - path should NOT start with a leading /
105106
#
106107
# Returns:
107108
# Absolute path to the jupyter notebook directory for the given notebook test target
108109
function _get_jupyter_notebook_directory()
109110
{
110-
local notebook_id="${1:-}"
111-
local subpath="${2:-}"
111+
local release="${1:-}"
112+
local notebook_id="${2:-}"
113+
local subpath="${3:-}"
112114

113-
local jupyter_base="${root_repo_directory}/jupyter"
115+
local jupyter_base="${root_repo_directory}/${release}/jupyter"
114116
local directory="${jupyter_base}/${notebook_id}/${os_flavor}-${python_flavor}${subpath:+"/$subpath"}"
115117

116118
printf '%s' "${directory}"
@@ -264,11 +266,12 @@ function _create_test_versions_source_of_truth()
264266
# $1 : Name of the notebook identifier
265267
function _run_test()
266268
{
267-
local notebook_id="${1:-}"
269+
local release="${1:-}"
270+
local notebook_id="${2:-}"
268271

269272
local test_notebook_file='test_notebook.ipynb'
270273
local repo_test_directory=
271-
repo_test_directory="$(_get_jupyter_notebook_directory "${notebook_id}" "test")"
274+
repo_test_directory="$(_get_jupyter_notebook_directory "${release}" "${notebook_id}" "test")"
272275
local output_file_prefix=
273276
output_file_prefix=$(tr '/' '-' <<< "${notebook_id}_${os_flavor}")
274277

@@ -323,8 +326,10 @@ function _image_derived_from_datascience()
323326
# Convenience function that will invoke the minimal and datascience papermill tests against the running notebook workload
324327
function _test_datascience_notebook()
325328
{
326-
_run_test "${jupyter_minimal_notebook_id}"
327-
_run_test "${jupyter_datascience_notebook_id}"
329+
local release="${1:-}"
330+
331+
_run_test "${release}" "${jupyter_minimal_notebook_id}"
332+
_run_test "${release}" "${jupyter_datascience_notebook_id}"
328333
}
329334

330335
# Description:
@@ -335,6 +340,7 @@ function _test_datascience_notebook()
335340
# - All relevant tests based on the notebook_id are invoked
336341
function _handle_test()
337342
{
343+
local release="${1:-}"
338344
local notebook_id=
339345

340346
# Due to existing logic - cuda accelerator value needs to be treated as empty string
@@ -369,15 +375,16 @@ function _handle_test()
369375
"${kbin}" exec "${notebook_workload_name}" -- /bin/sh -c "python3 -m pip install papermill"
370376

371377
if _image_derived_from_datascience "${notebook_id}" ; then
372-
_test_datascience_notebook
378+
_test_datascience_notebook "${release}"
373379
fi
374380

375381
if [ -n "${notebook_id}" ] && ! [ "${notebook_id}" = "${jupyter_datascience_notebook_id}" ]; then
376-
_run_test "${notebook_id}"
382+
_run_test "${release}" "${notebook_id}"
377383
fi
378384
}
379385

380-
test_target="${1:-}"
386+
release="${1:-}"
387+
test_target="${2:-}"
381388

382389
# Hard-coded list of supported "notebook_id" values - based on notebooks/ repo Makefile
383390
jupyter_minimal_notebook_id='minimal'
@@ -410,5 +417,5 @@ _wait_for_workload "${notebook_name}"
410417

411418
notebook_workload_name=$("${kbin}" get pods -l app="${notebook_name}" -o jsonpath='{.items[0].metadata.name}')
412419

413-
_handle_test
420+
_handle_test "${release}"
414421

0 commit comments

Comments
 (0)