RHOAIENG-19043: chore(tests): fix 2024b ci builds after repo restructuring

Author: jiridanek

Makefile

@@ -74,7 +74,7 @@ define build_image
 		$(eval BUILD_ARGS := --build-arg BASE_IMAGE=$(BASE_IMAGE_NAME)),
 		$(eval BUILD_ARGS :=)
 	)
-	$(ROOT_DIR)/scripts/sandbox.py --dockerfile '$(2)/Dockerfile' -- \
+	$(ROOT_DIR)/scripts/sandbox.py --dockerfile '$(2)/Dockerfile' --build-context '$(RELEASE)' -- \
 		$(CONTAINER_ENGINE) build $(CONTAINER_BUILD_CACHE_ARGS) --tag $(IMAGE_NAME) --file '$(2)/Dockerfile' $(BUILD_ARGS) {}\;
 endef
 
@@ -94,10 +94,10 @@ endef
 # BUILD_DEPENDENT_IMAGES: only build images that were explicitly given as a goal on command line
 # PUSH_IMAGES: allows skipping podman push
 define image
-	$(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...)
+	$(info #*# Image build directory: <$(RELEASE)/$(2)> #(MACHINE-PARSED LINE)#*#...)
 
 	$(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))),
-		$(call build_image,$(1),$(2),$(3))
+		$(call build_image,$(1),$(RELEASE)/$(2),$(3))
 
 		$(if $(PUSH_IMAGES:no=),
 			$(call push_image,$(1))
@@ -269,7 +269,7 @@ endif
 deploy9-%: bin/kubectl bin/yq
 	$(eval TARGET := $(shell echo $* | sed 's/-ubi9-python.*//'))
 	$(eval PYTHON_VERSION := $(shell echo $* | sed 's/.*-python-//'))
-	$(eval NOTEBOOK_DIR := $(subst -,/,$(subst cuda-,,$(TARGET)))/ubi9-python-$(PYTHON_VERSION)/kustomize/base)
+	$(eval NOTEBOOK_DIR := $(RELEASE)/$(subst -,/,$(subst cuda-,,$(TARGET)))/ubi9-python-$(PYTHON_VERSION)/kustomize/base)
 ifndef NOTEBOOK_TAG
 	$(eval NOTEBOOK_TAG := $*-$(IMAGE_TAG))
 endif
@@ -311,7 +311,7 @@ undeploy-c9s-%: bin/kubectl
 .PHONY: test
 test-%: bin/kubectl
 	$(info # Running tests for $* notebook...)
-	@./scripts/test_jupyter_with_papermill.sh $*
+	@./scripts/test_jupyter_with_papermill.sh $(RELEASE) $*
 
 # Validate that runtime image meets minimum criteria
 # This validation is created from subset of https://github.com/elyra-ai/elyra/blob/9c417d2adc9d9f972de5f98fd37f6945e0357ab9/Makefile#L325

ci/cached-builds/gen_gha_matrix_jobs.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import argparse
 import itertools
 import json

scripts/sandbox.py

@@ -19,12 +19,14 @@
 
 class Args(argparse.Namespace):
     dockerfile: pathlib.Path
+    build_context: pathlib.Path
     remaining: list[str]
 
 
 def main() -> int:
     p = argparse.ArgumentParser(allow_abbrev=False)
     p.add_argument("--dockerfile", type=pathlib.Path, required=True)
+    p.add_argument("--build-context", type=pathlib.Path, required=True)
     p.add_argument('remaining', nargs=argparse.REMAINDER)
 
     args = cast(Args, p.parse_args())
@@ -42,12 +44,12 @@ def main() -> int:
         subprocess.check_call([MAKE, "bin/buildinputs"], cwd=ROOT_DIR)
     stdout = subprocess.check_output([ROOT_DIR / "bin/buildinputs", str(args.dockerfile)],
                                      text=True, cwd=ROOT_DIR)
-    prereqs = [pathlib.Path(file) for file in json.loads(stdout)] if stdout != "\n" else []
+    prereqs = [args.build_context / pathlib.Path(file) for file in json.loads(stdout)] if stdout != "\n" else []
     print(f"{prereqs=}")
 
     with tempfile.TemporaryDirectory(delete=True) as tmpdir:
         setup_sandbox(prereqs, pathlib.Path(tmpdir))
-        command = [arg if arg != "{};" else tmpdir for arg in args.remaining[1:]]
+        command = [arg if arg != "{};" else tmpdir / args.build_context for arg in args.remaining[1:]]
         print(f"running {command=}")
         try:
             subprocess.check_call(command)

scripts/test_jupyter_with_papermill.sh

@@ -99,18 +99,20 @@ function _get_accelerator_flavor()
 #   Returns the absolute path of notebook resources in the notebooks/ repo based on the notebook name
 #
 # Arguments:
-#   $1 : Name of the notebook identifier
-#   $2 : [optional] Subdirectory to append to computed absolute path
+#   $1 : Release to test (2024a, 2024n, ...)
+#   $2 : Name of the notebook identifier
+#   $3 : [optional] Subdirectory to append to computed absolute path
 #		- path should NOT start with a leading /
 #
 # Returns:
 #   Absolute path to the jupyter notebook directory for the given notebook test target
 function _get_jupyter_notebook_directory()
 {
-    local notebook_id="${1:-}"
-    local subpath="${2:-}"
+    local release="${1:-}"
+    local notebook_id="${2:-}"
+    local subpath="${3:-}"
 
-    local jupyter_base="${root_repo_directory}/jupyter"
+    local jupyter_base="${root_repo_directory}/${release}/jupyter"
     local directory="${jupyter_base}/${notebook_id}/${os_flavor}-${python_flavor}${subpath:+"/$subpath"}"
 
     printf '%s' "${directory}"
@@ -264,11 +266,12 @@ function _create_test_versions_source_of_truth()
 #   $1 : Name of the notebook identifier
 function _run_test()
 {
-    local notebook_id="${1:-}"
+    local release="${1:-}"
+    local notebook_id="${2:-}"
 
     local test_notebook_file='test_notebook.ipynb'
     local repo_test_directory=
-    repo_test_directory="$(_get_jupyter_notebook_directory "${notebook_id}" "test")"
+    repo_test_directory="$(_get_jupyter_notebook_directory "${release}" "${notebook_id}" "test")"
     local output_file_prefix=
     output_file_prefix=$(tr '/' '-' <<< "${notebook_id}_${os_flavor}")
 
@@ -323,8 +326,10 @@ function _image_derived_from_datascience()
 #	Convenience function that will invoke the minimal and datascience papermill tests against the running notebook workload
 function _test_datascience_notebook()
 {
-    _run_test "${jupyter_minimal_notebook_id}"
-    _run_test "${jupyter_datascience_notebook_id}"
+    local release="${1:-}"
+
+    _run_test "${release}" "${jupyter_minimal_notebook_id}"
+    _run_test "${release}" "${jupyter_datascience_notebook_id}"
 }
 
 # Description:
@@ -335,6 +340,7 @@ function _test_datascience_notebook()
 #		- All relevant tests based on the notebook_id are invoked
 function _handle_test()
 {
+    local release="${1:-}"
     local notebook_id=
 
     # Due to existing logic - cuda accelerator value needs to be treated as empty string
@@ -369,15 +375,16 @@ function _handle_test()
     "${kbin}" exec "${notebook_workload_name}" -- /bin/sh -c "python3 -m pip install papermill"
 
     if _image_derived_from_datascience "${notebook_id}" ; then
-        _test_datascience_notebook
+        _test_datascience_notebook "${release}"
     fi
 
     if [ -n "${notebook_id}" ] && ! [ "${notebook_id}" = "${jupyter_datascience_notebook_id}" ]; then
-        _run_test "${notebook_id}"
+        _run_test "${release}" "${notebook_id}"
     fi
 }
 
-test_target="${1:-}"
+release="${1:-}"
+test_target="${2:-}"
 
 # Hard-coded list of supported "notebook_id" values - based on notebooks/ repo Makefile
 jupyter_minimal_notebook_id='minimal'
@@ -410,5 +417,5 @@ _wait_for_workload "${notebook_name}"
 
 notebook_workload_name=$("${kbin}" get pods -l app="${notebook_name}" -o jsonpath='{.items[0].metadata.name}')
 
-_handle_test
+_handle_test "${release}"