Move to custom idpf kernel

As a workaround for the current idpf issues, try testing the upstream
kernel with the supposed fix

Signed-off-by: Salvatore Daniele <[email protected]>

Author: SalDaniele

extraConfigDpu.py

@@ -1,7 +1,7 @@
 from clustersConfig import ClustersConfig
 import host
 from k8sClient import K8sClient
-from concurrent.futures import Future
+from concurrent.futures import Future, ThreadPoolExecutor
 import os
 import time
 from typing import Optional, List
@@ -13,6 +13,7 @@
 import common
 import re
 from dpuVendor import detect_dpu
+from kernel import ensure_IIC_500_kernel_is_installed
 
 MICROSHIFT_KUBECONFIG = "/root/kubeconfig.microshift"
 
@@ -167,6 +168,22 @@ def ExtraConfigDpuHost(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[s
     lh = host.LocalHost()
     client = K8sClient(cc.kubeconfig)
 
+    # Workaround to ensure patched kernel is deployed on IPU host to avoid issue in IIC-500
+    def helper(h: host.Host) -> Optional[host.Result]:
+        ensure_IIC_500_kernel_is_installed(h)
+        return
+
+    executor = ThreadPoolExecutor(max_workers=len(cc.workers))
+    f = []
+    # Assuming that all workers have a DPU
+    for e in cc.workers:
+        logger.info(f"Calling helper function for node {e.node}")
+        h = host.RemoteHost(e.node)
+        f.append(executor.submit(helper, h))
+
+    for thread in f:
+        logger.info(thread.result())
+
     imgReg = imageRegistry.ensure_local_registry_running(lh, delete_all=False)
     imgReg.ocp_trust(client)
     # Need to trust the registry in OCP / Microshift

ipu.py

@@ -7,6 +7,7 @@
 from clustersConfig import NodeConfig
 from bmc import BmcConfig
 from clusterNode import ClusterNode
+from kernel import ensure_IIC_500_kernel_is_installed
 import host
 from bmc import BMC
 import common
@@ -94,6 +95,7 @@ def start(self, iso_or_image_path: str) -> bool:
         if ipu_bmc.version() != "1.8.0":
             logger.error_and_exit(f"Unexpected version {ipu_bmc.version()}, should be 1.8.0")
         self._boot_iso(iso_or_image_path)
+        self._patch_ipu_host_kernel()
         return True
 
     def has_booted(self) -> bool:
@@ -123,6 +125,12 @@ def helper(node: NodeConfig, iso_address: str) -> str:
                 iso_address = f"http://{lh_ip}:{str(http_server.port)}/{iso_name}"
                 logger.info(helper(node, iso_address))
 
+    def _patch_ipu_host_kernel(self) -> None:
+        # Workaround to ensure patched kernel is deployed on IPU host to avoid issue in IIC-500
+        assert self.config.dpu_host is not None
+        ipu_host = host.RemoteHost(self.config.dpu_host)
+        ensure_IIC_500_kernel_is_installed(ipu_host)
+
     def post_boot(self, *, desired_ip_range: Optional[tuple[str, str]] = None) -> bool:
         # As a WA for https://issues.redhat.com/browse/IIC-527 we need to reload the idpf driver since this seems to fail
         # after an IMC reboot (which occurs during the RHEL installation)

kernel.py

@@ -3,21 +3,22 @@
 import time
 
 KERNEL_RPMS = [
-    "https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot/vol/rhel-9/packages/kernel/5.14.0/427.2.1.el9_4/x86_64/kernel-5.14.0-427.2.1.el9_4.x86_64.rpm",
-    "https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot/vol/rhel-9/packages/kernel/5.14.0/427.2.1.el9_4/x86_64/kernel-core-5.14.0-427.2.1.el9_4.x86_64.rpm",
-    "https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot/vol/rhel-9/packages/kernel/5.14.0/427.2.1.el9_4/x86_64/kernel-modules-5.14.0-427.2.1.el9_4.x86_64.rpm",
-    "https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot/vol/rhel-9/packages/kernel/5.14.0/427.2.1.el9_4/x86_64/kernel-modules-core-5.14.0-427.2.1.el9_4.x86_64.rpm",
-    "https://download-01.beak-001.prod.iad2.dc.redhat.com/brewroot/vol/rhel-9/packages/kernel/5.14.0/427.2.1.el9_4/x86_64/kernel-modules-extra-5.14.0-427.2.1.el9_4.x86_64.rpm",
+    "https://download.devel.redhat.com/brewroot/work/tasks/2286/66882286/kernel-5.14.0-570.idpf.IIC_500.el9_6.x86_64.rpm",
+    "https://download.devel.redhat.com/brewroot/work/tasks/2286/66882286/kernel-core-5.14.0-570.idpf.IIC_500.el9_6.x86_64.rpm",
+    "https://download.devel.redhat.com/brewroot/work/tasks/2286/66882286/kernel-modules-5.14.0-570.idpf.IIC_500.el9_6.x86_64.rpm",
+    "https://download.devel.redhat.com/brewroot/work/tasks/2286/66882286/kernel-modules-core-5.14.0-570.idpf.IIC_500.el9_6.x86_64.rpm",
+    "https://download.devel.redhat.com/brewroot/work/tasks/2286/66882286/kernel-modules-extra-5.14.0-570.idpf.IIC_500.el9_6.x86_64.rpm",
 ]
 
 
-def ensure_rhel_9_4_kernel_is_installed(h: host.Host) -> None:
+def ensure_IIC_500_kernel_is_installed(h: host.Host) -> None:
     h.ssh_connect("core")
     ret = h.run("uname -r")
-    if "el9_4" in ret.out:
+    if "5.14.0-570.idpf.IIC_500.el9_6.x86_64" in ret.out:
+        logger.info("5.14.0-570.idpf.IIC_500.el9_6.x86_64 kernel already installed, skipping")
         return
 
-    logger.info(f"Installing RHEL 9.4 kernel on {h.hostname()}")
+    logger.info(f"Installing 5.14.0-570.idpf.IIC_500.el9_6.x86_64 kernel on {h.hostname()}")
 
     wd = "working_dir"
     h.run(f"rm -rf {wd}")
@@ -45,4 +46,4 @@ def ensure_rhel_9_4_kernel_is_installed(h: host.Host) -> None:
     h.ssh_connect("core")
     ret = h.run("uname -r")
     if "el9_4" not in ret.out:
-        logger.error_and_exit(f"Failed to install rhel 9.4 kernel on host {h.hostname()}")
+        logger.error_and_exit(f"Failed to install 5.14.0-570.idpf.IIC_500.el9_6.x86_64 kernel on host {h.hostname()}")