Dpu: Add check that vsp ds is running

Signed-off-by: Salvatore Daniele <[email protected]>

Author: SalDaniele

extraConfigDpu.py

@@ -211,6 +211,19 @@ def dpu_operator_start(client: K8sClient, repo: Optional[str]) -> None:
     client.oc_run_or_die("wait --for=condition=Ready pod --all -n openshift-dpu-operator --timeout=5m")
 
 
+def ensure_vsp_ds_running(client: K8sClient) -> None:
+    retries = 10
+    for _ in range(retries):
+        desired_pods = int(client.oc_run_or_die("get ds vsp -o jsonpath='{.status.desiredNumberScheduled}'").out)
+        available_pods = int(client.oc_run_or_die("get ds vsp -o jsonpath='{.status.numberAvailable}'").out)
+        if available_pods != desired_pods:
+            logger.info(f"Waiting for VSP ds to scale up. Desired pods: {desired_pods} Available pods: {available_pods}")
+            time.sleep(10)
+        else:
+            break
+    else:
+        logger.error_and_exit("Vsp pods failed to reach ready state")
+
 def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str, Future[Optional[host.Result]]]) -> None:
     [f.result() for (_, f) in futures.items()]
     logger.info("Running post config step to start DPU operator on IPU")
@@ -246,6 +259,7 @@ def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str,
         vendor_plugin.start(vendor_plugin.vsp_image_name(imgReg), client)
     else:
         vendor_plugin.build_push_start(lh, client, imgReg)
+    ensure_vsp_ds_running(client)
 
     git_repo_setup(repo, repo_wipe=False, url=DPU_OPERATOR_REPO)
     if cfg.rebuild_dpu_operators_images:
@@ -293,6 +307,7 @@ def ExtraConfigDpuHost(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[s
     h.ssh_connect("core")
     vendor_plugin = init_vendor_plugin(h, node.kind or "")
     vendor_plugin.build_push_start(lh, client, imgReg)
+    ensure_vsp_ds_running(client)
 
     git_repo_setup(repo, repo_wipe=False, url=DPU_OPERATOR_REPO)
     if cfg.rebuild_dpu_operators_images: