Add manual script to enable p1 networking

Once the fxp-net_linux-networking.pkg package has been loaded, we need
to enable primary networking by designating enp0s1f0d5 as the primary
networking interface on the acc.

By default when the ACC first comes up, this package will not be loaded
(we will use the default package which enables networking on P0/P1 on
the d1/d2 interfaces respectively.

However after the first time we install the vsp, this will become
required.

Signed-off-by: Salvatore Daniele <[email protected]>

Author: SalDaniele

extraConfigDpu.py

@@ -151,11 +151,15 @@ def wait_vsp_ds_running(client: K8sClient) -> None:
 
 def ensure_p4_pod_running(lh: host.Host, acc: host.Host, imgReg: ImageRegistry) -> None:
     lh.run_or_die(f"podman pull --tls-verify=false {P4_IMG}")
-    lh.run_or_die(f"podman tag {P4_IMG} {imgReg.url()}/kubecon-aarch64")
-    lh.run_or_die(f"podman push {imgReg.url()}/kubecon-aarch64")
+    local_img = f"{imgReg.url()}/kubecon-aarch64:latest"
+    lh.run_or_die(f"podman tag {P4_IMG} {local_img}")
+    lh.run_or_die(f"podman push {local_img}")
     uname = acc.run("uname -r").out.strip()
+    # If p4 pod already exists from previous run, kill this first.
+    acc.run(f"podman ps --filter ancestor={local_img} --format '{{{{.ID}}}}' | xargs -r podman kill")
     logger.info("Manually starting P4 container")
-    cmd = f"podman run -d --privileged -v /lib/modules/{uname}:/lib/modules/{uname} -v /opt/p4/p4-cp-nws/var/run:/opt/p4/p4-cp-nws/var/run -v /sys:/sys -p 9559:9559 {imgReg.url()}/kubecon-aarch64"
+    acc.run_or_die("mkdir -p /opt/p4/p4-cp-nws/var/run/openvswitch")  # WA https://issues.redhat.com/browse/IIC-421
+    cmd = f"podman run -d --privileged -v /lib/modules/{uname}:/lib/modules/{uname} -v /opt/p4/p4-cp-nws/var/run:/opt/p4/p4-cp-nws/var/run -v /sys:/sys -p 9559:9559 {local_img}"
     acc.run_or_die(cmd)
     # Occasionally the P4 pod fails to start
     while True:
@@ -165,6 +169,15 @@ def ensure_p4_pod_running(lh: host.Host, acc: host.Host, imgReg: ImageRegistry)
         logger.info("Failed to start p4 container, retrying")
         acc.run_or_die(cmd)
 
+    # WA: There is a race condition if the vsp initializes before the p4 has finished programming the default routes
+    logger.info("Waiting for P4 container to finish initialization")
+    container_id = acc.run_or_die(f"podman ps --filter ancestor={local_img} --format '{{{{.ID}}}}'").out.strip()
+    while True:
+        logs = acc.run_or_die(f"podman logs {container_id} 2>&1").out
+        if "Attempting P4RT communication" in logs:
+            break
+        time.sleep(5)
+
 
 def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str, Future[Optional[host.Result]]]) -> None:
     [f.result() for (_, f) in futures.items()]

ipu.py

@@ -159,7 +159,11 @@ def _prepare_imc(self, server_with_key: str) -> None:
 nohup sh -c '
     while true; do
         sleep 30
-        python /usr/bin/scripts/cfg_acc_apf_x2.py
+        if [ -f /work/scripts/ipu_port1_setup.sh ]; then
+            /work/scripts/ipu_port1_setup.sh
+        else
+            break
+        fi
         ping -c 1 192.168.0.2
         if [ $? -eq 0 ]; then
             break
@@ -184,6 +188,10 @@ def _prepare_imc(self, server_with_key: str) -> None:
         imc.run("cp /etc/imc-redfish-configuration.json /work/redfish/")
         imc.run(f"echo {self.password} | bash /usr/bin/ipu-redfish-generate-password-hash.sh")
 
+        # WA: We need to manually install this file to enable networking with the fxp-net_linux-networking.pkg
+        imc.copy_to("./manifests/dpu/ipu_port1_setup.sh", "/work/scripts/ipu_port1_setup.sh")
+        imc.run("chmod +x /work/scripts/ipu_port1_setup.sh")
+
         imc.run("reboot")
         time.sleep(10)
         imc.wait_ping()

manifests/dpu/ipu_port1_setup.sh

@@ -0,0 +1,6 @@
+# This script needs to be run on reboot each time in order for P1 to be used as the primary interface via enp0s1f0d5
+echo "Configure enp0s1f0d5 as designated primary network interface to Microshift OVS"
+echo "#Add to VSI Group 1 :  enp0s1f0d5 [vsi: 0x0C]"
+devmem 0x20292002a0 64 0x800005000000000c
+devmem 0x2029200388 64 0x1
+devmem 0x20292002a0 64 0xa00005000000000c