Add workaround for flaky acc first boot

Sometimes after the first boot, things just don't work.

If we don't run the connectivity script constantly, we run the risk of
not running it in the window we need to.

If we do run the connectivity script constantly, we seem to be able to
break the node policy rules, such that there is no connectivity on any
port no matter what we do.

Instead, add a more moderate script to run ipu_port1_setup.sh, and just
reboot the IMC if things have taken too long. After the first reboot,
behavior of the ACC seems much more predictable

Signed-off-by: Salvatore Daniele <[email protected]>

Author: SalDaniele

ipu.py

@@ -50,6 +50,25 @@ def __init__(self, config: NodeConfig, external_port: str, network_api_port: str
         self.external_port = external_port
         self.network_api_port = network_api_port
         self.config = config
+    
+    def _wait_for_acc_with_retry(self, acc: host.Host):
+        # Typically if the acc booted properly it will take < 20 minutes to come up (including the 10 min sleep we do during boot)
+        timeout = 1500
+        while True:
+            if acc.ping():
+                logger.info("ACC responded to ping, connecting")
+                break
+            time.sleep(20)
+            timeout -= 20
+            if timeout <= 0:
+                logger.info("ACC has not responded in a reasonable amount of time, rebooting IMC")
+                imc = host.RemoteHost(self.config.bmc)
+                imc.ssh_connect(self.config.bmc_user, self.config.bmc_password)
+                imc.run("reboot")
+                timeout = 240
+        
+        acc.ssh_connect("root", "redhat")
+        logger.info(acc.run("uname -a"))
 
     def _boot_iso(self, iso: str) -> None:
         assert self.config.ip
@@ -58,8 +77,8 @@ def _boot_iso(self, iso: str) -> None:
         self._redfish_boot_ipu(self.external_port, self.config, iso)
         # wait on install + reboot to complete
         acc = host.RemoteHost(self.config.ip)
-        acc.ssh_connect("root", "redhat")
-        logger.info(acc.run("uname -a"))
+        # WA since we can't reliably expect the acc to get a dhcp lease due to https://issues.redhat.com/browse/IIC-427
+        self._wait_for_acc_with_retry(acc)
         # configure_iso_network_port(self.network_api_port, self.config.ip)
 
     def start(self, iso_or_image_path: str, executor: ThreadPoolExecutor) -> None:
@@ -149,11 +168,12 @@ def _prepare_imc(self, server_with_key: str) -> None:
 nohup sh -c '
     while true; do
         if [ -f /work/scripts/ipu_port1_setup.sh ]; then
-            ping -c 1 -W 2 192.168.0.2
-            if [ $? -eq 0 ]; then
+            while [ $count -lt 20 ]; do
                 /work/scripts/ipu_port1_setup.sh
-                sleep 1
-            fi
+                count=$((count + 1))
+                sleep $count
+            done
+            break
         else
             break
         fi