add timeout to init container command

Author: pmossman

airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java

@@ -119,6 +119,11 @@ public class KubePodProcess extends Process implements KubePod {
   private static final int KILLED_EXIT_CODE = 143;
   private static final int STDIN_REMOTE_PORT = 9001;
 
+  // init container should fail if SUCCESS_FILE_NAME is not copied within INIT_RETRY_TIMEOUT_MINUTES
+  private static final double INIT_SLEEP_PERIOD_SECONDS = 0.1;
+  private static final Duration INIT_RETRY_TIMEOUT_MINUTES = Duration.ofMinutes(5);
+  private static final double INIT_RETRY_MAX_ITERATIONS = INIT_RETRY_TIMEOUT_MINUTES.toSeconds() / INIT_SLEEP_PERIOD_SECONDS;
+
   private final KubernetesClient fabricClient;
   private final Pod podDefinition;
   // Necessary since it is not possible to retrieve the pod's actual exit code upon termination. This
@@ -159,7 +164,16 @@ private static Container getInit(final boolean usesStdin,
       initEntrypointStr = String.format("mkfifo %s && ", STDIN_PIPE_FILE) + initEntrypointStr;
     }
 
-    initEntrypointStr = initEntrypointStr + String.format(" && until [ -f %s ]; do sleep 0.1; done;", SUCCESS_FILE_NAME);
+    initEntrypointStr = initEntrypointStr +
+        String.format(
+            " ITERATION=0 && MAX_ITERATION=%s && "
+                + "until [ -f %s -o $ITERATION -ge $MAX_ITERATION ]; "
+                + "do ((ITERATION=ITERATION+1)) && sleep %s; done; "
+                + "if [ -f %s ]; then exit 0; else exit 1; fi",
+            INIT_RETRY_MAX_ITERATIONS,
+            SUCCESS_FILE_NAME,
+            INIT_SLEEP_PERIOD_SECONDS,
+            SUCCESS_FILE_NAME);
 
     return new ContainerBuilder()
         .withName(INIT_CONTAINER_NAME)