Wait for docker build (#6013)

This is a short-term mitigation for
pytorch/pytorch#141885 in which any changes
touching `.ci/docker` would cause all the builds to fail until docker
build workflow finishes building the images.

At the moment, we don't have a good way to tell the build workflow to
wait for the new docker image, so my fix here attempts to inject a delay
when the action is called by `_linux_build`. It will wait up to 90
minutes for the Docker build to finish

### Testing 

pytorch/pytorch#142177

Author: huydhn

.github/actions/calculate-docker-image/action.yml

@@ -87,6 +87,7 @@ runs:
         DOCKER_IMAGE: ${{ steps.calculate-image.outputs.docker-image }}
         DOCKER_TAG: ${{ steps.calculate-image.outputs.docker-tag }}
         DOCKER_REGISTRY: ${{ inputs.docker-registry }}
+        DOCKER_PUSH: ${{ inputs.push }}
       run: |
         set +e
         set -x
@@ -101,10 +102,25 @@ runs:
 
         retry login "${DOCKER_REGISTRY}"
 
-        # Check if image already exists, if it does then skip building it
-        if docker manifest inspect "${DOCKER_IMAGE}"; then
-          exit 0
-        fi
+        START_TIME=$(date +%s)
+        # Wait up to 90 minutes
+        while [[ $(( $(date +%s) - 5400 )) -lt $START_TIME ]]; do
+          # Check if image already exists, if it does then skip building it
+          if docker manifest inspect "${DOCKER_IMAGE}"; then
+            exit 0
+          fi
+
+          # NB: This flag is used by Docker build workflow to push the image to ECR, so we can
+          # use this to differentiate between the Docker build and regular build jobs. For the
+          # latter, it will wait for the Docker images to become available before continuing
+          if [ "${DOCKER_PUSH:-false}" == "true" ]; then
+            # It's a Docker build job, let's build the image
+            break
+          else
+            # It's a regular build job, wait for the image to become available
+            sleep 300
+          fi
+        done
 
         # NB: This part requires a full checkout. Otherwise, the merge base will
         # be empty.  The default action would be to continue rebuild the image