ci-operator/templates/e2e-metal: fix csr approval and bootstrap log gathering

Carries changes from [1] and [2]

[1]: openshift#3573
[2]: openshift#3475

Author: abhinavdahiya

ci-operator/templates/openshift/installer/cluster-launch-installer-metal-e2e.yaml

@@ -299,16 +299,23 @@ objects:
         echo "Removing bootstrap host from control plane api pool"
         (cd /tmp/artifacts/terraform && terraform apply -auto-approve=true -var=bootstrap_dns="false")
 
-        echo "Approving any pending CSR requests"
-        oc get csr -ojson | jq -r '.items[] | select(.status == {} ) | .metadata.name' | xargs oc adm certificate approve
+        function approve_csrs() {
+          while [[ ! -f /tmp/exit ]] && [[ ! -f /tmp/setup-success ]]; do
+            oc get csr -ojson | jq -r '.items[] | select(.status == {} ) | .metadata.name' | xargs --no-run-if-empty oc adm certificate approve
+            sleep 15
+          done
+        }
 
-        until oc get configs.imageregistry.operator.openshift.io cluster >/dev/null
-        do
-          echo "waiting for configs.imageregistry.operator.openshift.io cluster to exit failed. Retrying in 1 minute..."
-          sleep 1m
-        done
+        function update_image_registry() {
+          while true; do
+            sleep 10;
+            oc get configs.imageregistry.operator.openshift.io/cluster > /dev/null && break
+          done
+          oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"storage":{"emptyDir":{}}}}'
+        }
 
-        oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"storage":{"emptyDir":{}}}}'
+        approve_csrs &
+        update_image_registry &
 
         echo "Completing UPI setup"
         openshift-install --dir=/tmp/artifacts/installer wait-for install-complete &
@@ -366,14 +373,9 @@ objects:
           mkdir -p /tmp/artifacts/pods /tmp/artifacts/nodes /tmp/artifacts/metrics /tmp/artifacts/bootstrap /tmp/artifacts/network
 
 
-          if [ -f /tmp/artifacts/installer/terraform.tfstate ]
+          if [ -f /tmp/artifacts/terraform/terraform.tfstate ]
           then
-              # we don't have jq, so the python equivalent of
-              # jq '.modules[].resources."aws_instance.bootstrap".primary.attributes."public_ip" | select(.)'
-              bootstrap_ip=$(python -c \
-                  'import sys, json; d=reduce(lambda x,y: dict(x.items() + y.items()), map(lambda x: x["resources"], json.load(sys.stdin)["modules"])); k="aws_instance.bootstrap"; print d[k]["primary"]["attributes"]["public_ip"] if k in d else ""' \
-                  < /tmp/artifacts/installer/terraform.tfstate
-              )
+              bootstrap_ip=$(terraform output -json | jq -r '.bootstrap_ip.value')
 
               if [ -n "${bootstrap_ip}" ]
               then
@@ -388,6 +390,15 @@ objects:
                         --key /tmp/artifacts/installer/tls/journal-gatewayd.key \
                         --url "https://${bootstrap_ip}:19531/entries?_SYSTEMD_UNIT=${service}.service"
                 done
+                if ! whoami &> /dev/null; then
+                  if [ -w /etc/passwd ]; then
+                    echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd
+                  fi
+                fi
+                eval $(ssh-agent)
+                ssh-add /etc/openshift-installer/ssh-privatekey
+                ssh -A -o PreferredAuthentications=publickey -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null core@${bootstrap_ip} /bin/bash -x /usr/local/bin/installer-gather.sh
+                scp -o PreferredAuthentications=publickey -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null core@${bootstrap_ip}:log-bundle.tar.gz /tmp/artifacts/installer/bootstrap-logs.tar.gz
               fi
           else
               echo "No terraform statefile found. Skipping collection of bootstrap logs."