step-registry: setup the commands to allow running on bastian

abhinavdahiya · abhinavdahiya · commit 4ea92d9bdb61 · 2020-06-17T18:04:02.000-07:00
To enable internal clusters current plan is to run the steps from the bastian instances. Since there can be multiple of these running to isolate the binaries and files
this change makes 2 changes,

- WORSPACE, this defaults to /tmp
All the files created as part of the steps, that are not in shared or artifact dir must be created in the WORKSPACE dir. This ensures that on bastians we can provide each test separate dir for the files.
- PATH
WORKSPACE is added to the PATH, this allows the wrapper to copy all the required binaries for that test to the WORKSPACE and the step can continue to use these transparently.
diff --git a/ci-operator/step-registry/gather/extra/gather-extra-commands.sh b/ci-operator/step-registry/gather/extra/gather-extra-commands.sh
@@ -16,7 +16,9 @@ function queue() {
   fi
 }
 
-export PATH=$PATH:/tmp/shared
+export HOME=/tmp
+export WORKSPACE=${WORKSPACE:-/tmp}
+export PATH="${PATH}:${WORKSPACE}"
 
 if test ! -f "${KUBECONFIG}"
 then
@@ -27,11 +29,11 @@ fi
 echo "Gathering artifacts ..."
 mkdir -p ${ARTIFACT_DIR}/pods ${ARTIFACT_DIR}/nodes ${ARTIFACT_DIR}/metrics ${ARTIFACT_DIR}/bootstrap ${ARTIFACT_DIR}/network ${ARTIFACT_DIR}/oc_cmds
 
-oc --insecure-skip-tls-verify --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.metadata.name}{"\n"}{end}' > /tmp/nodes
-oc --insecure-skip-tls-verify --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.spec.providerID}{"\n"}{end}' | sed 's|.*/||' > /tmp/node-provider-IDs
-oc --insecure-skip-tls-verify --request-timeout=5s -n openshift-machine-api get machines -o jsonpath --template '{range .items[*]}{.spec.providerID}{"\n"}{end}' | sed 's|.*/||' >> /tmp/node-provider-IDs
-oc --insecure-skip-tls-verify --request-timeout=5s get pods --all-namespaces --template '{{ range .items }}{{ $name := .metadata.name }}{{ $ns := .metadata.namespace }}{{ range .spec.containers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ range .spec.initContainers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ end }}' > /tmp/containers
-oc --insecure-skip-tls-verify --request-timeout=5s get pods -l openshift.io/component=api --all-namespaces --template '{{ range .items }}-n {{ .metadata.namespace }} {{ .metadata.name }}{{ "\n" }}{{ end }}' > /tmp/pods-api
+oc --insecure-skip-tls-verify --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.metadata.name}{"\n"}{end}' > ${WORKSPACE}/nodes
+oc --insecure-skip-tls-verify --request-timeout=5s get nodes -o jsonpath --template '{range .items[*]}{.spec.providerID}{"\n"}{end}' | sed 's|.*/||' > ${WORKSPACE}/node-provider-IDs
+oc --insecure-skip-tls-verify --request-timeout=5s -n openshift-machine-api get machines -o jsonpath --template '{range .items[*]}{.spec.providerID}{"\n"}{end}' | sed 's|.*/||' >> ${WORKSPACE}/node-provider-IDs
+oc --insecure-skip-tls-verify --request-timeout=5s get pods --all-namespaces --template '{{ range .items }}{{ $name := .metadata.name }}{{ $ns := .metadata.namespace }}{{ range .spec.containers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ range .spec.initContainers }}-n {{ $ns }} {{ $name }} -c {{ .name }}{{ "\n" }}{{ end }}{{ end }}' > ${WORKSPACE}/containers
+oc --insecure-skip-tls-verify --request-timeout=5s get pods -l openshift.io/component=api --all-namespaces --template '{{ range .items }}-n {{ .metadata.namespace }} {{ .metadata.name }}{{ "\n" }}{{ end }}' > ${WORKSPACE}/pods-api
 
 queue ${ARTIFACT_DIR}/config-resources.json oc --insecure-skip-tls-verify --request-timeout=5s get apiserver.config.openshift.io authentication.config.openshift.io build.config.openshift.io console.config.openshift.io dns.config.openshift.io featuregate.config.openshift.io image.config.openshift.io infrastructure.config.openshift.io ingress.config.openshift.io network.config.openshift.io oauth.config.openshift.io project.config.openshift.io scheduler.config.openshift.io -o json
 queue ${ARTIFACT_DIR}/apiservices.json oc --insecure-skip-tls-verify --request-timeout=5s get apiservices -o json
@@ -94,30 +96,30 @@ FILTER=gzip queue ${ARTIFACT_DIR}/openapi.json.gz oc --insecure-skip-tls-verify
 while IFS= read -r i; do
   mkdir -p ${ARTIFACT_DIR}/nodes/$i
   queue ${ARTIFACT_DIR}/nodes/$i/heap oc --insecure-skip-tls-verify get --request-timeout=20s --raw /api/v1/nodes/$i/proxy/debug/pprof/heap
-done < /tmp/nodes
+done < ${WORKSPACE}/nodes
 
 FILTER=gzip queue ${ARTIFACT_DIR}/nodes/masters-journal.gz oc --insecure-skip-tls-verify adm node-logs --role=master --unify=false
 FILTER=gzip queue ${ARTIFACT_DIR}/nodes/masters-journal-previous.gz oc --insecure-skip-tls-verify adm node-logs --boot=-1 --role=master --unify=false
 FILTER=gzip queue ${ARTIFACT_DIR}/nodes/workers-journal.gz oc --insecure-skip-tls-verify adm node-logs --role=worker --unify=false
 FILTER=gzip queue ${ARTIFACT_DIR}/nodes/workers-journal-previous.gz oc --insecure-skip-tls-verify adm node-logs --boot=-1 --role=worker --unify=false
 
 # Snapshot iptables-save on each node for debugging possible kube-proxy issues
-oc --insecure-skip-tls-verify get --request-timeout=20s -n openshift-sdn -l app=sdn pods --template '{{ range .items }}{{ .metadata.name }}{{ "\n" }}{{ end }}' > /tmp/sdn-pods
+oc --insecure-skip-tls-verify get --request-timeout=20s -n openshift-sdn -l app=sdn pods --template '{{ range .items }}{{ .metadata.name }}{{ "\n" }}{{ end }}' > ${WORKSPACE}/sdn-pods
 while IFS= read -r i; do
   queue ${ARTIFACT_DIR}/network/iptables-save-$i oc --insecure-skip-tls-verify rsh --timeout=20 -n openshift-sdn -c sdn $i iptables-save -c
-done < /tmp/sdn-pods
+done < ${WORKSPACE}/sdn-pods
 
 while IFS= read -r i; do
   file="$( echo "$i" | cut -d ' ' -f 3 | tr -s ' ' '_' )"
   queue ${ARTIFACT_DIR}/metrics/${file}-heap oc --insecure-skip-tls-verify exec $i -- /bin/bash -c 'oc --insecure-skip-tls-verify get --raw /debug/pprof/heap --server "https://$( hostname ):8443" --config /etc/origin/master/admin.kubeconfig'
   queue ${ARTIFACT_DIR}/metrics/${file}-controllers-heap oc --insecure-skip-tls-verify exec $i -- /bin/bash -c 'oc --insecure-skip-tls-verify get --raw /debug/pprof/heap --server "https://$( hostname ):8444" --config /etc/origin/master/admin.kubeconfig'
-done < /tmp/pods-api
+done < ${WORKSPACE}/pods-api
 
 while IFS= read -r i; do
   file="$( echo "$i" | cut -d ' ' -f 2,3,5 | tr -s ' ' '_' )"
   FILTER=gzip queue ${ARTIFACT_DIR}/pods/${file}.log.gz oc --insecure-skip-tls-verify logs --request-timeout=20s $i
   FILTER=gzip queue ${ARTIFACT_DIR}/pods/${file}_previous.log.gz oc --insecure-skip-tls-verify logs --request-timeout=20s -p $i
-done < /tmp/containers
+done < ${WORKSPACE}/containers
 
 echo "Snapshotting prometheus (may take 15s) ..."
 queue ${ARTIFACT_DIR}/metrics/prometheus.tar.gz oc --insecure-skip-tls-verify exec -n openshift-monitoring prometheus-k8s-0 -- tar cvzf - -C /prometheus .
diff --git a/ci-operator/step-registry/gather/must-gather/gather-must-gather-commands.sh b/ci-operator/step-registry/gather/must-gather/gather-must-gather-commands.sh
@@ -4,6 +4,10 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
+export HOME=/tmp
+export WORKSPACE=${WORKSPACE:-/tmp}
+export PATH="${PATH}:${WORKSPACE}"
+
 if test ! -f "${KUBECONFIG}"
 then
 	echo "No kubeconfig, so no point in calling must-gather."
diff --git a/ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh b/ci-operator/step-registry/ipi/deprovision/deprovision/ipi-deprovision-deprovision-commands.sh
@@ -9,21 +9,26 @@ trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wa
 export AWS_SHARED_CREDENTIALS_FILE=$CLUSTER_PROFILE_DIR/.awscred
 export AZURE_AUTH_LOCATION=$CLUSTER_PROFILE_DIR/osServicePrincipal.json
 export GOOGLE_CLOUD_KEYFILE_JSON=$CLUSTER_PROFILE_DIR/gce.json
+export HOME=/tmp
+export WORKSPACE=${WORKSPACE:-/tmp}
+export PATH="${PATH}:${WORKSPACE}"
 
 echo "Deprovisioning cluster ..."
 if [[ ! -s "${SHARED_DIR}/metadata.json" ]]; then
   echo "Skipping: ${SHARED_DIR}/metadata.json not found."
   exit
 fi
 
-cp -ar "${SHARED_DIR}" /tmp/installer
-openshift-install --dir /tmp/installer destroy cluster &
+dir=${WORKSPACE}/installer
+mkdir -p "${dir}/"
+cp -ar "${SHARED_DIR}"/* "${dir}/"
+openshift-install --dir "${dir}" destroy cluster &
 
 set +e
 wait "$!"
 ret="$?"
 set -e
 
-cp /tmp/installer/.openshift_install.log "${ARTIFACT_DIR}"
+cp "${dir}"/.openshift_install.log "${ARTIFACT_DIR}"
 
 exit "$ret"
diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
@@ -16,6 +16,8 @@ export PULL_SECRET_PATH=${CLUSTER_PROFILE_DIR}/pull-secret
 export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE=${RELEASE_IMAGE_LATEST}
 export OPENSHIFT_INSTALL_INVOKER=openshift-internal-ci/${JOB_NAME}/${BUILD_ID}
 export HOME=/tmp
+export WORKSPACE=${WORKSPACE:-/tmp}
+export PATH="${PATH}:${WORKSPACE}"
 
 case "${CLUSTER_TYPE}" in
 aws) export AWS_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/.awscred;;
@@ -25,8 +27,8 @@ vsphere) ;;
 *) echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'"
 esac
 
-dir=/tmp/installer
-mkdir "${dir}/"
+dir=${WORKSPACE}/installer
+mkdir -p "${dir}/"
 cp "${SHARED_DIR}/install-config.yaml" "${dir}/"
 
 # move private key to ~/.ssh/ so that installer can use it to gather logs on
diff --git a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh
@@ -7,13 +7,12 @@ set -o pipefail
 export AWS_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/.awscred
 export AZURE_AUTH_LOCATION=${CLUSTER_PROFILE_DIR}/osServicePrincipal.json
 export GCP_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/gce.json
-export HOME=/tmp/home
-export PATH=/usr/libexec/origin:$PATH
+export HOME=/tmp
+export WORKSPACE=${WORKSPACE:-/tmp}
+export PATH=/usr/libexec/origin:${WORKSPACE}:$PATH
 
 trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
 
-mkdir -p "${HOME}"
-
 # if the cluster profile included an insights secret, install it to the cluster to
 # report support data from the support-operator
 if [[ -f "${CLUSTER_PROFILE_DIR}/insights-live.yaml" ]]; then
@@ -47,16 +46,16 @@ vsphere) export TEST_PROVIDER=vsphere;;
 *) echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'"; exit 1;;
 esac
 
-mkdir -p /tmp/output
-cd /tmp/output
+mkdir -p ${WORKSPACE}/output
+cd ${WORKSPACE}/output
 
 if [[ "${CLUSTER_TYPE}" == gcp ]]; then
-    pushd /tmp
+    pushd ${WORKSPACE}
     curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-256.0.0-linux-x86_64.tar.gz
     tar -xzf google-cloud-sdk-256.0.0-linux-x86_64.tar.gz
-    export PATH=$PATH:/tmp/google-cloud-sdk/bin
+    export PATH=$PATH:${WORKSPACE}/google-cloud-sdk/bin
     mkdir gcloudconfig
-    export CLOUDSDK_CONFIG=/tmp/gcloudconfig
+    export CLOUDSDK_CONFIG=${WORKSPACE}/gcloudconfig
     gcloud auth activate-service-account --key-file="${GCP_SHARED_CREDENTIALS_FILE}"
     gcloud config set project openshift-gce-devel-ci
     popd
@@ -69,5 +68,5 @@ fi
 
 openshift-tests run "${test_suite}" \
     --provider "${TEST_PROVIDER}" \
-    -o /tmp/artifacts/e2e.log \
-    --junit-dir /tmp/artifacts/junit
+    -o ${ARTIFACT_DIR}/e2e.log \
+    --junit-dir ${ARTIFACT_DIR}/junit
