1+ #! /bin/bash
2+ # shellcheck disable=SC2153
3+ # ARTIFACT_DIR is provided by the CI environment (Prow/OpenShift CI)
4+
5+ set -o nounset
6+ set -o errexit
7+ set -o pipefail
8+ set -x
9+
10+ # Fix user IDs in a container
11+ ~ /fix_uid.sh
12+
13+ # # SLCM VARs
14+ DCI_REMOTE_CI=" $( cat /var/run/project-02/slcm-container/DCI_REMOTE_CI) "
15+ CLOUD_RAN_PARTNER_REPO=" $( cat /var/run/project-02/slcm-container/cloud_ran_partner_repo) "
16+ CLOUD_RAN_PARTNER_REPO_VERSION=" $( cat /var/run/project-02/slcm-container/cloud_ran_partner_repo_version) "
17+ REMOTE_USER=" $( cat /var/run/project-02/slcm-container/remote_user) "
18+ CLUSTER_CONFIGS_DIR=" $( cat /var/run/project-02/slcm-container/cluster_configs_dir) "
19+ HUB_KUBECONFIG_PATH=" $( cat /var/run/project-02/slcm-container/hub_kubeconfig_path) "
20+ PODMAN_AUTH_PATH=" $( cat /var/run/project-02/slcm-container/PODMAN_AUTH_PATH) "
21+ VAULT_PASSWORD=$( cat /var/run/project-02/slcm-container/VAULT_PASSWORD)
22+ ECO_GOTESTS_CONTAINER=" $( cat /var/run/project-02/slcm-container/ECO_GOTESTS_CONTAINER) "
23+ ECO_VALIDATION_CONTAINER=" $( cat /var/run/project-02/slcm-container/ECO_VALIDATION_CONTAINER) "
24+ TB1SLCM1=" $( cat /var/run/project-02/slcm-container/tb1slcm1) "
25+ TB2SLCM1=" $( cat /var/run/project-02/slcm-container/tb2slcm1) "
26+ SKIP_DCI=" $( cat /var/run/project-02/slcm-container/SKIP_DCI) "
27+ STAMP=" $( cat /var/run/project-02/slcm-container/STAMP) "
28+ LATENCY_DURATION=" $( cat /var/run/project-02/slcm-container/LATENCY_DURATION) "
29+ OCP_VERSION=" $( cat /var/run/project-02/slcm-container/OCP_VERSION) "
30+ SITE_NAME=" $( cat /var/run/project-02/slcm-container/SITE_NAME) "
31+ DCI_PIPELINE_FILES=" $( cat /var/run/project-02/slcm-container/DCI_PIPELINE_FILES) "
32+ EDU_PTP=" $( cat /var/run/project-02/slcm-container/EDU_PTP) "
33+ COPY_TO_PROW=" $( cat /var/run/project-02/slcm-container/COPY_TO_PROW) "
34+
35+ SLCM_VAULT=/var/run/project-02/vault-data/vault_data
36+ cp $SLCM_VAULT playbooks/run_slcm_vault
37+ chmod 0600 playbooks/run_slcm_vault
38+
39+ # # VPN
40+ VPN_URL=" $( cat /var/run/bastion1/vpn-url) "
41+ VPN_USERNAME=" $( cat /var/run/bastion1/vpn-username) "
42+ VPN_PASSWORD=$( cat /var/run/bastion1/vpn-password)
43+
44+ # # SSH
45+ SSH_KEY_PATH=/var/run/telcov10n/ansible_ssh_private_key
46+ SSH_KEY=~ /key
47+ IFNAME=tun10
48+
49+ cp $SSH_KEY_PATH $SSH_KEY
50+ chmod 600 $SSH_KEY
51+
52+ SSHOPTS=(
53+ -o ' ConnectTimeout=5'
54+ -o ' StrictHostKeyChecking=no'
55+ -o ' UserKnownHostsFile=/dev/null'
56+ -o ' ServerAliveInterval=90'
57+ -o LogLevel=ERROR
58+ -i " ${SSH_KEY} "
59+ )
60+
61+ # # JUMP SERVER
62+ JUMP_SERVER_ADDRESS=" $( cat /var/run/bastion1/jump-server) "
63+ JUMP_SERVER_USER=" $( cat /var/run/telcov10n/ansible_user) "
64+
65+ # # COPY JUNIT FILES FUNCTION
66+ copy_junit_files () {
67+
68+ ansible-playbook -i slcm_inventory.yml playbooks/run_slcm_container.yml -e @slcm_vars.yml --tags setup_vpn --skip-tags kill_vpn | tee ${ARTIFACT_DIR} /ansible_setup_vpn.log
69+
70+ local REMOTE_JUNIT_DIR=" /tmp/prow_pipeline_${BUILD_ID} "
71+ local JUMP_TEMP_DIR=" /tmp/prow_junit_${BUILD_ID} "
72+ local success=0
73+
74+ echo " === Two-Step File Copy Process ==="
75+
76+ # Test basic connectivity first
77+ echo " Testing connectivity to jump server..."
78+ if ! ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} " " echo 'Jump server reachable'" ; then
79+ echo " ERROR: Cannot reach jump server"
80+ return 1
81+ fi
82+
83+ echo " Testing connectivity from jump server to target..."
84+ if ! ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
85+ " ssh -i ${SSH_KEY} ${SSHOPTS[*]} -o ConnectTimeout=10 ${REMOTE_USER} @${TB2SLCM1} 'echo Connection successful'" ; then
86+ echo " ERROR: Cannot reach target server from jump server"
87+ return 1
88+ fi
89+
90+ # Create temporary directory on jump server
91+ echo " Creating temporary directory on jump server..."
92+ ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
93+ " mkdir -p ${JUMP_TEMP_DIR} "
94+
95+ # Step 1: Target -> Jump Server
96+ echo " === Step 1: Target Server -> Jump Server ==="
97+
98+ # Check and list files on target server
99+ echo " Checking for XML files on target server..."
100+ target_files=$( ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
101+ " ssh -i ${SSH_KEY} ${SSHOPTS[*]} ${REMOTE_USER} @${TB2SLCM1} 'ls -la ${REMOTE_JUNIT_DIR} /*.xml 2>/dev/null'" || echo " " )
102+
103+ file_count=$( echo " $target_files " | grep -c " \.xml$" || echo " 0" )
104+
105+ if [[ " $file_count " -gt 0 ]]; then
106+ echo " Found $file_count XML files on target server:"
107+ echo " $target_files "
108+ ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
109+ " scp -i ${SSH_KEY} ${SSHOPTS[*]} ${REMOTE_USER} @${TB2SLCM1} :${REMOTE_JUNIT_DIR} /*.xml ${JUMP_TEMP_DIR} /"
110+
111+ # Verify copy to jump server and list files
112+ jump_files=$( ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
113+ " ls -la ${JUMP_TEMP_DIR} /*.xml 2>/dev/null" || echo " " )
114+
115+ jump_file_count=$( echo " $jump_files " | grep -c " \.xml$" || echo " 0" )
116+
117+ if [[ " $jump_file_count " -gt 0 ]]; then
118+ echo " Successfully copied $jump_file_count files to jump server:"
119+ echo " $jump_files "
120+ success=1
121+ else
122+ echo " WARNING: No files found on jump server after copy"
123+ fi
124+ else
125+ echo " No XML files found on target server"
126+ fi
127+
128+ # Step 2: Jump Server -> Prow Server (only if files exist)
129+ if [[ $success -eq 1 ]]; then
130+ echo " === Step 2: Jump Server -> Prow Server ==="
131+
132+ scp -i " ${SSH_KEY} " " ${SSHOPTS[@]} "
133+ " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} :${JUMP_TEMP_DIR} /*.xml"
134+ " ${ARTIFACT_DIR} /"
135+
136+ # Verify final copy
137+ local_file_count=$( ls -1 " ${ARTIFACT_DIR} " * .xml 2> /dev/null | wc -l || echo " 0" )
138+ if [[ " $local_file_count " -gt 0 ]]; then
139+ echo " Successfully copied $local_file_count files to prow server"
140+ echo " Files in artifact directory:"
141+ ls -la " ${ARTIFACT_DIR} " * .xml
142+
143+ # Cleanup remote directory on target server after successful copy
144+ echo " === Cleanup: Removing remote directory on target server ==="
145+ ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
146+ " ssh -i ${SSH_KEY} ${SSHOPTS[*]} ${REMOTE_USER} @${TB2SLCM1} 'rm -rf ${REMOTE_JUNIT_DIR} '" && \
147+ echo " Successfully removed ${REMOTE_JUNIT_DIR} from target server" || \
148+ echo " WARNING: Could not remove ${REMOTE_JUNIT_DIR} from target server"
149+ else
150+ echo " WARNING: No files found in artifacts directory after copy"
151+ echo " Keeping remote directory ${REMOTE_JUNIT_DIR} on target server due to copy failure"
152+ fi
153+ fi
154+
155+ # Cleanup
156+ echo " Cleaning up temporary directory on jump server..."
157+ ssh -i " ${SSH_KEY} " " ${SSHOPTS[@]} " " ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
158+ " rm -rf ${JUMP_TEMP_DIR} " || echo " Cleanup warning: Could not remove temp directory"
159+
160+ ansible-playbook -i slcm_inventory.yml playbooks/run_slcm_container.yml -e @slcm_vars.yml --tags kill_vpn | tee ${ARTIFACT_DIR} /ansible_kill_vpn.log
161+ }
162+
163+ # # CHECK TEST RESULTS FILES FUNCTION
164+ check_test_failures () {
165+ shopt -s globstar nullglob
166+ local artifact_dir=" ${ARTIFACT_DIR:- .} "
167+ local failed=0
168+ local found_files=0
169+
170+ for file in " ${artifact_dir} " ** /* .xml; do
171+ found_files=1
172+ if grep -q ' <testsuite[^>]*failures="[^0"]' " $file " ; then
173+ echo " Test failure found in: $file "
174+ failed=1
175+ elif grep -q ' <failure' " $file " ; then
176+ echo " Failure element found in: $file "
177+ failed=1
178+ fi
179+ done
180+
181+ if [[ $found_files -eq 0 ]]; then
182+ echo " No test result XML files found in ${artifact_dir} "
183+ return 1
184+ fi
185+
186+ return $failed
187+ }
188+
189+ # # INVENTORY
190+ cat << END_INVENTORY > slcm_inventory.yml
191+ ---
192+ all:
193+ hosts:
194+ localhost:
195+ ansible_connection: local
196+ children:
197+ jumphost:
198+ hosts:
199+ jump_host:
200+ ansible_host: "${JUMP_SERVER_ADDRESS} "
201+ ansible_user: "${JUMP_SERVER_USER} "
202+ ansible_ssh_common_args: "${SSHOPTS[@]} "
203+ vpn_username: "${VPN_USERNAME} "
204+ vpn_password: "${VPN_PASSWORD} "
205+ vpn_url: "${VPN_URL} "
206+ tun_name: "${IFNAME} "
207+ targets:
208+ hosts:
209+ "${TB2SLCM1} ":
210+ ansible_host: "${TB2SLCM1} "
211+ ansible_ssh_common_args: >-
212+ -i "${SSH_KEY} " ${SSHOPTS[*]}
213+ -o ProxyCommand="ssh -W %h:%p ${SSHOPTS[*]} -i "${SSH_KEY} " -q ${JUMP_SERVER_USER} @${JUMP_SERVER_ADDRESS} "
214+ vars:
215+ artifacts_dir: "${ARTIFACT_DIR} "
216+ remote_user: "${REMOTE_USER} "
217+ END_INVENTORY
218+
219+ # # VARs
220+ cat << END_VARS > slcm_vars.yml
221+ ---
222+ DCI_REMOTE_CI: "${DCI_REMOTE_CI} "
223+ cloud_ran_partner_repo: "${CLOUD_RAN_PARTNER_REPO} "
224+ cloud_ran_partner_repo_version: "${CLOUD_RAN_PARTNER_REPO_VERSION} "
225+ remote_user: "${REMOTE_USER} "
226+ cluster_configs_dir: "${CLUSTER_CONFIGS_DIR} "
227+ hub_kubeconfig_path: "${HUB_KUBECONFIG_PATH} "
228+ PODMAN_AUTH_PATH: "${PODMAN_AUTH_PATH} "
229+ VAULT_PASSWORD: "${VAULT_PASSWORD} "
230+ ECO_GOTESTS_CONTAINER: "${ECO_GOTESTS_CONTAINER} "
231+ ECO_VALIDATION_CONTAINER: "${ECO_VALIDATION_CONTAINER} "
232+ PROW_PIPELINE_ID: "${BUILD_ID} "
233+ SKIP_DCI: "${SKIP_DCI} "
234+ STAMP: "${STAMP} "
235+ LATENCY_DURATION: "${LATENCY_DURATION} "
236+ OCP_VERSION: "${OCP_VERSION} "
237+ SITE_NAME: "${SITE_NAME} "
238+ DCI_PIPELINE_FILES: "${DCI_PIPELINE_FILES} "
239+ EDU_PTP: "${EDU_PTP} "
240+ COPY_TO_PROW: "${COPY_TO_PROW} "
241+ PROW_JUNIT_TEMP_DIR: "/tmp/prow_pipeline_${BUILD_ID} "
242+ infra_hosts:
243+ tb1slcm1: "${TB1SLCM1} "
244+ tb2slcm1: "${TB2SLCM1} "
245+ END_VARS
246+
247+ ansible-galaxy collection install ansible.posix
248+
249+ set +e
250+ ansible-playbook -i slcm_inventory.yml playbooks/run_slcm_container.yml -e @slcm_vars.yml | tee ${ARTIFACT_DIR} /ansible_slcm_inventory.log
251+ ANSIBLE_EXIT_CODE=$?
252+ set -e
253+
254+ if [[ $ANSIBLE_EXIT_CODE -ne 0 ]]; then
255+ echo " SLCM playbook failed with exit code $ANSIBLE_EXIT_CODE , but continuing to evaluate test results..."
256+ fi
257+
258+ copy_junit_files
259+ check_test_failures
260+
261+ if ! check_test_failures; then
262+ echo " Failed: either test failures detected or no test results found!"
263+ exit 1
264+ else
265+ echo " All tests passed!"
266+ fi
0 commit comments