Merge pull request #118 from tomeon/test-suite-improvements

Test suite improvements

Author: tomeon

run-tests

@@ -14,18 +14,36 @@
 # GNU General Public License for more details.
 
 # Set colour escape sequences
-RED='\033[0;31m'
-YELLOW='\033[1;33m'
-ORANGE='\033[0;33m'
-GREEN='\033[0;32m'
-DARK='\033[1;30m'
-RESET='\033[0m'
+
+PS4='+ ${BASH_SOURCE:-$0}@${LINENO:-0}${FUNCNAME:+#${FUNCNAME}()}: '
+
+if [[ -n "${NO_COLOR:-}" ]] || ! colors="$(tput colors 2>/dev/null)" || (( "${colors:-0}" < 16 )); then
+  RED=''
+  YELLOW=''
+  ORANGE=''
+  GREEN=''
+  DARK=''
+  RESET=''
+else
+  RED='\033[0;31m'
+  YELLOW='\033[1;33m'
+  ORANGE='\033[0;33m'
+  GREEN='\033[0;32m'
+  DARK='\033[1;30m'
+  RESET='\033[0m'
+fi
+
 # Set Pass/Fail signatures
 PASS="✓"
 FAIL="✗"
+
 # Counters
 COUNT_PASS=0
 COUNT_FAIL=0
+
+# Names of files in which test cases failed
+declare -A FAILED
+
 # Flag for determining whether a test script called the `runtest' function
 RUNTEST_CALLED=0
 
@@ -284,6 +302,9 @@ function _pass {
 
 function _fail {
   COUNT_FAIL=$((COUNT_FAIL+1))
+  if [[ -v TEST ]]; then
+    FAILED["$TEST"]=1
+  fi
   ( >&2 echo -e "  ${RED}${FAIL} ${@}${RESET}" )
 }
 
@@ -342,16 +363,28 @@ evaltest() {
   TEST_BUSCTL_STATUS_RC=""
   # Keep this random, as we will never know the ifindex up-front
   ip_ifindex=$((RANDOM%=64))
+  # Same for the device
+  dev="tun${RANDOM}"
 
   # Clear foreign_option_*
-  for foreign_option_varname in "${!foreign_option_@}"; do
-    declare "${foreign_option_varname}="
-  done
+  unset "${!foreign_option_@}"
 
   # Import the test configuration
-  source "${TEST}"
+  source "${TEST}" || return
+
+  if (( RUNTEST_CALLED > 0 )); then
+    return
+  fi
+
+  declare -a foreign_options || return
+
+  local i=0
+  local opt
+  for opt in "${foreign_options[@]}" in; do
+    declare "foreign_option_$(( i += 1 ))=${opt}"
+  done
 
-  (( RUNTEST_CALLED > 0 )) || runtest
+  runtest
 }
 
 echo "update-systemd-resolved Test Suite"
@@ -376,5 +409,13 @@ done
 echo -e "  ${GREEN}${PASS} ${COUNT_PASS} Passed${RESET}"
 echo -e "  ${RED}${FAIL} ${COUNT_FAIL} Failed${RESET}"
 
+if (( "${#FAILED[@]}" > 0 )); then
+  echo -e "\n  ${YELLOW}The following files have failing test cases:${RESET}"
+
+  for failed in "${!FAILED[@]}"; do
+    echo -e "    ${ORANGE}${failed}${RESET}"
+  done
+fi
+
 # Make sure we fail if there are failed tests
 [[ ${COUNT_FAIL} -eq 0 ]]

tests/01_no_updates.sh

@@ -1,6 +1,5 @@
 # Emulate OpenVPN environment
 script_type="up"
-dev="tun01"
 
 TEST_TITLE="No Updates"
 TEST_BUSCTL_CALLED=1

tests/02_single_ipv4_dns.sh

@@ -1,6 +1,6 @@
 script_type="up"
-dev="tun02"
-foreign_option_1="dhcp-option DNS 1.23.4.56"
+
+foreign_options=("dhcp-option DNS 1.23.4.56")
 
 TEST_TITLE="Single IPv4 DNS Server"
 TEST_BUSCTL_CALLED=1

tests/03a_multiple_ipv4_dns_1.sh

@@ -1,7 +1,10 @@
 script_type="up"
-dev="tun03"
-foreign_option_1="dhcp-option DNS 1.23.4.56"
-foreign_option_2="dhcp-option DNS 5.6.7.89"
+
+
+foreign_options=(
+  "dhcp-option DNS 1.23.4.56"
+  "dhcp-option DNS 5.6.7.89"
+)
 
 TEST_TITLE="Multiple IPv4 DNS Servers (Part 1)"
 TEST_BUSCTL_CALLED=1

tests/03b_multiple_ipv4_dns_2.sh

@@ -1,8 +1,10 @@
 script_type="up"
-dev="tun03"
-foreign_option_1="dhcp-option DNS 1.23.4.56"
-foreign_option_2="dhcp-option DNS 5.6.7.89"
-foreign_option_3="dhcp-option DNS 34.5.67.8"
+
+foreign_options=(
+  "dhcp-option DNS 1.23.4.56"
+  "dhcp-option DNS 5.6.7.89"
+  "dhcp-option DNS 34.5.67.8"
+)
 
 TEST_TITLE="Multiple IPv4 DNS Servers (Part 2)"
 TEST_BUSCTL_CALLED=1

tests/04a_single_dns_domain.sh

@@ -1,6 +1,6 @@
 script_type="up"
-dev="tun04"
-foreign_option_1="dhcp-option DOMAIN example.com"
+
+foreign_options=("dhcp-option DOMAIN example.com")
 
 TEST_TITLE="Single DNS Domain"
 TEST_BUSCTL_CALLED=1

tests/04b_multiple_dns_domains.sh

@@ -1,7 +1,9 @@
 script_type="up"
-dev="tun04"
-foreign_option_1="dhcp-option DOMAIN example.com"
-foreign_option_2="dhcp-option DOMAIN example.co"
+
+foreign_options=(
+  "dhcp-option DOMAIN example.com"
+  "dhcp-option DOMAIN example.co"
+)
 
 TEST_TITLE="Multiple DNS Domains"
 TEST_BUSCTL_CALLED=1

tests/05a_dns_domain_and_search_1.sh

@@ -1,7 +1,9 @@
 script_type="up"
-dev="tun05"
-foreign_option_1="dhcp-option DOMAIN example.com"
-foreign_option_2="dhcp-option DOMAIN-SEARCH example.org"
+
+foreign_options=(
+  "dhcp-option DOMAIN example.com"
+  "dhcp-option DOMAIN-SEARCH example.org"
+)
 
 TEST_TITLE="DNS Single Domain and Single Search"
 TEST_BUSCTL_CALLED=1

tests/05b_dns_domain_and_search_2.sh

@@ -1,8 +1,10 @@
 script_type="up"
-dev="tun05"
-foreign_option_1="dhcp-option DOMAIN example.com"
-foreign_option_2="dhcp-option DOMAIN-SEARCH example.org"
-foreign_option_3="dhcp-option DOMAIN-SEARCH example.net"
+
+foreign_options=(
+  "dhcp-option DOMAIN example.com"
+  "dhcp-option DOMAIN-SEARCH example.org"
+  "dhcp-option DOMAIN-SEARCH example.net"
+)
 
 TEST_TITLE="DNS Single Domain and Dual Search"
 TEST_BUSCTL_CALLED=1

tests/05c_dns_domain_and_search_3.sh

@@ -1,8 +1,10 @@
 script_type="up"
-dev="tun05"
-foreign_option_1="dhcp-option DOMAIN-SEARCH example.org"
-foreign_option_2="dhcp-option DOMAIN example.com"
-foreign_option_3="dhcp-option DOMAIN-SEARCH example.net"
+
+foreign_options=(
+  "dhcp-option DOMAIN-SEARCH example.org"
+  "dhcp-option DOMAIN example.com"
+  "dhcp-option DOMAIN-SEARCH example.net"
+)
 
 TEST_TITLE="DNS Single Domain and Dual Search (with Order Check)"
 TEST_BUSCTL_CALLED=1