tooling: Add test UUIDs

In order for Cloe to be used in certain contexts, quality measures
need to be uniquely identifiable and trackable in the code base.
To this end, we are adding UUIDs to all system test cases. These
will not necessarily be referred to inside the repository, but
additions, moves, and removals can be easily tracked with Git.

Author: cassava

Makefile

@@ -116,6 +116,9 @@ todos:
 	${AG} FIXME
 	${AG} XXX
 
+grep-uuids:
+	${AG} "\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
+
 find-missing-eol:
 	find . -type f -size +0 -exec gawk 'ENDFILE{if ($0 == "") print FILENAME}' {} \;
 

tests/README.md

@@ -100,3 +100,17 @@ JSON stack file.
 
 Each stack file tests a part of the Cloe runtime functionality, and is usually
 accompanied by a Bats file.
+
+Test UUIDs
+==========
+
+In order to make tracking quality measures possible, we assign tests UUIDs.
+These can be generated with the `uuidgen` program, for example. When creating
+a large amount of UUIDs, it can be useful to augment one's editor with the
+capability of inserting one quickly. For example, to map it to Ctrl+Y in Vim:
+
+    :imap <c-y> <esc>:read !uuidgen<cr>kJi
+
+Search the project for UUIDs with:
+
+    make grep-uuids

tests/setup_bats.bash

@@ -140,3 +140,10 @@ test_plugin_exists() {
 
     cloe_engine usage "${plugin}" &>/dev/null
 }
+
+testname() {
+    local expect="$1"
+    local schema="$2"
+    local uuid="$3"
+    printf "%-24s: %-46s : %s" "$expect" "$schema" "$uuid"
+}

tests/test_all_plugin_usages.bats

@@ -16,7 +16,7 @@ setup() {
     export cloe_plugins
 }
 
-@test "Expect run success      : cloe-engine usage" {
+@test $(testname "Expect run success" "cloe-engine usage" "93c5a785-8f07-4df2-92ee-6d47e728f201") {
     for file in "${cloe_plugins[@]}"; do
         echo "Checking: $file"
         run cloe_engine usage $file
@@ -28,7 +28,7 @@ setup() {
     done
 }
 
-@test "Expect run success      : cloe-engine usage -j" {
+@test $(testname "Expect run success" "cloe-engine usage -j" "7576ad69-1d16-43f8-9809-af0bf408e4f1") {
     if ! type jq &>/dev/null; then
         skip "required program jq not present"
     fi

tests/test_engine.bats

@@ -7,7 +7,7 @@ test_vtd_plugin_exists() {
     cloe_shell -c 'test -d "${VTD_ROOT}"'
 }
 
-@test "Expect schema equality  : test_engine_json_schema.json" {
+@test $(testname "Expect schema equality" "test_engine_json_schema.json" "4d368665-b666-4289-8a7a-b76ca53db688") {
     # Note: you will have to update the schema files every time you change the schema,
     # in order that this test completes successfully. Here's how you do it.
     #
@@ -26,245 +26,245 @@ test_vtd_plugin_exists() {
     fi
 }
 
-@test "Expect check success    : test_engine_smoketest.json" {
+@test $(testname "Expect check success" "test_engine_smoketest.json" "20c3f11e-4a93-4066-b61e-d485be5c8979") {
     cloe_engine check test_engine_smoketest.json
 }
 
-@test "Expect run success      : test_engine_smoketest.json" {
+@test $(testname "Expect run success" "test_engine_smoketest.json" "b590e751-dace-4139-913c-a4a812af70ac") {
     cloe_engine run test_engine_smoketest.json
 }
 
-@test "Expect check failure    : test_engine_bad_logging.json" {
+@test $(testname "Expect check failure" "test_engine_bad_logging.json" "107c36fe-7bd9-4559-b5e9-74b72baafd9f") {
     run cloe_engine check test_bad_logging.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check success    : test_engine_curl_succeed.json" {
+@test $(testname "Expect check success" "test_engine_curl_succeed.json" "5eff0c85-77f1-4792-9987-e46a36617d99") {
     if ! type curl &>/dev/null; then
         skip "required program curl not present"
     fi
 
     cloe_engine check test_engine_curl_succeed.json
 }
 
-@test "Expect run success      : test_engine_curl_succeed.json" {
+@test $(testname "Expect run success" "test_engine_curl_succeed.json" "f473cb96-7f2e-4ac1-801a-fd93343f6e24") {
     if ! type curl &>/dev/null; then
         skip "required program curl not present"
     fi
     cloe_engine run test_engine_curl_succeed.json
 }
 
-@test "Expect run failure      : test_engine_curl_succeed.json" {
+@test $(testname "Expect run failure" "test_engine_curl_succeed.json" "7aa4b455-ad74-4e5f-bf82-43761d7cd81b") {
     # When disabling the `enable_command_action` flag, curl
     # should not be used and the simulation should fail by default.
     run cloe_engine run test_engine_curl_succeed.json \
         <(echo '{ "version": "4", "engine": { "security": { "enable_command_action": false } } }')
     test $status -eq $exit_outcome_failure
 }
 
-@test "Expect check failure    : test_engine_empty.json" {
+@test $(testname "Expect check failure" "test_engine_empty.json" "d04369fb-e4af-4f80-aaa8-8352d3dec42e") {
     run cloe_engine check test_engine_empty.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
     test $(echo $output 2>&1 | wc -l) -lt 32
 }
 
-@test "Expect check failure    : test_engine_version_absent.json" {
+@test $(testname "Expect check failure" "test_engine_version_absent.json" "4de20aed-2a8f-442b-bdb3-1da76237ee1e") {
     run cloe_engine check test_engine_version_absent.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
     test $(echo $output 2>&1 | wc -l) -lt 32
 }
 
-@test "Expect check failure    : test_engine_version_wrong.json" {
+@test $(testname "Expect check failure" "test_engine_version_wrong.json" "2c883a3d-b877-4fca-88a1-8078208f0d2b") {
     run cloe_engine check test_engine_version_wrong.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
     test $(echo $output 2>&1 | wc -l) -lt 32
 }
 
-@test "Expect check success    : test_engine_fail_trigger.json" {
+@test $(testname "Expect check success" "test_engine_fail_trigger.json" "cd540e21-63e9-421f-8e7b-e113339253a2") {
     cloe_engine check test_engine_fail_trigger.json
 }
 
-@test "Expect run failure      : test_engine_fail_trigger.json" {
+@test $(testname "Expect run failure" "test_engine_fail_trigger.json" "c9bc16de-4902-4abf-90ac-d4bef0d0b26e") {
     run cloe_engine run test_engine_fail_trigger.json
     assert_exit_failure $status
     test $status -eq $exit_outcome_failure
     echo "$output" | grep '"outcome": "failure"'
 }
 
-@test "Expect check failure    : test_engine_hook_invalid.json" {
+@test $(testname "Expect check failure" "test_engine_hook_invalid.json" "cc3d8879-5ca5-4ffb-a78e-c9723ac82b91") {
     run cloe_engine check test_engine_hook_invalid.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check failure    : test_engine_hook_noexec.json" {
+@test $(testname "Expect check failure" "test_engine_hook_noexec.json" "3f35bb1f-307a-42e3-a2ea-739e60cab084") {
     run cloe_engine check test_engine_hook_noexec.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check success    : test_engine_hook_failure.json" {
+@test $(testname "Expect check success" "test_engine_hook_failure.json" "d6d8fe35-c513-4e43-8984-e9da0958c0fd") {
     cloe_engine check test_engine_hook_failure.json
 }
 
-@test "Expect run failure      : test_engine_hook_failure.json" {
+@test $(testname "Expect run failure" "test_engine_hook_failure.json" "798ee383-667d-4971-af2f-51c49c47b750") {
     run cloe_engine run test_engine_hook_failure.json
     test $status -eq $exit_outcome_aborted
 }
 
-@test "Expect check success    : test_engine_hook_ok.json" {
+@test $(testname "Expect check success" "test_engine_hook_ok.json" "2e3064bf-0e6d-4934-b25f-78cae54bd4d1") {
     cloe_engine check test_engine_hook_ok.json
 }
 
-@test "Expect run success      : test_engine_hook_ok.json" {
+@test $(testname "Expect run success" "test_engine_hook_ok.json" "303afa7e-3ebe-4db3-a802-84b2cdba97c5") {
     cloe_engine run test_engine_hook_ok.json
 }
 
-@test "Expect check success    : test_engine_ignore.json" {
+@test $(testname "Expect check success" "test_engine_ignore.json" "78a470a4-cbe1-4436-a43f-d956685b8bc9") {
     cloe_engine check test_engine_ignore.json
 }
 
-@test "Expect run success      : test_engine_ignore.json" {
+@test $(testname "Expect run success" "test_engine_ignore.json" "1738bf42-3784-4e9c-b4d4-76e94c8e5271") {
     cloe_engine run test_engine_ignore.json
 }
 
-@test "Expect check failure    : test_engine_include_nonexistent.json" {
+@test $(testname "Expect check failure" "test_engine_include_nonexistent.json" "bad115cc-0397-48e6-9a51-bdcfeaf6b024") {
     run cloe_engine check test_engine_include_nonexistent.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check failure    : test_engine_include_self.json" {
+@test $(testname "Expect check failure" "test_engine_include_self.json" "7b3e4010-19a0-4518-b8d7-00655af93755") {
     run cloe_engine check test_engine_include_self.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check failure    : test_engine_incomplete.json" {
+@test $(testname "Expect check failure" "test_engine_incomplete.json" "8baa711a-9279-4527-9ecc-6b685551a45f") {
     run cloe_engine check test_engine_incomplete.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect run failure      : test_engine_incomplete.json" {
+@test $(testname "Expect run failure" "test_engine_incomplete.json" "75a0c642-2373-4d6d-bd40-61e0f7840c57") {
     run cloe_engine run test_engine_incomplete.json
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check success    : test_engine_keep_alive.json" {
+@test $(testname "Expect check success" "test_engine_keep_alive.json" "254544dc-c17a-4a5c-8685-723ed1c758cf") {
     if ! type kill &>/dev/null; then
         skip "required program kill not present"
     fi
 
     cloe_engine check test_engine_keep_alive.json
 }
 
-@test "Expect run success      : test_engine_keep_alive.json" {
+@test $(testname "Expect run success" "test_engine_keep_alive.json" "0c5ace05-f5ca-4615-9c14-62a75b69651a") {
     if ! type kill &>/dev/null; then
         skip "required program kill not present"
     fi
 
     cloe_engine run test_engine_keep_alive.json
 }
 
-@test "Expect check success    : test_engine_namespaced_smoketest.json" {
+@test $(testname "Expect check success" "test_engine_namespaced_smoketest.json" "c9d4f8d3-aec7-404e-95f5-7a14cd8674c7") {
     cloe_engine check test_engine_namespaced_smoketest.json
 }
 
-@test "Expect run success      : test_engine_namespaced_smoketest.json" {
+@test $(testname "Expect run success" "test_engine_namespaced_smoketest.json" "c2fd481b-f135-4fb5-86f0-699af0f93497") {
     cloe_engine run test_engine_namespaced_smoketest.json
 }
 
-@test "Expect check failure    : test_engine_no_binding.json" {
+@test $(testname "Expect check failure" "test_engine_no_binding.json" "111a4f1f-7679-48a8-88de-a6773dab055e") {
     run cloe_engine check test_engine_no_binding.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect run failure      : test_engine_no_binding.json" {
+@test $(testname "Expect run failure" "test_engine_no_binding.json" "dc682e38-38e5-43d7-8989-b477cb0f6c2a") {
     run cloe_engine run test_engine_no_binding.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check failure    : test_engine_no_vehicle.json" {
+@test $(testname "Expect check failure" "test_engine_no_vehicle.json" "319a342a-88d2-4b07-a4de-1fef78b13d72") {
     run cloe_engine check test_engine_no_vehicle.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect run failure      : test_engine_no_vehicle.json" {
+@test $(testname "Expect run failure" "test_engine_no_vehicle.json" "4b4fcd8b-add1-451c-ad7b-9c0a3efb6525") {
     run cloe_engine run test_engine_no_vehicle.json
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check failure    : test_engine_unknown_vehicle.json" {
+@test $(testname "Expect check failure" "test_engine_unknown_vehicle.json" "3f754bea-4806-4f43-9c60-a78b13b43f6f") {
     run cloe_engine check test_engine_unknown_vehicle.json
     assert_check_failure $status $output
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect run failure      : test_engine_unknown_vehicle.json" {
+@test $(testname "Expect run failure" "test_engine_unknown_vehicle.json" "da3d7c55-6024-481b-bee4-32233df6c330") {
     run cloe_engine run test_engine_unknown_vehicle.json
     test $status -eq $exit_outcome_unknown
 }
 
-@test "Expect check success    : test_engine_pause.json" {
+@test $(testname "Expect check success" "test_engine_pause.json" "24528c78-f681-44ff-8b9c-5c172acf19b9") {
     if ! type curl &>/dev/null; then
         skip "required program curl not present"
     fi
 
     cloe_engine check test_engine_pause.json
 }
 
-@test "Expect run success      : test_engine_pause.json" {
+@test $(testname "Expect run success" "test_engine_pause.json" "845e3c9b-2a6d-469a-93a7-67fe9531c81e") {
     if ! type curl &>/dev/null; then
         skip "required program curl not present"
     fi
 
     cloe_engine run test_engine_pause.json
 }
 
-@test "Expect check success    : test_engine_sticky_trigger.json" {
+@test $(testname "Expect check success" "test_engine_sticky_trigger.json" "76400941-50f5-4185-9107-335005079569") {
     cloe_engine check test_engine_sticky_trigger.json
 }
 
-@test "Expect run success      : test_engine_sticky_trigger.json" {
+@test $(testname "Expect run success" "test_engine_sticky_trigger.json" "dcfbdd81-8bb0-4de3-ac68-486aa5a9ce66") {
     skip "not implemented yet"
     cloe_engine run test_engine_sticky_trigger.json
 }
 
-@test "Expect check success    : test_engine_stuck_controller.json" {
+@test $(testname "Expect check success" "test_engine_stuck_controller.json" "72f151f7-26c3-4a79-a0ec-8b0d4e678442") {
     cloe_engine check test_engine_stuck_controller.json
 }
 
-@test "Expect run aborted      : test_engine_stuck_controller.json" {
+@test $(testname "Expect run aborted" "test_engine_stuck_controller.json" "13a244da-f18e-4dec-af5c-d5dcb4c2cd36") {
     run cloe_engine run test_engine_stuck_controller.json
     assert_exit_failure $status
     echo "$output" | grep '"outcome": "aborted"'
 }
 
-@test "Expect check success    : test_engine_stuck_controller_continue.json" {
+@test $(testname "Expect check success" "test_engine_stuck_controller_continue.json" "19b3d407-c980-4507-9bd2-9d6dca5e1320") {
     cloe_engine check test_engine_stuck_controller_continue.json
 }
 
-@test "Expect run success      : test_engine_stuck_controller_continue.json" {
+@test $(testname "Expect run success" "test_engine_stuck_controller_continue.json" "5692f85d-87a2-4866-9875-647802ce1d62") {
     cloe_engine run test_engine_stuck_controller_continue.json
 }
 
-@test "Expect check success    : test_engine_watchdog.json" {
+@test $(testname "Expect check success" "test_engine_watchdog.json" "30e097cf-6e51-484b-a212-690769cd4c91") {
     if ! type curl &>/dev/null; then
         skip "required program curl not present"
     fi
 
     cloe_engine check test_engine_watchdog.json
 }
 
-@test "Expect run syskill      : test_engine_watchdog.json" {
+@test $(testname "Expect run syskill" "test_engine_watchdog.json" "058ff9b7-98dc-4583-8e80-c70e9c5e1f4e") {
     if ! type curl &>/dev/null; then
         skip "required program curl not present"
     fi
@@ -276,13 +276,13 @@ test_vtd_plugin_exists() {
     test $status -eq $exit_outcome_syskill
 }
 
-@test "Expect check/run success: test_engine_smoketest.json [ts=5ms]" {
+@test $(testname "Expect check/run success" "test_engine_smoketest.json [ts=5ms]" "1a31022c-e20c-4a9e-9373-ad54a3729442") {
     BATS_OPTIONAL_STACKS="option_timestep_5.json"
     cloe_engine check test_engine_smoketest.json
     cloe_engine run test_engine_smoketest.json
 }
 
-@test "Expect check/run success: test_engine_smoketest.json [ts=60ms]" {
+@test $(testname "Expect check/run success" "test_engine_smoketest.json [ts=60ms]" "e7957fa0-1145-4458-b665-eec51c1f0da5") {
     BATS_OPTIONAL_STACKS="option_timestep_60.json"
     cloe_engine check test_engine_smoketest.json
     cloe_engine run test_engine_smoketest.json

tests/test_engine_replica_smoketest.bats

@@ -6,7 +6,7 @@
 
 load setup_bats
 
-@test "Expect exact replication: test_engine_replica_smoketest.json" {
+@test $(testname "Expect exact replication" "test_engine_replica_smoketest.json" "f4fa1794-cc78-4ae6-aa5e-6a3ed3c7913c") {
     # Clean up in case temporary registry already exists.
     if [[ -d "$cloe_tmp_registry" ]]; then
         rm -r "$cloe_tmp_registry" || true