Fix incorrect RC with --fail-on-pipeline-failure

This fixes an important bug in --fail-on-pipeline-failure. Prior to
this commit, litani run-build would only terminate abnormally with
this flag set when ninja terminated abnormally. This commit fixes this
behavior so that litani run-build instead checks whether any pipeline
failed, and terminates abnormally if so.

When using --fail-on-pipeline-failure, `litani run-build` will now
terminate with an exit code of 0 if all pipelines passed, and 10 if at
least one of them failed.

Author: karkhaz

lib/run_build.py

@@ -12,6 +12,7 @@
 # permissions and limitations under the License
 
 import datetime
+import itertools
 import json
 import logging
 import os
@@ -219,6 +220,9 @@ async def run_build(args):
               f"file://{run_info['latest_symlink']}/html/index.html")
 
     if args.fail_on_pipeline_failure:
-        sys.exit(0 if runner.was_successful() else 1)
+        for _ in itertools.filterfalse(
+                lambda pipe: pipe["status"] != "fail", run["pipelines"]):
+            sys.exit(10)
+        sys.exit(0)
 
     sys.exit(0)

test/e2e/run

@@ -29,17 +29,18 @@ DESCRIPTION = "Execute a single Litani run and test the resulting run.json"
 EPILOG = "See test/e2e/README for the organization of this test suite"
 
 
-def run_cmd(cmd):
+def run_cmd(cmd, check):
     cmd_list = [str(c) for c in cmd]
     print(" ".join(cmd_list))
     try:
-        subprocess.run(cmd_list, check=True)
+        proc = subprocess.run(cmd_list, check=check)
     except subprocess.CalledProcessError:
         logging.error("Invocation failed")
         sys.exit(1)
     except FileNotFoundError:
         logging.error("Executable not found")
         sys.exit(1)
+    return proc
 
 
 def configure_args(*args, **kwargs):
@@ -57,10 +58,10 @@ def configure_args(*args, **kwargs):
     return cmd
 
 
-def run_litani(litani, subcommand, *args, **kwargs):
+def run_litani(litani, subcommand, *args, check=True, **kwargs):
     cmd = [litani, subcommand]
     cmd.extend(configure_args(*args, **kwargs))
-    run_cmd(cmd)
+    return run_cmd(cmd, check)
 
 
 def get_test_module(module_file):
@@ -130,8 +131,24 @@ def set_jobs(litani, run_dir, mod):
 def run_build(litani, run_dir, mod):
     os.chdir(run_dir)
     args = mod.get_run_build_args()
-    run_litani(
-        litani, "run-build", *args.get("args", []), **args.get("kwargs", {}))
+    proc = run_litani(
+        litani, "run-build",
+        *args.get("args", []), check=False, **args.get("kwargs", {}))
+
+    try:
+        expected_rc = mod.get_run_build_return_code()
+        if expected_rc != proc.returncode:
+            logging.error(
+                "Expected return code %d did not match actual "
+                "return code %d", expected_rc, proc.returncode)
+            sys.exit(1)
+    except AttributeError:
+        # test does not have a get_run_build_return_code method, so assume that
+        # run-build is expected to succeed
+        if proc.returncode:
+            logging.error("Invocation failed")
+            sys.exit(1)
+
 
 
 def init(litani, run_dir, mod):

test/e2e/tests/exit_rc.py

@@ -0,0 +1,44 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+SLOW = False
+
+def get_init_args():
+    return {
+        "kwargs": {
+            "project": "foo",
+        }
+    }
+
+
+def get_jobs():
+    return [{
+        "kwargs": {
+            "command": "false",
+            "ci-stage": "build",
+            "pipeline": "foo",
+            "ignore_returns": "1",
+        }
+    }]
+
+
+def get_run_build_args():
+    return {}
+
+
+def get_run_build_return_code():
+    return 0
+
+
+def check_run(run):
+    return True

test/e2e/tests/exit_rc_fail_on_pipeline_failure.py

@@ -0,0 +1,48 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+SLOW = False
+
+def get_init_args():
+    return {
+        "kwargs": {
+            "project": "foo",
+        }
+    }
+
+
+def get_jobs():
+    return [{
+        "kwargs": {
+            "command": "false",
+            "ci-stage": "build",
+            "pipeline": "foo",
+            "ignore_returns": "1",
+        }
+    }]
+
+
+def get_run_build_args():
+    return {
+        "args": [
+            "fail-on-pipeline-failure",
+        ]
+    }
+
+
+def get_run_build_return_code():
+    return 10
+
+
+def check_run(run):
+    return True