Patch drupal exit for behat tests #33

Author: deviantintegral

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased] - YYYY-MM-DD
+
+### Fixed
+
+* Certain types of PHP errors did not throw a failed exit code in Behat tests.
+  [_drupal_log_error() returns a 0 exit code on errors](https://www.drupal.org/project/drupal/issues/2927012)
+  has been applied to Drupal core until it's fixed upstream. [#33](https://github.com/deviantintegral/drupal_tests/pull/33)
+
 ## [0.3.0] - 2018-02-21
 
 ### Added

Dockerfile

@@ -57,11 +57,17 @@ RUN cd /var/www/html \
 
 COPY hooks/* /var/www/html/
 
+# Commit our preinstalled Drupal database for faster Behat tests.
 COPY drupal.sql.gz /var/www
 COPY settings.php /var/www
 RUN mkdir -p /var/www/html/sites/default/files/config_yt3arM1I65-zRJQc52H_nu_xyV-c4YyQ86uwM1E3JBCvD3CXL38O8JqAxqnWWj8rHRiigYrj0w/sync \
   && chown -Rv www-data /var/www/html/sites/default/files
 
+# Patch Drupal to avoid a bug where behat failures show as passes.
+# https://www.drupal.org/project/drupal/issues/2927012#comment-12467957
+RUN cd /var/www/html \
+  && curl https://www.drupal.org/files/issues/2927012.22-log-error-exit-code.patch | patch -p1
+
 # Add the vendor/bin directory to the $PATH
 ENV PATH="/var/www/html/vendor/bin:${PATH}"
 

fixtures/behat-fail.patch

@@ -0,0 +1,12 @@
+diff --git a/tests/src/Behat/features/bootstrap/NodeFeatureContext.php b/tests/src/Behat/features/bootstrap/NodeFeatureContext.php
+index a02a635..2610535 100644
+--- a/tests/src/Behat/features/bootstrap/NodeFeatureContext.php
++++ b/tests/src/Behat/features/bootstrap/NodeFeatureContext.php
+@@ -18,6 +18,7 @@ class NodeFeatureContext extends RawDrupalContext implements SnippetAcceptingCon
+    * @BeforeSuite
+    */
+   public static function prepare(BeforeSuiteScope $scope) {
++    \_drupal_exception_handler(new \Exception());
+     /** @var \Drupal\Core\Extension\ModuleHandler $moduleHandler */
+     $moduleHandler = \Drupal::service('module_handler');
+     if (!$moduleHandler->moduleExists('node')) {

test.sh

@@ -16,12 +16,24 @@ test_ci() {
   set -e
 
   # This module fails CS jobs currently so this is more informational.
-  circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-code-sniffer || true
-  circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-unit-kernel-tests
+  if [ ! -z $1 ]
+  then
+    circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-code-sniffer || true
+    circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-unit-kernel-tests
+
+    circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-behat-tests | tee behat.log
+    # We need to skip colour codes
+    egrep "1 scenario \\(.*1 passed" behat.log
+
+    # Test that a PHP FATAL error properly fails the job.
+    git apply ../fixtures/behat-fail.patch
 
-  circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-behat-tests | tee behat.log
-  # We need to skip colour codes
-  egrep "1 scenario \\(.*1 passed" behat.log
+    # circleci doesn't bubble the exit code from behat :(
+    circleci.sh -e CIRCLE_PROJECT_REPONAME=node build --job run-behat-tests | tee behat.log
+    grep -A9 'Behat tests failed' behat.log | tail -n 1 | grep '+ exit 1'
+
+    git reset --hard HEAD
+  fi
 }
 
 sudo apt-get update -y