AWS: add a cmd to find error in _training_successful (#591)

Author: frcaud

doc/whats_new/v0.11.rst

@@ -10,4 +10,5 @@ Changelog
 `ramp-database`
 ...............
 
+- AWS runner: better detection of training errors :pr:`591`
 - Switch to fetching starting kit repos via HTTP rather than using git clone to avoid being blocked by Github :pr:`592`.

ramp-engine/ramp_engine/aws/api.py

@@ -834,6 +834,11 @@ def _training_successful(config, instance_id, submission_name, actual_nb_folds=N
 
     cmd = "find {}|egrep 'fold.*/y_pred_test.npz'|wc -l".format(folder)
     nb_test_files = int(_run(config, instance_id, cmd, return_output=True))
+
+    cmd = f"find {folder}|egrep 'fold.*/error.txt'|wc -l"
+    n_error = int(_run(config, instance_id, cmd, return_output=True))
+    if n_error > 0:
+        return False
     if actual_nb_folds is not None:
         return nb_folds == nb_train_files == nb_test_files == actual_nb_folds
     else: