slurm_submit_python() don't print cmd if running slurm interactively

only in log file or at job submission time

Author: janosh

.gitignore

@@ -27,3 +27,4 @@ paper
 meeting-notes
 models/voronoi/*
 !models/voronoi/*.py
+pretrained

matbench_discovery/slurm.py

@@ -67,8 +67,9 @@ def slurm_submit_python(
     if array:
         cmd += ["--array", array]
 
-    running_as_slurm_job = "SLURM_JOB_ID" in os.environ
-    if running_as_slurm_job or "slurm-submit" in sys.argv:
+    is_log_file = not sys.stdout.isatty()
+    is_slurm_job = "SLURM_JOB_ID" in os.environ
+    if (is_slurm_job and is_log_file) or "slurm-submit" in sys.argv:
         # print sbatch command at submission time and into slurm log file
         # but not when running in command line or Jupyter
         print(" ".join(cmd))

models/wrenformer/slurm_array_wrenformer.py

@@ -56,7 +56,7 @@
 timestamp = f"{datetime.now():%Y-%m-%d@%H-%M-%S}"
 
 print(f"Job started running {datetime.now():%Y-%m-%d@%H-%M}")
-slurm_job_id = os.environ["SLURM_JOB_ID"]
+slurm_job_id = os.environ.get("SLURM_JOB_ID")
 slurm_array_task_id = int(os.environ.get("SLURM_ARRAY_TASK_ID", 0))
 
 print(f"{slurm_job_id=}")