Replace shell commands with subprocess.run

Signed-off-by: Bruno Alvisio <[email protected]>

Author: balvisio

Dockerfile

@@ -123,15 +123,15 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
     pip install .; \
 fi
 
-# On ARM, bits and bytes needs to be built from scratch
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
-    cd / && pip uninstall bitsandbytes && \
-    git clone --single-branch --branch 0.45.5 https://github.com/bitsandbytes-foundation/bitsandbytes.git && \
-    cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cuda -S . && make && pip install . && cd .. && rm -rf bitsandbytes; \
-fi
 ###############################################################################
 # /end ARM
 ###############################################################################
+
+# Bits and bytes needs to be built from scratch
+RUN cd / && pip uninstall bitsandbytes && \
+    git clone --single-branch --branch 0.45.5 https://github.com/bitsandbytes-foundation/bitsandbytes.git && \
+    cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cuda -S . && make && pip install . && cd .. && rm -rf bitsandbytes
+
 # Fix the version of scikit-misc to 0.3.1 because newer versions of scikit-misc require numpy >= 2.0 to be built.
 # Since there are not pre-built wheels for arm64, we need to install this specific version.
 # Once bionemo is compatible with numpy >= 2.0, we can remove this.

ci/scripts/pytest_runner.sh

@@ -57,7 +57,7 @@ SKIP_SLOW=false
 ONLY_SLOW=false
 ALLOW_NO_TESTS=false
 # TODO(@cspades): Ignore this Evo2 notebook test, which has a tendency to leave a 32GB orphaned process in GPU.
-declare -a IGNORE_FILES=("sub-packages/bionemo-evo2/examples/fine-tuning-tutorial.ipynb")
+declare -a IGNORE_FILES=()
 error=false
 
 # Parse command line arguments

ci/scripts/run_pytest_notebooks.sh

@@ -19,4 +19,4 @@
 # Enable strict mode with better error handling
 set -euox pipefail
 
-pytest -v --nbval-lax -p no:python docs/ sub-packages/
+pytest -v --nbval-lax -x -p no:python docs/ sub-packages/

pyproject.toml

@@ -114,12 +114,12 @@ known-first-party = ["bionemo", "infra_bionemo"]
 convention = "google"
 
 [tool.pytest.ini_options]
+xfail_strict = true
 norecursedirs = ["3rdparty"]
 addopts = [
     "--durations-min=30.0",
     "--durations=0",
     "--ignore=3rdparty",
-    "--ignore-glob=sub-packages/bionemo-evo2/examples/fine-tuning-tutorial.ipynb",
     "--ignore-glob=sub-packages/bionemo-moco/examples/discrete_data_interpolant_tutorial.ipynb"
 ]
 markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]

sub-packages/bionemo-core/src/bionemo/core/utils/subprocess_utils.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: LicenseRef-Apache2
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import shlex
+import subprocess
+from typing import Any, Dict
+
+
+logger = logging.getLogger(__name__)
+
+
+def run_subprocess_safely(command: str, timeout: int = 2000) -> Dict[str, Any]:
+    """Run a subprocess and raise an error if it fails.
+
+    Args:
+        command: The command to run.
+        timeout: The timeout for the command.
+
+    Returns:
+        The result of the subprocess.
+    """
+    try:
+        result = subprocess.run(shlex.split(command), capture_output=True, timeout=timeout, check=True, text=True)
+        return {"stdout": result.stdout, "stderr": result.stderr, "returncode": result.returncode}
+    except subprocess.TimeoutExpired as e:
+        logger.error(f"Command timed out. Command: {command}\nstdout:\n{e.stdout}\nstderr:\n{e.stderr}")
+        return {"error": "timeout", "stdout": e.stdout, "stderr": e.stderr, "returncode": None}
+
+    except subprocess.CalledProcessError as e:
+        logger.error(
+            f"Command failed. Command: {command}\nreturncode: {e.returncode}\nstdout:\n{e.stdout}\nstderr:\n{e.stderr}"
+        )
+        return {"error": "non-zero exit", "stdout": e.stdout, "stderr": e.stderr, "returncode": e.returncode}
+
+    except FileNotFoundError as e:
+        logger.error(f"Command not found. Command: {command}\nstderr:\n{str(e)}")
+        return {"error": "not found", "stdout": "", "stderr": str(e), "returncode": None}
+
+    except Exception as e:
+        # catch-all for other unexpected errors
+        return {"error": "other", "message": str(e), "stdout": "", "stderr": "", "returncode": None}

sub-packages/bionemo-evo2/examples/fine-tuning-tutorial.ipynb

@@ -96,6 +96,8 @@
     "%%capture\n",
     "import os\n",
     "\n",
+    "from bionemo.core.utils.subprocess_utils import run_subprocess_safely\n",
+    "\n",
     "\n",
     "concat_path = \"chr20_21_22.fa\"\n",
     "if not os.path.exists(concat_path):\n",
@@ -301,7 +303,18 @@
     "    --create-tensorboard-logger \\\n",
     "    --ckpt-async-save\"\"\"\n",
     "\n",
-    "!{train_cmd}"
+    "print(f\"Running command: {train_cmd}\")\n",
+    "\n",
+    "result = run_subprocess_safely(train_cmd)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert result[\"returncode\"] == 0, result"
    ]
   },
   {

sub-packages/bionemo-evo2/examples/zeroshot_brca1.ipynb

@@ -50,7 +50,9 @@
     "import seaborn as sns\n",
     "import torch\n",
     "from Bio import SeqIO\n",
-    "from sklearn.metrics import auc, roc_auc_score, roc_curve"
+    "from sklearn.metrics import auc, roc_auc_score, roc_curve\n",
+    "\n",
+    "from bionemo.core.utils.subprocess_utils import run_subprocess_safely"
    ]
   },
   {
@@ -697,7 +699,17 @@
    "source": [
     "%%capture\n",
     "print(f\"Running command: {predict_ref_command}\")\n",
-    "!{predict_ref_command}"
+    "\n",
+    "result = run_subprocess_safely(predict_ref_command)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert result[\"returncode\"] == 0, result"
    ]
   },
   {
@@ -715,7 +727,17 @@
    "source": [
     "%%capture\n",
     "print(f\"Running command: {predict_var_command}\")\n",
-    "!{predict_var_command}"
+    "\n",
+    "result = run_subprocess_safely(predict_var_command)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert result[\"returncode\"] == 0, result"
    ]
   },
   {