fix: automatically create batch output directories

Author: yaugenst-flex

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Bug in `LayerRefinementSpec` that refines grids outside the layer region when one in-plane dimension is of size infinity.
 - Querying tasks was sometimes erroring unexpectedly.
+- Fixed automatic creation of missing output directories.
 
 ## [2.8.0] - 2025-03-04
 

tests/test_web/test_webapi.py

@@ -1,5 +1,6 @@
 # Tests webapi and things that depend on it
 
+
 import numpy as np
 import pytest
 import responses
@@ -564,6 +565,34 @@ def test_batch(mock_webapi, mock_job_status, mock_load, tmp_path):
     assert b2.real_cost() == FLEX_UNIT * len(sims)
 
 
+@responses.activate
+def test_create_output_dirs(mock_webapi, tmp_path, monkeypatch):
+    """Test that Job and Batch create output directories if they don't exist."""
+    monkeypatch.setattr(f"{api_path}.load", lambda *args, **kwargs: True)
+    non_existent_dirs_job = tmp_path / "new/nested/folders/job"
+    output_file_job = non_existent_dirs_job / "output.hdf5"
+
+    assert not non_existent_dirs_job.exists()
+
+    sim = make_sim()
+    job = Job(simulation=sim, task_name=TASK_NAME, folder_name=PROJECT_NAME)
+    job.run(path=str(output_file_job))
+
+    assert non_existent_dirs_job.exists()
+    assert non_existent_dirs_job.is_dir()
+
+    non_existent_dirs_batch = tmp_path / "new/nested/folders/batch"
+
+    assert not non_existent_dirs_batch.exists()
+
+    sims = {TASK_NAME: make_sim()}
+    batch = Batch(simulations=sims, folder_name=PROJECT_NAME)
+    batch.run(path_dir=str(non_existent_dirs_batch))
+
+    assert non_existent_dirs_batch.exists()
+    assert non_existent_dirs_batch.is_dir()
+
+
 """ Async """
 
 

tidy3d/web/api/container.py

@@ -34,12 +34,13 @@
 class WebContainer(Tidy3dBaseModel, ABC):
     """Base class for :class:`Job` and :class:`Batch`, technically not used"""
 
+    from abc import abstractmethod
+
     @staticmethod
-    def _check_path_dir(path_dir: str) -> None:
-        """Make sure ``path_dir`` exists and create one if not."""
-        location = os.path.dirname(path_dir)
-        if len(location) > 0 and not os.path.exists(location):
-            os.makedirs(location, exist_ok=True)
+    @abstractmethod
+    def _check_path_dir(path: str) -> None:
+        """Make sure local output directory exists and create it if not."""
+        pass
 
     @staticmethod
     def _check_folder(folder_name: str) -> None:
@@ -205,7 +206,6 @@ def to_file(self, fname: str) -> None:
         -------
         >>> simulation.to_file(fname='folder/sim.json') # doctest: +SKIP
         """
-
         task_id_cached = self._cached_properties.get("task_id")
         self = self.updated_copy(task_id_cached=task_id_cached)
         super(Job, self).to_file(fname=fname)  # noqa: UP008
@@ -223,7 +223,6 @@ def run(self, path: str = DEFAULT_DATA_PATH) -> SimulationDataType:
         Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`]
             Object containing simulation results.
         """
-
         self.upload()
         self.start()
         self.monitor()
@@ -305,7 +304,7 @@ def download(self, path: str = DEFAULT_DATA_PATH) -> None:
         ----
         To load the data after download, use :meth:`Job.load`.
         """
-        self._check_path_dir(path_dir=path)
+        self._check_path_dir(path=path)
         web.download(task_id=self.task_id, path=path, verbose=self.verbose)
 
     def load(self, path: str = DEFAULT_DATA_PATH) -> SimulationDataType:
@@ -321,7 +320,7 @@ def load(self, path: str = DEFAULT_DATA_PATH) -> SimulationDataType:
         Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`]
             Object containing simulation results.
         """
-        self._check_path_dir(path_dir=path)
+        self._check_path_dir(path=path)
         data = web.load(task_id=self.task_id, path=path, verbose=self.verbose)
         if isinstance(self.simulation, ModeSolver):
             self.simulation._patch_data(data=data)
@@ -366,6 +365,19 @@ def estimate_cost(self, verbose: bool = True) -> float:
         """
         return web.estimate_cost(self.task_id, verbose=verbose, solver_version=self.solver_version)
 
+    @staticmethod
+    def _check_path_dir(path: str) -> None:
+        """Make sure parent directory of ``path`` exists and create it if not.
+
+        Parameters
+        ----------
+        path : str
+            Path to file to be created (including filename).
+        """
+        parent_dir = os.path.dirname(path)
+        if len(parent_dir) > 0 and not os.path.exists(parent_dir):
+            os.makedirs(parent_dir, exist_ok=True)
+
 
 class BatchData(Tidy3dBaseModel, Mapping):
     """
@@ -1030,3 +1042,15 @@ def estimate_cost(self, verbose: bool = True) -> float:
                 console.log("Could not get estimated batch cost!")
 
         return batch_cost
+
+    @staticmethod
+    def _check_path_dir(path_dir: str) -> None:
+        """Make sure ``path_dir`` exists and create it if not.
+
+        Parameters
+        ----------
+        path_dir : str
+            Directory path where files will be saved.
+        """
+        if len(path_dir) > 0 and not os.path.exists(path_dir):
+            os.makedirs(path_dir, exist_ok=True)