Add support for arbitrary commands in formatted executables

This commit allows formatted_executable definitions to have a `commands`
attribute, which can define the lines that should be in the formatted
output.

The default is to format all of the commands for the experiment, but
this can be override to format arbitrary strings / variables. Item in
the `commands` list will be expanded as a variable definition, and then
split over new-lines before formatting based on the formatted executable
definition.

Author: douglasjacobsen

etc/ramble/defaults/formatted_executables.yaml

@@ -1,3 +1,5 @@
 formatted_executables:
   command:
     join_separator: '\n'
+    commands:
+    - '{unformatted_command}'

lib/ramble/ramble/application.py

@@ -1138,6 +1138,8 @@ def _define_formatted_executables(self):
         based on the formatting requested.
         """
 
+        self.variables[self.keywords.unformatted_command] = "\n".join(self._command_list)
+
         for var_name, formatted_conf in self._formatted_executables.items():
             if var_name in self.variables:
                 raise FormattedExecutableError(
@@ -1159,13 +1161,17 @@ def _define_formatted_executables(self):
 
             indentation = " " * n_indentation
 
-            formatted_str = ""
-            for cmd in self._command_list:
-                if formatted_str:
-                    formatted_str += join_separator
-                formatted_str += indentation + prefix + cmd
+            commands_to_format = self._command_list
+            if namespace.commands in formatted_conf:
+                commands_to_format = formatted_conf[namespace.commands].copy()
+
+            formatted_lines = []
+            for command in commands_to_format:
+                expanded = self.expander.expand_var(command)
+                for out_line in expanded.split("\n"):
+                    formatted_lines.append(indentation + prefix + out_line)
 
-            self.variables[var_name] = formatted_str
+            self.variables[var_name] = join_separator.join(formatted_lines)
 
     def _derive_variables_for_template_path(self, workspace):
         """Define variables for template paths (for add_expand_vars)"""
@@ -2135,7 +2141,7 @@ def _copy_files(obj_inst, obj_type, repo_root):
 
         repo_path = os.path.join(workspace.named_deployment, "object_repo")
 
-        repo_lock = lk.Lock(repo_path)
+        repo_lock = lk.Lock(os.path.join(repo_path, ".ramble-obj-repo.lock"))
 
         with lk.WriteTransaction(repo_lock):
             _copy_files(self, ramble.repository.ObjectTypes.applications, repo_path)

lib/ramble/ramble/keywords.py

@@ -47,6 +47,7 @@
     "batch_submit": {"type": key_type.required, "level": output_level.variable},
     "mpi_command": {"type": key_type.required, "level": output_level.variable},
     "experiment_template_name": {"type": key_type.reserved, "level": output_level.key},
+    "unformatted_command": {"type": key_type.reserved, "level": output_level.variable},
 }
 
 

lib/ramble/ramble/namespace.py

@@ -59,6 +59,7 @@ class namespace:
     indentation = "indentation"
     prefix = "prefix"
     join_separator = "join_separator"
+    commands = "commands"
 
     # For variants
     package_manager = "package_manager"

lib/ramble/ramble/schema/formatted_executables.py

@@ -26,6 +26,13 @@
                 "prefix": {"type": "string", "default": ""},
                 "indentation": {"type": "number", "default": 0},
                 "join_separator": {"type": "string", "default": "\n"},
+                "commands": {
+                    "type": "array",
+                    "default": ["{unformatted_command}"],
+                    "items": {
+                        "type": "string",
+                    },
+                },
             },
         },
     }

lib/ramble/ramble/test/data/config/formatted_executables.yaml

@@ -1,3 +1,5 @@
 formatted_executables:
   command:
     join_separator: '\n'
+    commands:
+    - '{unformatted_command}'

lib/ramble/ramble/test/end_to_end/formatted_executables.py

@@ -36,6 +36,11 @@ def test_formatted_executables(mutable_config, mutable_mock_workspace_path, mock
       prefix: 'from_ws '
       indentation: 9
       join_separator: ';'
+    ws_test_def:
+      prefix: 'test_from_ws '
+      indentation: 2
+      commands:
+      - '{mpi_command} test'
   applications:
     basic:
       formatted_executables:
@@ -74,6 +79,7 @@ def test_formatted_executables(mutable_config, mutable_mock_workspace_path, mock
             f.write("{app_exec_def}\n")
             f.write("{wl_exec_def}\n")
             f.write("{exp_exec_def}\n")
+            f.write("{ws_test_def}\n")
         ws._re_read()
 
         workspace("setup", "--dry-run", global_args=["-w", workspace_name])
@@ -88,6 +94,7 @@ def test_formatted_executables(mutable_config, mutable_mock_workspace_path, mock
             assert ";" + " " * 9 + "from_ws echo" in data
             assert "\n" + " " * 11 + "from_wl echo" in data
             assert "\n" + " " * 10 + "from_exp echo" in data
+            assert "\n" + " " * 2 + "test_from_ws mpirun -n 16 -ppn 16 test" in data
 
 
 def test_redefined_executable_errors(