Enable variable modifications to layer changes on the same variable

lib/ramble/ramble/application.py

@@ -1091,7 +1091,7 @@ def _define_commands(
 
             for mod in self._modifier_instances:
                 if mod.applies_to_executable(exec_node.key):
-                    exec_vars.update(mod.modded_variables(self))
+                    exec_vars.update(mod.modded_variables(self, exec_vars))
 
             if isinstance(exec_node.attribute, ramble.util.executable.CommandExecutable):
                 # Process directive defined executables
@@ -1427,7 +1427,7 @@ def _make_experiments(self, workspace, app_inst=None):
             exec_vars = {}
 
             for mod in self._modifier_instances:
-                exec_vars.update(mod.modded_variables(self))
+                exec_vars.update(mod.modded_variables(self, exec_vars))
 
             for template_name, template_conf in workspace.all_templates():
                 expand_path = os.path.join(experiment_run_dir, template_name)

lib/ramble/ramble/modifier.py

@@ -244,21 +244,25 @@ def format_doc(self, **kwargs):
             results.write((" " * indent) + line + "\n")
         return results.getvalue()
 
-    def modded_variables(self, app):
+    def modded_variables(self, app, extra_vars={}):
         mods = {}
 
         if self._usage_mode not in self.variable_modifications:
             return mods
 
         for var, var_mod in self.variable_modifications[self._usage_mode].items():
             if var_mod["method"] in ["append", "prepend"]:
-                # var_str = app.expander.expansion_str(var)
-                # prev_val = app.expander.expand_var(var_str)
-                prev_val = app.variables[var]
+                if var in extra_vars:
+                    prev_val = extra_vars[var]
+                elif var in app.variables:
+                    prev_val = app.variables[var]
+                else:
+                    prev_val = ""
+
                 if var_mod["method"] == "append":
-                    mods[var] = f'{prev_val} {var_mod["modification"]}'
+                    mods[var] = f'{prev_val}{var_mod["modification"]}'
                 else:  # method == prepend
-                    mods[var] = f'{var_mod["modification"]} {prev_val}'
+                    mods[var] = f'{var_mod["modification"]}{prev_val}'
             else:  # method == set
                 mods[var] = var_mod["modification"]
 

lib/ramble/ramble/test/modifier_functionality/mock_layered_modifications.py

@@ -0,0 +1,58 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+
+from ramble.test.dry_run_helpers import dry_run_config, SCOPES
+import ramble.test.modifier_functionality.modifier_helpers as modifier_helpers
+
+import ramble.workspace
+from ramble.main import RambleCommand
+
+workspace = RambleCommand("workspace")
+
+
+def test_layered_variable_modifications(
+    mutable_mock_workspace_path, mutable_applications, mock_modifiers
+):
+    workspace_name = "test_gromacs_dry_run_mock_spack_mod"
+
+    test_modifiers = [
+        (SCOPES.experiment, modifier_helpers.named_modifier("test-mod")),
+        (SCOPES.experiment, modifier_helpers.named_modifier("test-mod-2")),
+    ]
+
+    test_template = """
+{test_var_mod}
+"""
+
+    with ramble.workspace.create(workspace_name) as ws1:
+        ws1.write()
+
+        config_path = os.path.join(ws1.config_dir, ramble.workspace.config_file_name)
+        template_path = os.path.join(ws1.config_dir, "test.tpl")
+
+        with open(template_path, "w+") as f:
+            f.write(test_template)
+
+        dry_run_config("modifiers", test_modifiers, config_path, "gromacs", "water_bare")
+
+        ws1._re_read()
+
+        workspace("concretize", global_args=["-D", ws1.root])
+        workspace("setup", "--dry-run", global_args=["-D", ws1.root])
+
+        rendered_template = os.path.join(
+            ws1.experiment_dir, "gromacs", "water_bare", "test_exp", "test"
+        )
+        assert os.path.exists(rendered_template)
+
+        with open(rendered_template) as f:
+            data = f.read()
+            assert "test-mod-2-append" in data
+            assert "test-mod-append" in data

var/ramble/repos/builtin.mock/modifiers/test-mod-2/modifier.py

@@ -0,0 +1,31 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+from ramble.modkit import *  # noqa: F403
+
+
+class TestMod2(BasicModifier):
+    """Define a test modifier
+
+    This modifier contains a variable modification that can be layered on top of
+    the modifications defined in test-mod
+    """
+
+    name = "test-mod-2"
+
+    tags("test")
+
+    mode("test", description="This is a test mode")
+    default_mode("test")
+
+    variable_modification(
+        "test_var_mod",
+        " test-mod-2-append",
+        method="append",
+        modes=["test"],
+    )

var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py

@@ -32,6 +32,13 @@ class TestMod(BasicModifier):
         "exp-scope", description="This is a test mode at the experiment scope"
     )
 
+    variable_modification(
+        "test_var_mod",
+        " test-mod-append",
+        method="append",
+        modes=["test"],
+    )
+
     variable_modification(
         "mpi_command",
         'echo "prefix_mpi_command" >> {log_file}; ',

var/ramble/repos/builtin/modifiers/intel-aps/modifier.py

@@ -36,7 +36,7 @@ class IntelAps(BasicModifier):
         "aps_flags", "-c mpi -r {aps_log_dir}", method="set", modes=["mpi"]
     )
     variable_modification(
-        "mpi_command", "aps {aps_flags}", method="append", modes=["mpi"]
+        "mpi_command", "aps {aps_flags} ", method="append", modes=["mpi"]
     )
 
     archive_pattern("aps_*_results_dir/*")