Merge pull request #953 from douglasjacobsen/env-vars-in-exec

Add env-var passthrough option

Author: douglasjacobsen

etc/ramble/defaults/config.yaml

@@ -24,6 +24,7 @@ config:
   workspace_dirs: $ramble/var/ramble/workspaces
   report_dirs: ~/.ramble/reports
   include_phase_dependencies: false
+  resolve_variables_in_subprocesses: false
   spack:
     install:
       flags: --reuse

lib/ramble/ramble/main.py

@@ -441,6 +441,12 @@ def make_argument_parser(**kwargs):
         help="when running in a workspace, use its application repository",
     )
 
+    parser.add_argument(
+        "--resolve-variables-in-subprocesses",
+        action="store_true",
+        help="Allow resolution of environment variables when launching subprocesses",
+    )
+
     parser.add_argument(
         "-k",
         "--insecure",
@@ -612,6 +618,10 @@ def setup_main_options(args):
         logger.warn("You asked for --insecure. Will NOT check SSL certificates.")
         ramble.config.set("config:verify_ssl", False, scope="command_line")
 
+    # If the user asked for it allow env-vars to resolve in subprocess calls
+    if args.resolve_variables_in_subprocesses:
+        ramble.config.set("config:resolve_variables_in_subprocesses", True, scope="command_line")
+
     # Use the ramble config command to handle parsing the config strings
     for config_var in args.config_vars or []:
         ramble.config.add(fullpath=config_var, scope="command_line")

lib/ramble/ramble/pipeline.py

@@ -555,6 +555,8 @@ def __init__(
     def _execute(self):
         super()._execute()
 
+        resolve_env_vars = ramble.config.get("config:resolve_variables_in_subprocesses", False)
+
         if not self.suppress_run_header:
             logger.all_msg("Running executors...")
 
@@ -568,6 +570,8 @@ def _execute(self):
 
             app_inst.add_expand_vars(self.workspace)
             exec_str = app_inst.expander.expand_var(self.executor)
+            if resolve_env_vars:
+                exec_str = os.path.expandvars(exec_str)
             exec_parts = shlex.split(exec_str)
             exec_name = exec_parts[0]
             exec_args = exec_parts[1:]

lib/ramble/ramble/schema/config.py

@@ -116,6 +116,11 @@
             },
             "additionalProperties": False,
         },
+        "resolve_variables_in_subprocesses": {
+            "type": "boolean",
+            "default": False,
+            "additionalProperties": False,
+        },
     },
     "additionalProperties": False,
 }

share/ramble/ramble-completion.bash

@@ -264,7 +264,7 @@ complete -o bashdefault -o default -F _bash_completion_ramble ramble
 _ramble() {
     if $list_options
     then
-        RAMBLE_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --disable-passthrough -N --disable-logger -P --disable-progress-bar --timestamp --pdb -w --workspace -D --workspace-dir -W --no-workspace --use-workspace-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock --mock-applications --mock-modifiers --mock-package-managers --mock-workflow-managers --mock-base-applications --mock-base-modifiers --mock-base-package-managers --mock-base-workflow-managers -p --profile --sorted-profile --lines --profile-restrictions -v --verbose --stacktrace -V --version --print-shell-vars"
+        RAMBLE_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --disable-passthrough -N --disable-logger -P --disable-progress-bar --timestamp --pdb -w --workspace -D --workspace-dir -W --no-workspace --use-workspace-repo --resolve-variables-in-subprocesses -k --insecure -l --enable-locks -L --disable-locks -m --mock --mock-applications --mock-modifiers --mock-package-managers --mock-workflow-managers --mock-base-applications --mock-base-modifiers --mock-base-package-managers --mock-base-workflow-managers -p --profile --sorted-profile --lines --profile-restrictions -v --verbose --stacktrace -V --version --print-shell-vars"
     else
         RAMBLE_COMPREPLY="attributes clean commands config debug deployment docs edit help info license list mirror on python repo results software-definitions style unit-test workspace"
     fi