Merge pull request #889 from GoogleCloudPlatform/v0.6.0

Merge release v0.6.0 into main

Author: douglasjacobsen

.coveragerc

@@ -28,7 +28,7 @@
 # - N814: camelcase imported as constant
 #
 [flake8]
-ignore = E129,E221,E241,E272,E731,W503,W504,F999,N801,N813,N814,F403,F405
+ignore = E129,E203,E221,E241,E272,E731,W503,W504,F999,N801,N813,N814,F403,F405
 max-line-length = 99
 
 per-file-ignores =

.flake8

@@ -8,7 +8,7 @@
 # dependents. So we add exceptions for checks related to undefined names.
 #
 # Note that we also add *per-line* exemptions for certain patterns in the
-# `ramble flake8` command.  This is where F403 for `from ramble import *`
+# `ramble style` command.  This is where F403 for `from ramble import *`
 # is added (because we *only* allow that wildcard).
 #
 # See .flake8 for regular exceptions.
@@ -20,5 +20,5 @@
 # - F821: undefined name `name`
 #
 [flake8]
-ignore = E129,E221,E241,E272,E731,E501,W503,W504,W605,F405,F821,F999,N801,N813,N814
+ignore = E129,E203,E221,E241,E272,E731,E501,W503,W504,W605,F405,F821,F999,N801,N813,N814
 max-line-length = 79

.flake8_applications

@@ -0,0 +1,43 @@
+# Workflow for deploying a single redirect page to GitHub Pages
+name: gh-pages deploy
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["develop"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Only deploy the redirect directory
+          path: './lib/ramble/docs/_gh_pages_redirect'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/gh-pages-deploy.yaml

@@ -1,12 +1,21 @@
 repos:
 - repo: local
   hooks:
-  - id: flake8
-    name: flake8
-    entry: share/ramble/qa/run-flake8-tests
+  - id: style
+    name: style
+    entry: share/ramble/qa/run-style-tests
     language: system
   - id: short-unit-tests
     name: short-unit-tests
     entry: share/ramble/qa/run-unit-tests
     language: system
-    stages: [push]
+    stages: [pre-push]
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.17.0
+  hooks:
+  - id: pyupgrade
+    name: python-old-syntax-check
+    args: [--py36-plus]
+    files: (lib/ramble/ramble/|var/ramble/repos/).*\.py$
+

.github/workflows/sphinx.yml

@@ -0,0 +1,14 @@
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: lib/ramble/docs/conf.py
+  fail_on_warning: false
+
+python:
+  install:
+    - requirements: lib/ramble/docs/requirements.txt

.pre-commit-config.yaml

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2022-2024 The Ramble Authors.
+   Copyright 2022-2025 The Ramble Authors.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

.readthedocs.yaml

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022-2023 Google LLC, Ramble Project Developers.
+Copyright (c) 2022-2025 Google LLC, Ramble Project Developers.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

LICENSE-APACHE

@@ -1,6 +1,7 @@
-Ramble is a multi-platform experimentation framework to increase exploration
-productivity and improve reproducibility. Ramble is capable of driving software
-installation, acquire input files, configure experiments, and extract results.
+Ramble is a multi-platform experimentation framework that increases exploration
+productivity and improves reproducibility. Ramble is capable of driving
+software installation, acquiring input files, configuring experiments, and
+extracting results.
 It works on Linux, macOS, and many supercomputers.
 
 Ramble can be used to configure a variety of experiments for applications.
@@ -14,9 +15,11 @@ Python, and Ramble’s dependencies are installed as per the dependency section
 below.
 Then:
 
-    $ git clone -c feature.manyFiles=true https://github.com/GoogleCloudPlatform/ramble.git
-    $ cd ramble/bin
-    $ ./ramble workspace create -d test_workspace -c ../examples/basic_hostname_config.yaml
+    git clone -c feature.manyFiles=true https://github.com/GoogleCloudPlatform/ramble.git
+    python3 -m venv ramble/env
+    . ramble/env/bin/activate
+    pip install -r ramble/requirements.txt
+    ./ramble/bin/ramble workspace create -d test_workspace -c ramble/examples/basic_hostname_config.yaml
 
 Dependencies
 ------------
@@ -25,22 +28,27 @@ Ramble’s python dependencies can be installed using the included requirements.
 
 e.g.
 
-    $ pip install -r requirements.txt
+    pip install -r requirements.txt
 
 We recommend Python >= 3.7 for Ramble, but a best effort attempt is made to
 support Python 3.6 as it is used by older operating systems such as Centos7.
 Specifically, you might need to update `pip` and downgrade `protobuf` when
 using Python 3.6.
 
-Outside of these requirements, ramble requires an existing installation of
-spack for some application definition. See
-[Spack’s documentation](https://github.com/spack/spack#-spack) to install Spack.
+Outside of these requirements, Ramble requires package managers to be
+configured if they will be used as part of the experiments Ramble creates.
+
+Although package manager support is not required to use Ramble, some
+experiments are more easily accomplished by allowing Ramble to drive the
+package manager execution. To allow this, package managers generally need to be
+installed indepednently from Ramble. For more information on this, see
+[Ramble's supported package managers](https://ramble.readthedocs.io/en/latest/package_managers.html).
 
 Documentation
 ----------------
 
 Ramble’s documentation can be viewed at
-[https://googlecloudplatform.github.io/ramble/](https://googlecloudplatform.github.io/ramble/).
+[https://ramble.readthedocs.io/](https://ramble.readthedocs.io/).
 
 For help with Ramble’s commands, run `ramble help` or `ramble help --all`.
 
@@ -63,6 +71,14 @@ Resources:
 
 Contributing
 ------------------------
+When developing features for Ramble, it can be helpful to install the
+development requirements instead of the user requirements:
+
+e.g.
+
+    pip install -r requirements-dev.txt
+
+
 Contributing to Ramble is relatively easy.  Just send us a
 [pull request](https://help.github.com/articles/using-pull-requests/).
 When you send your request, make ``develop`` the destination branch on the

LICENSE-MIT

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2022-2024 The Ramble Authors
+# Copyright 2022-2025 The Ramble Authors
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 # https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
@@ -15,8 +15,13 @@
 # See https://stackoverflow.com/a/47886254
 """:"
 # prefer python3, then python, then python2
-for cmd in python3 python python2; do
-   command -v > /dev/null $cmd && exec $cmd $0 "$@"
+## prefer RAMBLE_PYTHON environment variable, python3, python, then python2
+RAMBLE_PREFERRED_PYTHONS="python3 python python2 /usr/libexec/platform-python"
+for cmd in "${RAMBLE_PYTHON:-}" ${RAMBLE_PREFERRED_PYTHONS}; do
+    if command -v > /dev/null "$cmd"; then
+        export RAMBLE_PYTHON="$(command -v "$cmd")"
+        exec "${RAMBLE_PYTHON}" "$0" "$@"
+    fi
 done
 
 echo "==> Error: ramble could not find a python interpreter!" >&2
@@ -30,10 +35,9 @@ from __future__ import print_function
 import os
 import sys
 
-if sys.version_info[:2] < (2, 6):
+if sys.version_info[:2] < (3, 6):
     v_info = sys.version_info[:3]
-    sys.exit("Ramble requires Python 2.6 or higher."
-             "This is Python %d.%d.%d." % v_info)
+    sys.exit("Ramble requires Python 3.6 or higher." "This is Python %d.%d.%d." % v_info)
 
 # Find ramble's location and its prefix.
 ramble_file = os.path.realpath(os.path.expanduser(__file__))
@@ -46,9 +50,6 @@ sys.path.insert(0, ramble_lib_path)
 # Add external libs
 ramble_external_libs = os.path.join(ramble_lib_path, "external")
 
-if sys.version_info[:2] == (2, 6):
-    sys.path.insert(0, os.path.join(ramble_external_libs, 'py26'))
-
 sys.path.insert(0, ramble_external_libs)
 
 # Here we delete ruamel.yaml in case it has been already imported from site
@@ -57,11 +58,11 @@ sys.path.insert(0, ramble_external_libs)
 # Briefly: ruamel.yaml produces a .pth file when installed with pip that
 # makes the site installed package the preferred one, even though sys.path
 # is modified to point to another version of ruamel.yaml.
-if 'ruamel.yaml' in sys.modules:
-    del sys.modules['ruamel.yaml']
+if "ruamel.yaml" in sys.modules:
+    del sys.modules["ruamel.yaml"]
 
-if 'ruamel' in sys.modules:
-    del sys.modules['ruamel']
+if "ruamel" in sys.modules:
+    del sys.modules["ruamel"]
 
 import ramble.main  # noqa
 

README.md

@@ -1,4 +1,4 @@
-# Copyright 2022-2024 The Ramble Authors
+# Copyright 2022-2025 The Ramble Authors
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 # https://www.apache.org/licenses/LICENSE-2.0> or the MIT license

bin/ramble

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# This is the default ramble repository configuration. It includes the
+# builtin ramble base application repository.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-ramble-instance settings (overrides defaults):
+#   $RAMBLE_ROOT/etc/ramble/base_application_repos.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.ramble/base_application_repos.yaml
+# -------------------------------------------------------------------------
+base_application_repos:
+  - $ramble/var/ramble/repos/builtin

bin/ramble-python

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# This is the default ramble repository configuration. It includes the
+# builtin ramble base modifier repository.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-ramble-instance settings (overrides defaults):
+#   $RAMBLE_ROOT/etc/ramble/base_modifier_repos.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.ramble/base_modifier_repos.yaml
+# -------------------------------------------------------------------------
+base_modifier_repos:
+  - $ramble/var/ramble/repos/builtin

etc/ramble/defaults/base_application_repos.yaml

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# This is the default ramble repository configuration. It includes the
+# builtin ramble base package manager repository.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-ramble-instance settings (overrides defaults):
+#   $RAMBLE_ROOT/etc/ramble/base_package_manager_repos.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.ramble/base_package_manager_repos.yaml
+# -------------------------------------------------------------------------
+base_package_manager_repos:
+  - $ramble/var/ramble/repos/builtin

etc/ramble/defaults/base_modifier_repos.yaml

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# This is the default ramble repository configuration. It includes the
+# builtin ramble base workflow manager repository.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-ramble-instance settings (overrides defaults):
+#   $RAMBLE_ROOT/etc/ramble/base_workflow_manager_repos.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.ramble/base_workflow_manager_repos.yaml
+# -------------------------------------------------------------------------
+base_workflow_manager_repos:
+  - $ramble/var/ramble/repos/builtin

etc/ramble/defaults/base_package_manager_repos.yaml

@@ -19,9 +19,10 @@ config:
   verify_ssl: true
   connect_timeout: 10
   checksum: true
-  shell: sh
+  shell: bash
   input_cache: $ramble/var/ramble/cache
   workspace_dirs: $ramble/var/ramble/workspaces
+  report_dirs: ~/.ramble/reports
   include_phase_dependencies: false
   spack:
     install:
@@ -32,3 +33,5 @@ config:
       flags: ''
     env_create:
       flags: ''
+    env_view:
+      link_type: symlink

etc/ramble/defaults/base_workflow_manager_repos.yaml

@@ -1,3 +1,5 @@
 formatted_executables:
   command:
     join_separator: '\n'
+    commands:
+    - '{unformatted_command}'