parallel study evaluation

.github/workflows/pypi.yml

@@ -2,7 +2,11 @@ name: Build and Publish
 # based on official doc
 # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
 
-on: [push, workflow_dispatch]
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
 
 jobs:
     build:
@@ -31,7 +35,6 @@ jobs:
 
     publish-to-pypi:
       name: Publish to PyPI
-      if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes # tweak so it doesnt publish all tags
       needs:
         - build
       runs-on: ubuntu-22.04
@@ -40,6 +43,24 @@ jobs:
         id-token: write  # IMPORTANT: mandatory for trusted publishing
 
       steps:
+        - name: Extract tag name
+          run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+        - name: Validate tag format
+          id: validate_tag
+          run: |
+            if [[ "${{ env.TAG_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+)?$ ]]; then
+              echo "valid=true" >> $GITHUB_ENV
+            else
+              echo "valid=false" >> $GITHUB_ENV
+            fi
+
+        - name: Exit if invalid tag
+          if: env.valid == 'false'
+          run: |
+            echo "The tag ${{ env.TAG_NAME }} is not a valid semantic version. Exiting."
+            exit 1
+
         - name: Download all the distribution packages
           uses: actions/download-artifact@v4
           with:

.github/workflows/readthedocs.yml

@@ -0,0 +1,48 @@
+name: Update Docs with Versioning
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  build-docs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install sphinx
+
+    - name: Run Sphinx-apidoc
+      run: |
+        cd docs
+        sphinx-apidoc ../src -o ./source
+
+    - name: Commit and Push to Generated Docs Branch
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git checkout -b generated-docs
+        git add .
+        git commit -m "Update docs for version $GITHUB_REF_NAME [skip ci]" || echo "No changes to commit"
+        git push origin generated-docs --force
+        git tag docs/$GITHUB_REF_NAME
+        git push origin tag docs/$GITHUB_REF_NAME

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/source/requirements.txt

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -0,0 +1,75 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+import os
+import subprocess
+
+
+# Automatically retrieve the project version from Git
+def get_version():
+    try:
+        return subprocess.check_output(["git", "describe", "--tags"], encoding="utf-8").strip()
+    except Exception:
+        return "0.0.0"
+
+
+project = "AgentLab"
+author = "ServiceNow"
+release = get_version()  # Full version string including tags
+version = release  # Short version (e.g., 1.0)
+
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+extensions = [
+    "sphinx.ext.duration",
+    "sphinx.ext.doctest",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.intersphinx",
+    # "myst_parser",  # Add this to enable Markdown parsing
+    "sphinx.ext.napoleon",
+]
+
+intersphinx_mapping = {
+    "rtd": ("https://docs.readthedocs.io/en/stable/", None),
+    "python": ("https://docs.python.org/3/", None),
+    "sphinx": ("https://www.sphinx-doc.org/en/master/", None),
+}
+intersphinx_disabled_domains = ["std"]
+
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": True,
+    "show-inheritance": True,
+}
+
+source_suffix = {
+    ".rst": "restructuredtext",
+}
+
+
+templates_path = ["_templates"]
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "sphinx_rtd_theme"
+html_theme_options = {
+    "navigation_depth": -1,
+    "collapse_navigation": False,
+    "display_version": True,
+    "version_selector": True,
+}
+html_static_path = ["_static"]
+
+import sys
+
+sys.path.insert(0, os.path.abspath("../../src"))

docs/source/index.rst

@@ -0,0 +1,20 @@
+.. AgentLab documentation master file, created by
+   sphinx-quickstart on Tue Nov 26 11:21:39 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+AgentLab documentation
+======================
+
+ReadTheDoc Page for AgentLab: https://agentlab.readthedocs.io/en/latest/
+
+
+.. autosummary::
+   :toctree:
+   :recursive:
+
+    agentlab.agents
+    agentlab.analyze
+    agentlab.llm
+    agentlab.experiments
+    agentlab.ui_assistant

docs/source/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/requirements.txt

@@ -0,0 +1,2 @@
+agentlab
+sphinx-rtd-theme

src/agentlab/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.2.dev1"
+__version__ = "v0.3.2"

src/agentlab/agents/__init__.py

@@ -0,0 +1,13 @@
+"""
+AgentLab's pre-implemented agents.
+
+This module contains the agent implementations for AgentLab. With currently:
+
+- GenericAgent: Our baseline agent for evaluation
+
+- MostBasicAgent: A basic agent for learning our framework
+
+- TapeAgent: An agent that uses the Tape data structure to perform actions
+
+- VisualWebArenaAgent: An implentation of the agent used in WebArena and VisualWebArena
+"""

src/agentlab/agents/generic_agent/__init__.py

@@ -1,9 +1,17 @@
+"""
+Baseline agent for all ServiceNow papers
+
+This module contains the GenericAgent class, which is the baseline agent for all ServiceNow papers. \
+It is a simple agent that can be ran OOB on all BrowserGym environments. It is also shipped with \
+a few configurations that can be used to run it on different environments.
+"""
+
 from .agent_configs import (
     AGENT_3_5,
     AGENT_8B,
+    AGENT_CUSTOM,
     AGENT_LLAMA3_70B,
     AGENT_LLAMA31_70B,
-    AGENT_CUSTOM,
     RANDOM_SEARCH_AGENT,
     AGENT_4o,
     AGENT_4o_MINI,

src/agentlab/agents/generic_agent/agent_configs.py

@@ -1,3 +1,7 @@
+"""
+Basic flags and agent configurations for generic agents.
+"""
+
 import bgym
 
 from agentlab.agents import dynamic_prompting as dp