Clean makefile, pyproject.toml and CI (#229)

* Clean makefile / pyproject.toml / .github

* new tests after

* add back sqlalchemy

* disable docs tests in CI

* continue on error

* correct continue on error

* Remove all_docs test

Author: Wauplin

.github/workflows/quality.yml

@@ -16,20 +16,15 @@ jobs:
           python-version: "3.12"
 
       # Setup venv
-      # TODO: revisit when https://github.com/astral-sh/uv/issues/1526 is addressed.
       - name: Setup venv + uv
         run: |
           pip install --upgrade uv
           uv venv
 
       - name: Install dependencies
-        run: uv pip install "smolagents[test] @ ."
-      - run: uv run ruff check tests src # linter
-      - run: uv run ruff format --check tests src # formatter
+        run: uv pip install "smolagents[quality] @ ."
 
-      # Run type checking at least on smolagents root file to check all modules
-      # that can be lazy-loaded actually exist.
-      # - run: uv run mypy src/smolagents/__init__.py --follow-imports=silent --show-traceback
-
-      # Run mypy on full package
-      # - run: uv run mypy src
+      # Equivalent of "make quality" but step by step
+      - run: uv run ruff check examples src tests utils # linter
+      - run: uv run ruff format --check examples src tests utils # formatter
+      - run: uv run python utils/check_tests_in_ci.py

.github/workflows/tests.yml

@@ -20,9 +20,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-
       # Setup venv
-      # TODO: revisit when https://github.com/astral-sh/uv/issues/1526 is addressed.
       - name: Setup venv + uv
         run: |
           pip install --upgrade uv
@@ -33,33 +31,59 @@ jobs:
         run: |
           uv pip install "smolagents[test] @ ."
 
+      # Run all tests separately for individual feedback
+      # Use 'if success() || failure()' so that all tests are run even if one failed
+      # See https://stackoverflow.com/a/62112985
       - name: Agent tests
         run: |
-          uv run pytest -sv ./tests/test_agents.py
+          uv run pytest ./tests/test_agents.py
+        if: ${{ success() || failure() }}
+
       - name: Default tools tests
         run: |
-          uv run pytest -sv ./tests/test_default_tools.py
+          uv run pytest ./tests/test_default_tools.py
+        if: ${{ success() || failure() }}
+
+      # - name: Docs tests # Disabled for now (slow test + requires API keys)
+      #   run: |
+      #     uv run pytest ./tests/test_all_docs.py
+
       - name: Final answer tests
         run: |
-          uv run pytest -sv ./tests/test_final_answer.py
+          uv run pytest ./tests/test_final_answer.py
+        if: ${{ success() || failure() }}
+
       - name: Models tests
         run: |
-          uv run pytest -sv ./tests/test_models.py
+          uv run pytest ./tests/test_models.py
+        if: ${{ success() || failure() }}
+
       - name: Monitoring tests
         run: |
-          uv run pytest -sv ./tests/test_monitoring.py
+          uv run pytest ./tests/test_monitoring.py
+        if: ${{ success() || failure() }}
+
       - name: Python interpreter tests
         run: |
-          uv run pytest -sv ./tests/test_python_interpreter.py
+          uv run pytest ./tests/test_python_interpreter.py
+        if: ${{ success() || failure() }}
+
       - name: Search tests
         run: |
-          uv run pytest -sv ./tests/test_search.py
+          uv run pytest ./tests/test_search.py
+        if: ${{ success() || failure() }}
+
       - name: Tools tests
         run: |
-          uv run pytest -sv ./tests/test_tools.py
+          uv run pytest ./tests/test_tools.py
+        if: ${{ success() || failure() }}
+
       - name: Types tests
         run: |
-          uv run pytest -sv ./tests/test_types.py
+          uv run pytest ./tests/test_types.py
+        if: ${{ success() || failure() }}
+
       - name: Utils tests
         run: |
-          uv run pytest -sv ./tests/test_utils.py
+          uv run pytest ./tests/test_utils.py
+        if: ${{ success() || failure() }}

CONTRIBUTING.md

@@ -91,7 +91,7 @@ happy to make the changes or help you make a contribution if you're interested!
 
 ## I want to become a maintainer of the project. How do I get there?
 
-smolagents is a project led and managed by Hugging Face. We are  more than 
+smolagents is a project led and managed by Hugging Face. We are more than
 happy to have motivated individuals from other organizations join us as maintainers with the goal of helping smolagents
 make a dent in the world of Agents.
 

Makefile

@@ -1,53 +1,18 @@
 .PHONY: quality style test docs utils
 
-check_dirs := .
+check_dirs := examples src tests utils
 
-# Check that source code meets quality standards
-
-extra_quality_checks:
-	python utils/check_copies.py
-	python utils/check_dummies.py
-	python utils/check_repo.py
-	doc-builder style smolagents docs/source --max_len 119
-
-# this target runs checks on all files
+# Check code quality of the source code
 quality:
 	ruff check $(check_dirs)
 	ruff format --check $(check_dirs)
-	doc-builder style smolagents docs/source --max_len 119 --check_only
+	python utils/check_tests_in_ci.py
 
-# Format source code automatically and check is there are any problems left that need manual fixing
+# Format source code automatically
 style:
 	ruff check $(check_dirs) --fix
 	ruff format $(check_dirs)
-	doc-builder style smolagents docs/source --max_len 119
 
-# Run tests for the library
-test_big_modeling:
-	python -m pytest -s -v ./tests/test_big_modeling.py ./tests/test_modeling_utils.py $(if $(IS_GITHUB_CI),--report-log "$(PYTORCH_VERSION)_big_modeling.log",)
-
-test_core:
-	python -m pytest -s -v ./tests/ --ignore=./tests/test_examples.py $(if $(IS_GITHUB_CI),--report-log "$(PYTORCH_VERSION)_core.log",)
-
-test_cli:
-	python -m pytest -s -v ./tests/test_cli.py $(if $(IS_GITHUB_CI),--report-log "$(PYTORCH_VERSION)_cli.log",)
-
-
-# Since the new version of pytest will *change* how things are collected, we need `deepspeed` to 
-# run after test_core and test_cli
+# Run smolagents tests
 test:
-	$(MAKE) test_core
-	$(MAKE) test_cli
-	$(MAKE) test_big_modeling
-	$(MAKE) test_deepspeed
-	$(MAKE) test_fsdp
-
-test_examples:
-	python -m pytest -s -v ./tests/test_examples.py $(if $(IS_GITHUB_CI),--report-log "$(PYTORCH_VERSION)_examples.log",)
-
-# Same as test but used to install only the base dependencies
-test_prod:
-	$(MAKE) test_core
-
-test_rest:
-	python -m pytest -s -v ./tests/test_examples.py::FeatureExamplesTests $(if $(IS_GITHUB_CI),--report-log "$(PYTORCH_VERSION)_rest.log",)
+	pytest ./tests/

README.md

@@ -98,9 +98,27 @@ To contribute, follow our [contribution guide](https://github.com/huggingface/sm
 At any moment, feel welcome to open an issue, citing your exact error traces and package versions if it's a bug.
 It's often even better to open a PR with your proposed fixes/changes!
 
+To install dev dependencies, run:
+```
+pip install -e ".[dev]"
+```
+
+When making changes to the codebase, please check that it follows the repo's code quality requirements by running:
+To check code quality of the source code:
+```
+make quality
+```
+
+If the checks fail, you can run the formatter with:
+```
+make style
+```
+
+And commit the changes.
+
 To run tests locally, run this command:
 ```bash
-pytest -sv .
+pytest .
 ```
 
 ## Citing smolagents

examples/benchmark.ipynb

@@ -254,7 +254,10 @@
     "            if is_vanilla_llm:\n",
     "                llm = agent\n",
     "                answer = str(llm([{\"role\": \"user\", \"content\": question}]).content)\n",
-    "                token_count = {\"input\": llm.last_input_token_count, \"output\": llm.last_output_token_count}\n",
+    "                token_count = {\n",
+    "                    \"input\": llm.last_input_token_count,\n",
+    "                    \"output\": llm.last_output_token_count,\n",
+    "                }\n",
     "                intermediate_steps = str([])\n",
     "            else:\n",
     "                answer = str(agent.run(question))\n",
@@ -983,7 +986,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -1043,8 +1046,8 @@
     "\n",
     "\n",
     "# Usage (after running your previous data processing code):\n",
-    "mathjax_table = create_mathjax_table(pivot_df, formatted_df)\n",
-    "print(mathjax_table)"
+    "# mathjax_table = create_mathjax_table(pivot_df, formatted_df)\n",
+    "# print(mathjax_table)"
    ]
   }
  ],

examples/e2b_example.py

@@ -4,8 +4,9 @@
 
 load_dotenv()
 
+
 class GetCatImageTool(Tool):
-    name="get_cat_image"
+    name = "get_cat_image"
     description = "Get a cat image"
     inputs = {}
     output_type = "image"
@@ -27,17 +28,22 @@ def forward(self):
 get_cat_image = GetCatImageTool()
 
 agent = CodeAgent(
-    tools = [get_cat_image, VisitWebpageTool()],
+    tools=[get_cat_image, VisitWebpageTool()],
     model=HfApiModel(),
-    additional_authorized_imports=["Pillow", "requests", "markdownify"], # "duckduckgo-search", 
-    use_e2b_executor=True
+    additional_authorized_imports=[
+        "Pillow",
+        "requests",
+        "markdownify",
+    ],  # "duckduckgo-search",
+    use_e2b_executor=True,
 )
 
 agent.run(
-    "Return me an image of a cat. Directly use the image provided in your state.", additional_args={"cat_image":get_cat_image()}
-) # Asking to directly return the image from state tests that additional_args are properly sent to server.
+    "Return me an image of a cat. Directly use the image provided in your state.",
+    additional_args={"cat_image": get_cat_image()},
+)  # Asking to directly return the image from state tests that additional_args are properly sent to server.
 
 # Try the agent in a Gradio UI
 from smolagents import GradioUI
 
-GradioUI(agent).launch()
+GradioUI(agent).launch()

examples/gradio_upload.py

@@ -1,11 +1,5 @@
-from smolagents import (
-    CodeAgent,
-    HfApiModel,
-    GradioUI
-)
+from smolagents import CodeAgent, HfApiModel, GradioUI
 
-agent = CodeAgent(
-    tools=[], model=HfApiModel(), max_steps=4, verbosity_level=1
-)
+agent = CodeAgent(tools=[], model=HfApiModel(), max_steps=4, verbosity_level=1)
 
-GradioUI(agent, file_upload_folder='./data').launch()
+GradioUI(agent, file_upload_folder="./data").launch()

examples/inspect_runs.py

@@ -16,7 +16,9 @@
 # Let's setup the instrumentation first
 
 trace_provider = TracerProvider()
-trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter("http://0.0.0.0:6006/v1/traces")))
+trace_provider.add_span_processor(
+    SimpleSpanProcessor(OTLPSpanExporter("http://0.0.0.0:6006/v1/traces"))
+)
 
 SmolagentsInstrumentor().instrument(tracer_provider=trace_provider, skip_dep_check=True)
 

examples/rag.py

@@ -8,7 +8,9 @@
 
 
 knowledge_base = datasets.load_dataset("m-ric/huggingface_doc", split="train")
-knowledge_base = knowledge_base.filter(lambda row: row["source"].startswith("huggingface/transformers"))
+knowledge_base = knowledge_base.filter(
+    lambda row: row["source"].startswith("huggingface/transformers")
+)
 
 source_docs = [
     Document(page_content=doc["text"], metadata={"source": doc["source"].split("/")[1]})
@@ -26,6 +28,7 @@
 
 from smolagents import Tool
 
+
 class RetrieverTool(Tool):
     name = "retriever"
     description = "Uses semantic search to retrieve the parts of transformers documentation that could be most relevant to answer your query."
@@ -39,9 +42,7 @@ class RetrieverTool(Tool):
 
     def __init__(self, docs, **kwargs):
         super().__init__(**kwargs)
-        self.retriever = BM25Retriever.from_documents(
-            docs, k=10
-        )
+        self.retriever = BM25Retriever.from_documents(docs, k=10)
 
     def forward(self, query: str) -> str:
         assert isinstance(query, str), "Your search query must be a string"
@@ -56,14 +57,20 @@ def forward(self, query: str) -> str:
             ]
         )
 
+
 from smolagents import HfApiModel, CodeAgent
 
 retriever_tool = RetrieverTool(docs_processed)
 agent = CodeAgent(
-    tools=[retriever_tool], model=HfApiModel("meta-llama/Llama-3.3-70B-Instruct"), max_steps=4, verbosity_level=2
+    tools=[retriever_tool],
+    model=HfApiModel("meta-llama/Llama-3.3-70B-Instruct"),
+    max_steps=4,
+    verbosity_level=2,
 )
 
-agent_output = agent.run("For a transformers model training, which is slower, the forward or the backward pass?")
+agent_output = agent.run(
+    "For a transformers model training, which is slower, the forward or the backward pass?"
+)
 
 print("Final output:")
 print(agent_output)