Add a test-sources configuration option. (#2062)

* Add test_sources configuration option.

* Add pyodide implementation.

* Modify default test behavior to run in the project directory.

* Update the test for running in the project directory.

* Apply suggestions from code review

Co-authored-by: Joe Rickerby <[email protected]>

* Add shlex quoting to test-sources.

* Restructure ctypes example to avoid issues running from the project folder.

---------

Co-authored-by: Joe Rickerby <[email protected]>

Author: freakboy3742

README.md

@@ -143,6 +143,7 @@ Options
 |   | [`CIBW_DEPENDENCY_VERSIONS`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions)  | Specify how cibuildwheel controls the versions of the tools it uses |
 | **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.pypa.io/en/stable/options/#test-command)  | Execute a shell command to test each built wheel |
 |   | [`CIBW_BEFORE_TEST`](https://cibuildwheel.pypa.io/en/stable/options/#before-test)  | Execute a shell command before testing each wheel |
+|   | [`CIBW_TEST_SOURCES`](https://cibuildwheel.pypa.io/en/stable/options/#test-sources)  | Files and folders from the source tree that are copied into an isolated tree before running the tests |
 |   | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.pypa.io/en/stable/options/#test-requires)  | Install Python dependencies before running the tests |
 |   | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.pypa.io/en/stable/options/#test-extras)  | Install your wheel for testing using extras_require |
 |   | [`CIBW_TEST_SKIP`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip)  | Skip running tests on some builds |

cibuildwheel/linux.py

@@ -21,12 +21,12 @@
 from .util import (
     BuildFrontendConfig,
     BuildSelector,
+    copy_test_sources,
     find_compatible_wheel,
     get_build_verbosity_extra_flags,
     prepare_command,
     read_python_configs,
     split_config_settings,
-    test_fail_cwd_file,
     unwrap,
 )
 
@@ -401,9 +401,19 @@ def build_in_container(
                 package=container_package_dir,
                 wheel=wheel_to_test,
             )
-            test_cwd = testing_temp_dir / "test_cwd"
-            container.call(["mkdir", "-p", test_cwd])
-            container.copy_into(test_fail_cwd_file, test_cwd / "test_fail.py")
+
+            if build_options.test_sources:
+                test_cwd = testing_temp_dir / "test_cwd"
+                container.call(["mkdir", "-p", test_cwd])
+                copy_test_sources(
+                    build_options.test_sources,
+                    build_options.package_dir,
+                    test_cwd,
+                    copy_into=container.copy_into,
+                )
+            else:
+                # There are no test sources. Run the tests in the project directory.
+                test_cwd = PurePosixPath(container_project_path)
 
             container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env)
 

cibuildwheel/macos.py

@@ -31,6 +31,7 @@
     BuildSelector,
     call,
     combine_constraints,
+    copy_test_sources,
     detect_ci_provider,
     download,
     find_compatible_wheel,
@@ -44,7 +45,6 @@
     read_python_configs,
     shell,
     split_config_settings,
-    test_fail_cwd_file,
     unwrap,
     virtualenv,
 )
@@ -736,9 +736,17 @@ def build(options: Options, tmp_path: Path) -> None:
                         wheel=repaired_wheel,
                     )
 
-                    test_cwd = identifier_tmp_dir / "test_cwd"
-                    test_cwd.mkdir(exist_ok=True)
-                    (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text())
+                    if build_options.test_sources:
+                        test_cwd = identifier_tmp_dir / "test_cwd"
+                        test_cwd.mkdir(exist_ok=True)
+                        copy_test_sources(
+                            build_options.test_sources,
+                            build_options.package_dir,
+                            test_cwd,
+                        )
+                    else:
+                        # There are no test sources. Run the tests in the project directory.
+                        test_cwd = Path(".").resolve()
 
                     shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env)
 

cibuildwheel/options.py

@@ -9,7 +9,7 @@
 import functools
 import shlex
 import textwrap
-from collections.abc import Generator, Iterable, Set
+from collections.abc import Callable, Generator, Iterable, Set
 from pathlib import Path
 from typing import Any, Literal, Mapping, Sequence, Union  # noqa: TID251
 
@@ -92,6 +92,7 @@ class BuildOptions:
     dependency_constraints: DependencyConstraints | None
     test_command: str | None
     before_test: str | None
+    test_sources: list[str]
     test_requires: list[str]
     test_extras: str
     test_groups: list[str]
@@ -171,11 +172,12 @@ class ListFormat(OptionFormat):
     A format that joins lists with a separator.
     """
 
-    def __init__(self, sep: str) -> None:
+    def __init__(self, sep: str, quote: Callable[[str], str] | None = None) -> None:
         self.sep = sep
+        self.quote = quote if quote else lambda s: s
 
     def format_list(self, value: SettingList) -> str:
-        return self.sep.join(str(v) for v in value)
+        return self.sep.join(self.quote(str(v)) for v in value)
 
     def merge_values(self, before: str, after: str) -> str:
         return f"{before}{self.sep}{after}"
@@ -711,6 +713,11 @@ def build_options(self, identifier: str | None) -> BuildOptions:
             dependency_versions = self.reader.get("dependency-versions")
             test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && "))
             before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && "))
+            test_sources = shlex.split(
+                self.reader.get(
+                    "test-sources", option_format=ListFormat(sep=" ", quote=shlex.quote)
+                )
+            )
             test_requires = self.reader.get(
                 "test-requires", option_format=ListFormat(sep=" ")
             ).split()
@@ -819,6 +826,7 @@ def build_options(self, identifier: str | None) -> BuildOptions:
             return BuildOptions(
                 globals=self.globals,
                 test_command=test_command,
+                test_sources=test_sources,
                 test_requires=[*test_requires, *test_requirements_from_groups],
                 test_extras=test_extras,
                 test_groups=test_groups,

cibuildwheel/pyodide.py

@@ -21,6 +21,7 @@
     BuildSelector,
     call,
     combine_constraints,
+    copy_test_sources,
     download,
     ensure_node,
     extract_zip,
@@ -31,7 +32,6 @@
     read_python_configs,
     shell,
     split_config_settings,
-    test_fail_cwd_file,
     virtualenv,
 )
 
@@ -387,9 +387,17 @@ def build(options: Options, tmp_path: Path) -> None:
                     package=build_options.package_dir.resolve(),
                 )
 
-                test_cwd = identifier_tmp_dir / "test_cwd"
-                test_cwd.mkdir(exist_ok=True)
-                (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text())
+                if build_options.test_sources:
+                    test_cwd = identifier_tmp_dir / "test_cwd"
+                    test_cwd.mkdir(exist_ok=True)
+                    copy_test_sources(
+                        build_options.test_sources,
+                        build_options.package_dir,
+                        test_cwd,
+                    )
+                else:
+                    # There are no test sources. Run the tests in the project directory.
+                    test_cwd = Path(".").resolve()
 
                 shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env)
 

cibuildwheel/resources/cibuildwheel.schema.json

@@ -426,6 +426,21 @@
       ],
       "title": "CIBW_TEST_EXTRAS"
     },
+    "test-sources": {
+      "description": "Test files that are required by the test environment",
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      ],
+      "title": "CIBW_TEST_SOURCES"
+    },
     "test-groups": {
       "description": "Install extra groups when testing",
       "oneOf": [
@@ -529,6 +544,9 @@
               "test-extras": {
                 "$ref": "#/$defs/inherit"
               },
+              "test-sources": {
+                "$ref": "#/$defs/inherit"
+              },
               "test-requires": {
                 "$ref": "#/$defs/inherit"
               }
@@ -618,6 +636,9 @@
           "test-extras": {
             "$ref": "#/properties/test-extras"
           },
+          "test-sources": {
+            "$ref": "#/properties/test-sources"
+          },
           "test-groups": {
             "$ref": "#/properties/test-groups"
           },
@@ -728,6 +749,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-sources": {
+          "$ref": "#/properties/test-sources"
+        },
         "test-groups": {
           "$ref": "#/properties/test-groups"
         },
@@ -776,6 +800,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-sources": {
+          "$ref": "#/properties/test-sources"
+        },
         "test-groups": {
           "$ref": "#/properties/test-groups"
         },
@@ -837,6 +864,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-sources": {
+          "$ref": "#/properties/test-sources"
+        },
         "test-groups": {
           "$ref": "#/properties/test-groups"
         },
@@ -885,6 +915,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-sources": {
+          "$ref": "#/properties/test-sources"
+        },
         "test-groups": {
           "$ref": "#/properties/test-groups"
         },

cibuildwheel/resources/defaults.toml

@@ -19,6 +19,7 @@ repair-wheel-command = ""
 
 test-command = ""
 before-test = ""
+test-sources = []
 test-requires = []
 test-extras = []
 test-groups = []

cibuildwheel/resources/testing_temp_dir_file.py

@@ -17,7 +17,7 @@
 import typing
 import urllib.request
 from collections import defaultdict
-from collections.abc import Generator, Iterable, Mapping, MutableMapping, Sequence
+from collections.abc import Callable, Generator, Iterable, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
 from enum import Enum
 from functools import lru_cache, total_ordering
@@ -36,6 +36,7 @@
 from packaging.version import Version
 from platformdirs import user_cache_path
 
+from . import errors
 from ._compat import tomllib
 from .architecture import Architecture
 from .errors import FatalError
@@ -66,8 +67,6 @@
 
 free_thread_enable_313: Final[Path] = resources_dir / "free-threaded-enable-313.xml"
 
-test_fail_cwd_file: Final[Path] = resources_dir / "testing_temp_dir_file.py"
-
 
 class EnableGroups(enum.Enum):
     """
@@ -425,6 +424,42 @@ def move_file(src_file: Path, dst_file: Path) -> Path:
     return Path(resulting_file).resolve(strict=True)
 
 
+def copy_into_local(src: Path, dst: PurePath) -> None:
+    """Copy a path from src to dst, regardless of whether it's a file or a directory."""
+    # Ensure the target folder location exists
+    Path(dst.parent).mkdir(exist_ok=True, parents=True)
+
+    if src.is_dir():
+        shutil.copytree(src, dst)
+    else:
+        shutil.copy(src, dst)
+
+
+def copy_test_sources(
+    test_sources: list[str],
+    package_dir: Path,
+    test_dir: PurePath,
+    copy_into: Callable[[Path, PurePath], None] = copy_into_local,
+) -> None:
+    """Copy the list of test sources from the package to the test directory.
+
+    :param test_sources: A list of test paths, relative to the package_dir.
+    :param package_dir: The root of the package directory.
+    :param test_dir: The folder where test sources should be placed.
+    :param copy_info: The copy function to use. By default, does a local
+        filesystem copy; but an OCIContainer.copy_info method (or equivalent)
+        can be provided.
+    """
+    for test_path in test_sources:
+        source = package_dir.resolve() / test_path
+
+        if not source.exists():
+            msg = f"Test source {test_path} does not exist."
+            raise errors.FatalError(msg)
+
+        copy_into(source, test_dir / test_path)
+
+
 class DependencyConstraints:
     def __init__(self, base_file_path: Path):
         assert base_file_path.exists()

cibuildwheel/util.py

@@ -27,6 +27,7 @@
     BuildSelector,
     call,
     combine_constraints,
+    copy_test_sources,
     download,
     extract_zip,
     find_compatible_wheel,
@@ -38,7 +39,6 @@
     read_python_configs,
     shell,
     split_config_settings,
-    test_fail_cwd_file,
     unwrap,
     virtualenv,
 )
@@ -572,9 +572,17 @@ def build(options: Options, tmp_path: Path) -> None:
                     package=options.globals.package_dir.resolve(),
                     wheel=repaired_wheel,
                 )
-                test_cwd = identifier_tmp_dir / "test_cwd"
-                test_cwd.mkdir()
-                (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text())
+                if build_options.test_sources:
+                    test_cwd = identifier_tmp_dir / "test_cwd"
+                    test_cwd.mkdir()
+                    copy_test_sources(
+                        build_options.test_sources,
+                        build_options.package_dir,
+                        test_cwd,
+                    )
+                else:
+                    # There are no test sources. Run the tests in the project directory.
+                    test_cwd = Path(".").resolve()
 
                 shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env)