Merge remote-tracking branch 'origin/main' into feat/distinct-pyodide-build

Author: joerick

.github/workflows/test.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14]
+        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-15]
         python_version: ['3.13']
         include:
         - os: ubuntu-latest
@@ -63,11 +63,6 @@ jobs:
     - name: Set up QEMU
       if: runner.os == 'Linux'
       uses: docker/setup-qemu-action@v3
-      with:
-        # This should be temporary
-        # xref https://github.com/docker/setup-qemu-action/issues/188
-        # xref https://github.com/tonistiigi/binfmt/issues/215
-        image: tonistiigi/binfmt:qemu-v8.1.5
 
     - name: Install dependencies
       run: |
@@ -169,11 +164,6 @@ jobs:
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
-      with:
-        # This should be temporary
-        # xref https://github.com/docker/setup-qemu-action/issues/188
-        # xref https://github.com/tonistiigi/binfmt/issues/215
-        image: tonistiigi/binfmt:qemu-v8.1.5
 
     - name: Run the emulation tests
       run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py

.github/workflows/update-dependencies.yml

@@ -23,7 +23,7 @@ jobs:
 
     # we use this step to grab a Github App auth token, so that PRs generated by this workflow
     # run the GHA tests.
-    - uses: actions/create-github-app-token@v1
+    - uses: actions/create-github-app-token@v2
       id: generate-token
       if: github.ref == 'refs/heads/main' && github.repository == 'pypa/cibuildwheel'
       with:

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.2
+  rev: v0.11.4
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -81,7 +81,7 @@ repos:
 
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.31.3
+  rev: 0.32.1
   hooks:
     - id: check-dependabot
     - id: check-github-actions

README.md

@@ -139,6 +139,7 @@ Options
 |   | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.pypa.io/en/stable/options/#environment-pass)  | Set environment variables on the host to pass-through to the container during the build. |
 |   | [`CIBW_BEFORE_ALL`](https://cibuildwheel.pypa.io/en/stable/options/#before-all)  | Execute a shell command on the build system before any wheels are built. |
 |   | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.pypa.io/en/stable/options/#before-build)  | Execute a shell command preparing each wheel's build |
+|   | [`CIBW_XBUILD_TOOLS`](https://cibuildwheel.pypa.io/en/stable/options/#xbuild-tools)  | Binaries on the path that should be included in an isolated cross-build environment. |
 |   | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.pypa.io/en/stable/options/#repair-wheel-command)  | Execute a shell command to repair each built wheel |
 |   | [`CIBW_MANYLINUX_*_IMAGE`<br/>`CIBW_MUSLLINUX_*_IMAGE`](https://cibuildwheel.pypa.io/en/stable/options/#linux-image)  | Specify alternative manylinux / musllinux Docker images |
 |   | [`CIBW_CONTAINER_ENGINE`](https://cibuildwheel.pypa.io/en/stable/options/#container-engine)  | Specify which container engine to use when building Linux wheels |

azure-pipelines.yml

@@ -7,7 +7,7 @@ pr:
 jobs:
 - job: linux_311
   timeoutInMinutes: 120
-  pool: {vmImage: 'Ubuntu-20.04'}
+  pool: {vmImage: 'Ubuntu-22.04'}
   steps:
     - task: UsePythonVersion@0
       inputs:
@@ -27,7 +27,7 @@ jobs:
     - bash: |
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
-        python ./bin/run_tests.py --num-processes 2
+        python ./bin/run_tests.py
 
 - job: windows_311
   pool: {vmImage: 'windows-2019'}

bin/generate_schema.py

@@ -181,12 +181,15 @@
   musllinux-x86_64-image:
     type: string
     description: Specify alternative manylinux / musllinux container images
+  xbuild-tools:
+    description: Binaries on the path that should be included in an isolated cross-build environment
+    type: string_array
   pyodide-version:
     type: string
     description: Specify the version of Pyodide to use
   repair-wheel-command:
-    type: string_array
     description: Execute a shell command to repair each built wheel.
+    type: string_array
   skip:
     description: Choose the Python versions to skip.
     type: string_array
@@ -276,6 +279,7 @@
       properties:
         before-all: {"$ref": "#/$defs/inherit"}
         before-build: {"$ref": "#/$defs/inherit"}
+        xbuild-tools: {"$ref": "#/$defs/inherit"}
         before-test: {"$ref": "#/$defs/inherit"}
         config-settings: {"$ref": "#/$defs/inherit"}
         container-engine: {"$ref": "#/$defs/inherit"}

bin/run_tests.py

@@ -8,7 +8,11 @@
 from pathlib import Path
 
 if __name__ == "__main__":
-    default_cpu_count = os.cpu_count() or 2
+    if sys.version_info < (3, 13):
+        default_cpu_count = os.cpu_count() or 2
+    else:
+        default_cpu_count = os.process_cpu_count() or 2
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--run-podman", action="store_true", default=False, help="run podman tests (linux only)"
@@ -34,15 +38,39 @@
         if args.run_podman:
             unit_test_args += ["--run-podman"]
 
+    print(
+        "\n\n================================== UNIT TESTS ==================================",
+        flush=True,
+    )
     subprocess.run(unit_test_args, check=True)
 
-    # integration tests
+    # Run the serial integration tests without multiple processes
+    serial_integration_test_args = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-m",
+        "serial",
+        "-x",
+        "--durations",
+        "0",
+        "--timeout=2400",
+        "test",
+        "-vv",
+    ]
+    print(
+        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
+        flush=True,
+    )
+    subprocess.run(serial_integration_test_args, check=True)
+
+    # Non-serial integration tests
     integration_test_args = [
         sys.executable,
         "-m",
         "pytest",
-        "--dist",
-        "loadgroup",
+        "-m",
+        "not serial",
         f"--numprocesses={args.num_processes}",
         "-x",
         "--durations",
@@ -55,4 +83,8 @@
     if sys.platform.startswith("linux") and args.run_podman:
         integration_test_args += ["--run-podman"]
 
+    print(
+        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
+        flush=True,
+    )
     subprocess.run(integration_test_args, check=True)

cibuildwheel/frontend.py

@@ -37,14 +37,19 @@ def options_summary(self) -> str | dict[str, str]:
 
 
 def _get_verbosity_flags(level: int, frontend: BuildFrontendName) -> list[str]:
-    if frontend == "pip":
-        if level > 0:
-            return ["-" + level * "v"]
-        if level < 0:
+    if level < 0:
+        if frontend == "pip":
             return ["-" + -level * "q"]
-    elif not 0 <= level < 2:
+
         msg = f"build_verbosity {level} is not supported for {frontend} frontend. Ignoring."
         log.warning(msg)
+
+    if level > 0:
+        if frontend == "pip":
+            return ["-" + level * "v"]
+        if level > 1:
+            return ["-" + (level - 1) * "v"]
+
     return []
 
 

cibuildwheel/options.py

@@ -94,6 +94,7 @@ class BuildOptions:
     environment: ParsedEnvironment
     before_all: str
     before_build: str | None
+    xbuild_tools: list[str] | None
     repair_command: str
     manylinux_images: dict[str, str] | None
     musllinux_images: dict[str, str] | None
@@ -719,6 +720,18 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions:
 
             test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && "))
             before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && "))
+            xbuild_tools: list[str] | None = shlex.split(
+                self.reader.get(
+                    "xbuild-tools", option_format=ListFormat(sep=" ", quote=shlex.quote)
+                )
+            )
+            # ["\u0000"] is a sentinel value used as a default, because TOML
+            # doesn't have an explicit NULL value. If xbuild-tools is set to the
+            # sentinel, it indicates that the user hasn't defined xbuild-tools
+            # *at all* (not even an `xbuild-tools = []` definition).
+            if xbuild_tools == ["\u0000"]:
+                xbuild_tools = None
+
             test_sources = shlex.split(
                 self.reader.get(
                     "test-sources", option_format=ListFormat(sep=" ", quote=shlex.quote)
@@ -838,6 +851,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions:
                 before_build=before_build,
                 before_all=before_all,
                 build_verbosity=build_verbosity,
+                xbuild_tools=xbuild_tools,
                 repair_command=repair_command,
                 environment=environment,
                 dependency_constraints=dependency_constraints,