Merge branch 'main' into raisesgroup-typing

Author: A5rocks

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ['pypy-3.10', '3.9', '3.10', '3.11', '3.12', '3.13']
+        python: ['3.9', '3.10', '3.11', '3.12', '3.13']
         arch: ['x86', 'x64']
         lsp: ['']
         lsp_extract_file: ['']
@@ -34,6 +34,11 @@ jobs:
             lsp: 'https://www.proxifier.com/download/legacy/ProxifierSetup342.exe'
             lsp_extract_file: ''
             extra_name: ', with IFS LSP'
+          - python: 'pypy-3.10'
+            arch: 'x64'
+            lsp: ''
+            lsp_extract_file: ''
+            extra_name: ''
           #- python: '3.9'
           #  arch: 'x64'
           #  lsp: 'http://download.pctools.com/mirror/updates/9.0.0.2308-SDavfree-lite_en.exe'

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.4
+    rev: v0.8.0
     hooks:
       - id: ruff
         types: [file]

newsfragments/3087.doc.rst

@@ -0,0 +1 @@
+Improve error message when run after gevent's monkey patching.

newsfragments/3112.bugfix.rst

@@ -0,0 +1,5 @@
+Rework foreign async generator finalization to track async generator
+ids rather than mutating ``ag_frame.f_locals``. This fixes an issue
+with the previous implementation: locals' lifetimes will no longer be
+extended by materialization in the ``ag_frame.f_locals`` dictionary that
+the previous finalization dispatcher logic needed to access to do its work.

pyproject.toml

@@ -125,15 +125,13 @@ select = [
     "Q",     # flake8-quotes
     "RUF",   # Ruff-specific rules
     "SIM",   # flake8-simplify
-    "TCH",   # flake8-type-checking
+    "TC",    # flake8-type-checking
     "UP",    # pyupgrade
     "W",     # Warning
     "YTT",   # flake8-2020
 ]
 extend-ignore = [
     'A002',    # builtin-argument-shadowing
-    'ANN101',  # missing-type-self
-    'ANN102',  # missing-type-cls
     'ANN401',  # any-type (mypy's `disallow_any_explicit` is better)
     'E402',    # module-import-not-at-top-of-file (usually OS-specific)
     'E501',    # line-too-long

src/trio/_core/_asyncgens.py

@@ -4,7 +4,7 @@
 import sys
 import warnings
 import weakref
-from typing import TYPE_CHECKING, NoReturn
+from typing import TYPE_CHECKING, NoReturn, TypeVar
 
 import attrs
 
@@ -16,14 +16,31 @@
 ASYNCGEN_LOGGER = logging.getLogger("trio.async_generator_errors")
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
     from types import AsyncGeneratorType
 
+    from typing_extensions import ParamSpec
+
+    _P = ParamSpec("_P")
+
     _WEAK_ASYNC_GEN_SET = weakref.WeakSet[AsyncGeneratorType[object, NoReturn]]
     _ASYNC_GEN_SET = set[AsyncGeneratorType[object, NoReturn]]
 else:
     _WEAK_ASYNC_GEN_SET = weakref.WeakSet
     _ASYNC_GEN_SET = set
 
+_R = TypeVar("_R")
+
+
+@_core.disable_ki_protection
+def _call_without_ki_protection(
+    f: Callable[_P, _R],
+    /,
+    *args: _P.args,
+    **kwargs: _P.kwargs,
+) -> _R:
+    return f(*args, **kwargs)
+
 
 @attrs.define(eq=False)
 class AsyncGenerators:
@@ -35,6 +52,11 @@ class AsyncGenerators:
     # regular set so we don't have to deal with GC firing at
     # unexpected times.
     alive: _WEAK_ASYNC_GEN_SET | _ASYNC_GEN_SET = attrs.Factory(_WEAK_ASYNC_GEN_SET)
+    # The ids of foreign async generators are added to this set when first
+    # iterated. Usually it is not safe to refer to ids like this, but because
+    # we're using a finalizer we can ensure ids in this set do not outlive
+    # their async generator.
+    foreign: set[int] = attrs.Factory(set)
 
     # This collects async generators that get garbage collected during
     # the one-tick window between the system nursery closing and the
@@ -51,10 +73,10 @@ def firstiter(agen: AsyncGeneratorType[object, NoReturn]) -> None:
                 # An async generator first iterated outside of a Trio
                 # task doesn't belong to Trio. Probably we're in guest
                 # mode and the async generator belongs to our host.
-                # The locals dictionary is the only good place to
+                # A strong set of ids is one of the only good places to
                 # remember this fact, at least until
-                # https://bugs.python.org/issue40916 is implemented.
-                agen.ag_frame.f_locals["@trio_foreign_asyncgen"] = True
+                # https://github.com/python/cpython/issues/85093 is implemented.
+                self.foreign.add(id(agen))
                 if self.prev_hooks.firstiter is not None:
                     self.prev_hooks.firstiter(agen)
 
@@ -76,13 +98,16 @@ def finalize_in_trio_context(
                 # have hit it.
                 self.trailing_needs_finalize.add(agen)
 
+        @_core.enable_ki_protection
         def finalizer(agen: AsyncGeneratorType[object, NoReturn]) -> None:
-            agen_name = name_asyncgen(agen)
             try:
-                is_ours = not agen.ag_frame.f_locals.get("@trio_foreign_asyncgen")
-            except AttributeError:  # pragma: no cover
+                self.foreign.remove(id(agen))
+            except KeyError:
                 is_ours = True
+            else:
+                is_ours = False
 
+            agen_name = name_asyncgen(agen)
             if is_ours:
                 runner.entry_queue.run_sync_soon(
                     finalize_in_trio_context,
@@ -105,8 +130,9 @@ def finalizer(agen: AsyncGeneratorType[object, NoReturn]) -> None:
                 )
             else:
                 # Not ours -> forward to the host loop's async generator finalizer
-                if self.prev_hooks.finalizer is not None:
-                    self.prev_hooks.finalizer(agen)
+                finalizer = self.prev_hooks.finalizer
+                if finalizer is not None:
+                    _call_without_ki_protection(finalizer, agen)
                 else:
                     # Host has no finalizer.  Reimplement the default
                     # Python behavior with no hooks installed: throw in
@@ -116,7 +142,7 @@ def finalizer(agen: AsyncGeneratorType[object, NoReturn]) -> None:
                     try:
                         # If the next thing is a yield, this will raise RuntimeError
                         # which we allow to propagate
-                        closer.send(None)
+                        _call_without_ki_protection(closer.send, None)
                     except StopIteration:
                         pass
                     else:

src/trio/_core/_run.py

@@ -2953,6 +2953,13 @@ async def checkpoint_if_cancelled() -> None:
         _KqueueStatistics as IOStatistics,
     )
 else:  # pragma: no cover
+    _patchers = sorted({"eventlet", "gevent"}.intersection(sys.modules))
+    if _patchers:
+        raise NotImplementedError(
+            "unsupported platform or primitives trio depends on are monkey-patched out by "
+            + ", ".join(_patchers),
+        )
+
     raise NotImplementedError("unsupported platform")
 
 from ._generated_instrumentation import *

src/trio/_core/_tests/test_guest_mode.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import contextlib
-import contextvars
 import queue
 import signal
 import socket
@@ -11,6 +10,7 @@
 import time
 import traceback
 import warnings
+import weakref
 from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence
 from functools import partial
 from math import inf
@@ -22,6 +22,7 @@
 )
 
 import pytest
+import sniffio
 from outcome import Outcome
 
 import trio
@@ -234,7 +235,8 @@ async def trio_main(in_host: InHost) -> str:
 
 
 def test_guest_mode_sniffio_integration() -> None:
-    from sniffio import current_async_library, thread_local as sniffio_library
+    current_async_library = sniffio.current_async_library
+    sniffio_library = sniffio.thread_local
 
     async def trio_main(in_host: InHost) -> str:
         async def synchronize() -> None:
@@ -458,9 +460,9 @@ def aiotrio_run(
 
     async def aio_main() -> T:
         nonlocal run_sync_soon_not_threadsafe
-        trio_done_fut = loop.create_future()
+        trio_done_fut: asyncio.Future[Outcome[T]] = loop.create_future()
 
-        def trio_done_callback(main_outcome: Outcome[object]) -> None:
+        def trio_done_callback(main_outcome: Outcome[T]) -> None:
             print(f"trio_fn finished: {main_outcome!r}")
             trio_done_fut.set_result(main_outcome)
 
@@ -479,9 +481,11 @@ def trio_done_callback(main_outcome: Outcome[object]) -> None:
             strict_exception_groups=strict_exception_groups,
         )
 
-        return (await trio_done_fut).unwrap()  # type: ignore[no-any-return]
+        return (await trio_done_fut).unwrap()
 
     try:
+        # can't use asyncio.run because that fails on Windows (3.8, x64, with
+        # Komodia LSP) and segfaults on Windows (3.9, x64, with Komodia LSP)
         return loop.run_until_complete(aio_main())
     finally:
         loop.close()
@@ -655,8 +659,6 @@ async def trio_main(in_host: InHost) -> None:
 
 @restore_unraisablehook()
 def test_guest_mode_asyncgens() -> None:
-    import sniffio
-
     record = set()
 
     async def agen(label: str) -> AsyncGenerator[int, None]:
@@ -683,9 +685,49 @@ async def trio_main() -> None:
 
         gc_collect_harder()
 
-    # Ensure we don't pollute the thread-level context if run under
-    # an asyncio without contextvars support (3.6)
-    context = contextvars.copy_context()
-    context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True)
+    aiotrio_run(trio_main, host_uses_signal_set_wakeup_fd=True)
 
     assert record == {("asyncio", "asyncio"), ("trio", "trio")}
+
+
+@restore_unraisablehook()
+def test_guest_mode_asyncgens_garbage_collection() -> None:
+    record: set[tuple[str, str, bool]] = set()
+
+    async def agen(label: str) -> AsyncGenerator[int, None]:
+        class A:
+            pass
+
+        a = A()
+        a_wr = weakref.ref(a)
+        assert sniffio.current_async_library() == label
+        try:
+            yield 1
+        finally:
+            library = sniffio.current_async_library()
+            with contextlib.suppress(trio.Cancelled):
+                await sys.modules[library].sleep(0)
+
+            del a
+            if sys.implementation.name == "pypy":
+                gc_collect_harder()
+
+            record.add((label, library, a_wr() is None))
+
+    async def iterate_in_aio() -> None:
+        await agen("asyncio").asend(None)
+
+    async def trio_main() -> None:
+        task = asyncio.ensure_future(iterate_in_aio())
+        done_evt = trio.Event()
+        task.add_done_callback(lambda _: done_evt.set())
+        with trio.fail_after(1):
+            await done_evt.wait()
+
+        await agen("trio").asend(None)
+
+        gc_collect_harder()
+
+    aiotrio_run(trio_main, host_uses_signal_set_wakeup_fd=True)
+
+    assert record == {("asyncio", "asyncio", True), ("trio", "trio", True)}

src/trio/_dtls.py

@@ -36,7 +36,7 @@
     from types import TracebackType
 
     # See DTLSEndpoint.__init__ for why this is imported here
-    from OpenSSL import SSL  # noqa: TCH004
+    from OpenSSL import SSL  # noqa: TC004
     from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack
 
     from trio._socket import AddressFormat

src/trio/_socket.py

@@ -12,7 +12,6 @@
 from typing import (
     TYPE_CHECKING,
     Any,
-    Literal,
     SupportsIndex,
     TypeVar,
     Union,
@@ -337,7 +336,7 @@ def fromshare(info: bytes) -> SocketType:
     TypeT: TypeAlias = int
     FamilyDefault = _stdlib_socket.AF_INET
 else:
-    FamilyDefault: Literal[None] = None
+    FamilyDefault: None = None
     FamilyT: TypeAlias = Union[int, AddressFamily, None]
     TypeT: TypeAlias = Union[_stdlib_socket.socket, int]
 

src/trio/_subprocess_platform/__init__.py

@@ -8,7 +8,7 @@
 import trio
 
 from .. import _core, _subprocess
-from .._abc import ReceiveStream, SendStream  # noqa: TCH001
+from .._abc import ReceiveStream, SendStream  # noqa: TC001
 
 _wait_child_exiting_error: ImportError | None = None
 _create_child_pipe_error: ImportError | None = None

src/trio/_tests/test_testing_raisesgroup.py

@@ -353,9 +353,9 @@ def check_errno_is_5(e: OSError) -> bool:
 def test_matcher_tostring() -> None:
     assert str(Matcher(ValueError)) == "Matcher(ValueError)"
     assert str(Matcher(match="[a-z]")) == "Matcher(match='[a-z]')"
-    pattern_no_flags = re.compile("noflag", 0)
+    pattern_no_flags = re.compile(r"noflag", 0)
     assert str(Matcher(match=pattern_no_flags)) == "Matcher(match='noflag')"
-    pattern_flags = re.compile("noflag", re.IGNORECASE)
+    pattern_flags = re.compile(r"noflag", re.IGNORECASE)
     assert str(Matcher(match=pattern_flags)) == f"Matcher(match={pattern_flags!r})"
     assert (
         str(Matcher(ValueError, match="re", check=bool))

src/trio/_tests/test_threads.py

@@ -220,7 +220,7 @@ def f(name: str) -> Callable[[None], threading.Thread]:
     # test that you can set a custom name, and that it's reset afterwards
     async def test_thread_name(name: str) -> None:
         thread = await to_thread_run_sync(f(name), thread_name=name)
-        assert re.match("Trio thread [0-9]*", thread.name)
+        assert re.match(r"Trio thread [0-9]*", thread.name)
 
     await test_thread_name("")
     await test_thread_name("fobiedoo")
@@ -301,7 +301,7 @@ async def test_thread_name(name: str, expected: str | None = None) -> None:
 
         os_thread_name = _get_thread_name(thread.ident)
         assert os_thread_name is not None, "should skip earlier if this is the case"
-        assert re.match("Trio thread [0-9]*", os_thread_name)
+        assert re.match(r"Trio thread [0-9]*", os_thread_name)
 
     await test_thread_name("")
     await test_thread_name("fobiedoo")

test-requirements.in

@@ -13,7 +13,7 @@ cryptography>=41.0.0  # cryptography<41 segfaults on pypy3.10
 black; implementation_name == "cpython"
 mypy  # Would use mypy[faster-cache], but orjson has build issues on pypy
 orjson; implementation_name == "cpython"
-ruff >= 0.6.6
+ruff >= 0.8.0
 astor          # code generation
 uv >= 0.2.24
 codespell

test-requirements.txt

@@ -113,7 +113,7 @@ pytest==8.3.3
     # via -r test-requirements.in
 requests==2.32.3
     # via sphinx
-ruff==0.7.3
+ruff==0.8.0
     # via -r test-requirements.in
 sniffio==1.3.1
     # via -r test-requirements.in