From fba5efa2a04871b845403443f85b18a20d303e3a Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 16:53:28 +0100 Subject: [PATCH 01/12] Add a test for union forward references --- Lib/test/test_annotationlib.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 0890be529a7e52..e5dd8dc20e26b0 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -936,6 +936,28 @@ def __call__(self): annotationlib.get_annotations(obj, format=format), {} ) + def test_union_forwardref(self): + # Test unions with '|' syntax equal unions with typing.Union[] with forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + match = ( + str, + support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + self.assertEqual( + typing.get_args(annos["pipe"]), + typing.get_args(annos["union"]) + ) + + self.assertEqual(typing.get_args(annos["pipe"]), match) + self.assertEqual(typing.get_args(annos["union"]), match) + + def test_pep695_generic_class_with_future_annotations(self): ann_module695 = inspect_stringized_annotations_pep695 A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) From d8109538837278622b4f3809d5ae51af942ad9ac Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 16:54:52 +0100 Subject: [PATCH 02/12] Make stringifiers create unions if create_unions is True --- Lib/annotationlib.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 971f636f9714d7..c095efd6f3626a 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -392,12 +392,19 @@ def binop(self, other): __mod__ = _make_binop(ast.Mod()) __lshift__ = _make_binop(ast.LShift()) __rshift__ = _make_binop(ast.RShift()) - __or__ = _make_binop(ast.BitOr()) __xor__ = _make_binop(ast.BitXor()) __and__ = _make_binop(ast.BitAnd()) __floordiv__ = _make_binop(ast.FloorDiv()) __pow__ = _make_binop(ast.Pow()) + def __or__(self, other): + if self.__stringifier_dict__.create_unions: + return types.UnionType[self, other] + + return self.__make_new( + ast.BinOp(self.__get_ast(), ast.BitOr(), self.__convert_to_ast(other)) + ) + del _make_binop def _make_rbinop(op: ast.AST): @@ -416,12 +423,19 @@ def rbinop(self, other): __rmod__ = _make_rbinop(ast.Mod()) __rlshift__ = _make_rbinop(ast.LShift()) __rrshift__ = _make_rbinop(ast.RShift()) - __ror__ = _make_rbinop(ast.BitOr()) __rxor__ = _make_rbinop(ast.BitXor()) __rand__ = _make_rbinop(ast.BitAnd()) __rfloordiv__ = _make_rbinop(ast.FloorDiv()) __rpow__ = _make_rbinop(ast.Pow()) + def __ror__(self, other): + if self.__stringifier_dict__.create_unions: + return types.UnionType[other, self] + + return self.__make_new( + ast.BinOp(self.__convert_to_ast(other), ast.BitOr(), self.__get_ast()) + ) + del _make_rbinop def _make_compare(op): @@ -459,12 +473,13 @@ def unary_op(self): class _StringifierDict(dict): - def __init__(self, namespace, globals=None, owner=None, is_class=False): + def __init__(self, namespace, globals=None, owner=None, is_class=False, create_unions=False): super().__init__(namespace) self.namespace = namespace self.globals = globals self.owner = owner self.is_class = is_class + self.create_unions = create_unions self.stringifiers = [] def __missing__(self, key): @@ -569,7 +584,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # that returns a bool and an defined set of attributes. namespace = {**annotate.__builtins__, **annotate.__globals__} is_class = isinstance(owner, type) - globals = _StringifierDict(namespace, annotate.__globals__, owner, is_class) + globals = _StringifierDict( + namespace, + annotate.__globals__, + owner, + is_class, + create_unions=True + ) if annotate.__closure__: freevars = annotate.__code__.co_freevars new_closure = [] From 1908a4a6ca0a70e45128237a07f0e307ff283eef Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 16:59:03 +0100 Subject: [PATCH 03/12] modify broken test, move test to forwardref format group --- Lib/test/test_annotationlib.py | 50 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index e5dd8dc20e26b0..fba1f11197c87d 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -115,8 +115,11 @@ def f( self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) alpha_anno = anno["alpha"] - self.assertIsInstance(alpha_anno, ForwardRef) - self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) + self.assertIsInstance(alpha_anno, Union) + self.assertEqual( + typing.get_args(alpha_anno), + (support.EqualToForwardRef("some", owner=f), support.EqualToForwardRef("obj", owner=f)) + ) beta_anno = anno["beta"] self.assertIsInstance(beta_anno, ForwardRef) @@ -126,6 +129,27 @@ def f( self.assertIsInstance(gamma_anno, ForwardRef) self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + def test_partially_nonexistent_union(self): + # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + match = ( + str, + support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + self.assertEqual( + typing.get_args(annos["pipe"]), + typing.get_args(annos["union"]) + ) + + self.assertEqual(typing.get_args(annos["pipe"]), match) + self.assertEqual(typing.get_args(annos["union"]), match) + class TestSourceFormat(unittest.TestCase): def test_closure(self): @@ -936,28 +960,6 @@ def __call__(self): annotationlib.get_annotations(obj, format=format), {} ) - def test_union_forwardref(self): - # Test unions with '|' syntax equal unions with typing.Union[] with forwardrefs - class UnionForwardrefs: - pipe: str | undefined - union: Union[str, undefined] - - annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) - - match = ( - str, - support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) - ) - - self.assertEqual( - typing.get_args(annos["pipe"]), - typing.get_args(annos["union"]) - ) - - self.assertEqual(typing.get_args(annos["pipe"]), match) - self.assertEqual(typing.get_args(annos["union"]), match) - - def test_pep695_generic_class_with_future_annotations(self): ann_module695 = inspect_stringized_annotations_pep695 A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) From 97abdc7f44f17d84d248e4f762d5499e5091cbb7 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 18:21:20 +0100 Subject: [PATCH 04/12] Apparently trim trailing whitespace was turned off --- Lib/annotationlib.py | 14 +++++++------- Lib/test/test_annotationlib.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c095efd6f3626a..b4cecb41d13c46 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -400,7 +400,7 @@ def binop(self, other): def __or__(self, other): if self.__stringifier_dict__.create_unions: return types.UnionType[self, other] - + return self.__make_new( ast.BinOp(self.__get_ast(), ast.BitOr(), self.__convert_to_ast(other)) ) @@ -431,11 +431,11 @@ def rbinop(self, other): def __ror__(self, other): if self.__stringifier_dict__.create_unions: return types.UnionType[other, self] - + return self.__make_new( ast.BinOp(self.__convert_to_ast(other), ast.BitOr(), self.__get_ast()) ) - + del _make_rbinop def _make_compare(op): @@ -585,10 +585,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): namespace = {**annotate.__builtins__, **annotate.__globals__} is_class = isinstance(owner, type) globals = _StringifierDict( - namespace, - annotate.__globals__, - owner, - is_class, + namespace, + annotate.__globals__, + owner, + is_class, create_unions=True ) if annotate.__closure__: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index fba1f11197c87d..6f17c85659c34e 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -117,7 +117,7 @@ def f( alpha_anno = anno["alpha"] self.assertIsInstance(alpha_anno, Union) self.assertEqual( - typing.get_args(alpha_anno), + typing.get_args(alpha_anno), (support.EqualToForwardRef("some", owner=f), support.EqualToForwardRef("obj", owner=f)) ) From dd36a625070c267715265dea5488a4d72df54846 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 13:37:00 -0700 Subject: [PATCH 05/12] Alternative approach --- Lib/annotationlib.py | 147 ++++++++++++++++++++------------- Lib/ast.py | 1 + Lib/test/test_annotationlib.py | 84 +++++++++++++++---- 3 files changed, 161 insertions(+), 71 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index b4cecb41d13c46..310c6a1801c970 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -38,6 +38,7 @@ class Format(enum.IntEnum): "__weakref__", "__arg__", "__globals__", + "__extra_names__", "__code__", "__ast_node__", "__cell__", @@ -82,6 +83,7 @@ def __init__( # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None self.__cell__ = None + self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None # value later. self.__code__ = None @@ -151,6 +153,8 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): if not self.__forward_is_class__ or param_name not in globals: globals[param_name] = param locals.pop(param_name, None) + if self.__extra_names__: + locals = {**locals, **self.__extra_names__} arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): @@ -274,6 +278,7 @@ def __init__( cell=None, *, stringifier_dict, + extra_names=None, ): # Either an AST node or a simple str (for the common case where a ForwardRef # represent a single name). @@ -285,6 +290,7 @@ def __init__( self.__code__ = None self.__ast_node__ = node self.__globals__ = globals + self.__extra_names__ = extra_names self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict @@ -292,28 +298,33 @@ def __init__( def __convert_to_ast(self, other): if isinstance(other, _Stringifier): if isinstance(other.__ast_node__, str): - return ast.Name(id=other.__ast_node__) - return other.__ast_node__ - elif isinstance(other, slice): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif other is None or type(other) in (str, int, float, bool, complex): + return ast.Constant(value=other), None + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + return ast.Slice( - lower=( - self.__convert_to_ast(other.start) - if other.start is not None - else None - ), - upper=( - self.__convert_to_ast(other.stop) - if other.stop is not None - else None - ), - step=( - self.__convert_to_ast(other.step) - if other.step is not None - else None - ), - ) + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names else: - return ast.Constant(value=other) + return self.__convert_to_ast(other) def __get_ast(self): node = self.__ast_node__ @@ -321,13 +332,19 @@ def __get_ast(self): return ast.Name(id=node) return node - def __make_new(self, node): + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) stringifier = _Stringifier( node, self.__globals__, self.__owner__, self.__forward_is_class__, stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names, ) self.__stringifier_dict__.stringifiers.append(stringifier) return stringifier @@ -343,27 +360,37 @@ def __getitem__(self, other): if self.__ast_node__ == "__classdict__": raise KeyError if isinstance(other, tuple): - elts = [self.__convert_to_ast(elt) for elt in other] + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) other = ast.Tuple(elts) else: - other = self.__convert_to_ast(other) + other, extra_names = self.__convert_to_ast_getitem(other) assert isinstance(other, ast.AST), repr(other) - return self.__make_new(ast.Subscript(self.__get_ast(), other)) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) def __getattr__(self, attr): return self.__make_new(ast.Attribute(self.__get_ast(), attr)) def __call__(self, *args, **kwargs): - return self.__make_new( - ast.Call( - self.__get_ast(), - [self.__convert_to_ast(arg) for arg in args], - [ - ast.keyword(key, self.__convert_to_ast(value)) - for key, value in kwargs.items() - ], - ) - ) + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) def __iter__(self): yield self.__make_new(ast.Starred(self.__get_ast())) @@ -378,8 +405,9 @@ def __format__(self, format_spec): def _make_binop(op: ast.AST): def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) return self.__make_new( - ast.BinOp(self.__get_ast(), op, self.__convert_to_ast(other)) + ast.BinOp(self.__get_ast(), op, rhs), extra_names ) return binop @@ -392,25 +420,19 @@ def binop(self, other): __mod__ = _make_binop(ast.Mod()) __lshift__ = _make_binop(ast.LShift()) __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) __xor__ = _make_binop(ast.BitXor()) __and__ = _make_binop(ast.BitAnd()) __floordiv__ = _make_binop(ast.FloorDiv()) __pow__ = _make_binop(ast.Pow()) - def __or__(self, other): - if self.__stringifier_dict__.create_unions: - return types.UnionType[self, other] - - return self.__make_new( - ast.BinOp(self.__get_ast(), ast.BitOr(), self.__convert_to_ast(other)) - ) - del _make_binop def _make_rbinop(op: ast.AST): def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) return self.__make_new( - ast.BinOp(self.__convert_to_ast(other), op, self.__get_ast()) + ast.BinOp(new_other, op, self.__get_ast()), extra_names ) return rbinop @@ -423,29 +445,24 @@ def rbinop(self, other): __rmod__ = _make_rbinop(ast.Mod()) __rlshift__ = _make_rbinop(ast.LShift()) __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) __rxor__ = _make_rbinop(ast.BitXor()) __rand__ = _make_rbinop(ast.BitAnd()) __rfloordiv__ = _make_rbinop(ast.FloorDiv()) __rpow__ = _make_rbinop(ast.Pow()) - def __ror__(self, other): - if self.__stringifier_dict__.create_unions: - return types.UnionType[other, self] - - return self.__make_new( - ast.BinOp(self.__convert_to_ast(other), ast.BitOr(), self.__get_ast()) - ) - del _make_rbinop def _make_compare(op): def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) return self.__make_new( ast.Compare( left=self.__get_ast(), ops=[op], - comparators=[self.__convert_to_ast(other)], - ) + comparators=[rhs], + ), + extra_names, ) return compare @@ -479,8 +496,9 @@ def __init__(self, namespace, globals=None, owner=None, is_class=False, create_u self.globals = globals self.owner = owner self.is_class = is_class - self.create_unions = create_unions + self.create_unions = False self.stringifiers = [] + self.next_id = 1 def __missing__(self, key): fwdref = _Stringifier( @@ -493,6 +511,11 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + def call_evaluate_function(evaluate, format, *, owner=None): """Call an evaluate function. Evaluate functions are normally generated for @@ -559,9 +582,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) annos = func(Format.VALUE_WITH_FAKE_GLOBALS) if _is_evaluate: - return annos if isinstance(annos, str) else repr(annos) + return _stringify_single(annos) return { - key: val if isinstance(val, str) else repr(val) + key: _stringify_single(val) for key, val in annos.items() } elif format == Format.FORWARDREF: @@ -640,6 +663,16 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): raise ValueError(f"Invalid format: {format!r}") +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + else: + return repr(anno) + + def get_annotate_function(obj): """Get the __annotate__ function for an object. diff --git a/Lib/ast.py b/Lib/ast.py index 507fec5f2d3890..91624d2a7d6246 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -276,6 +276,7 @@ def iter_fields(node): Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` that is present on *node*. """ + print(node) for field in node._fields: try: yield field, getattr(node, field) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 6f17c85659c34e..2351880840575e 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -115,11 +115,8 @@ def f( self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) alpha_anno = anno["alpha"] - self.assertIsInstance(alpha_anno, Union) - self.assertEqual( - typing.get_args(alpha_anno), - (support.EqualToForwardRef("some", owner=f), support.EqualToForwardRef("obj", owner=f)) - ) + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) beta_anno = anno["beta"] self.assertIsInstance(beta_anno, ForwardRef) @@ -137,19 +134,20 @@ class UnionForwardrefs: annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) - match = ( - str, - support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + pipe = annos["pipe"] + self.assertIsInstance(pipe, ForwardRef) + self.assertEqual( + pipe.evaluate(globals={"undefined": int}), + str | int, ) - + union = annos["union"] + self.assertIsInstance(union, Union) + arg1, arg2 = typing.get_args(union) + self.assertIs(arg1, str) self.assertEqual( - typing.get_args(annos["pipe"]), - typing.get_args(annos["union"]) + arg2, support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) ) - self.assertEqual(typing.get_args(annos["pipe"]), match) - self.assertEqual(typing.get_args(annos["union"]), match) - class TestSourceFormat(unittest.TestCase): def test_closure(self): @@ -280,6 +278,64 @@ def f( }, ) + def test_getitem(self): + def f(x: undef1[str, undef2]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "undef1[str, undef2]"}) + + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) + fwdref = anno["x"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual( + fwdref.evaluate(globals={"undef1": dict, "undef2": float}), dict[str, float] + ) + + def test_slice(self): + def f(x: a[b:c]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c]"}) + + def f(x: a[b:c, d:e]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c, d:e]"}) + + obj = slice(1, 1, 1) + def f(x: obj): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "obj"}) + + def test_literals(self): + def f( + a: 1, + b: 1.0, + c: "hello", + d: b"hello", + e: True, + f: None, + g: ..., + h: 1j, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "a": "1", + "b": "1.0", + "c": 'hello', + "d": "b'hello'", + "e": "True", + "f": "None", + "g": "...", + "h": "1j", + }, + ) + def test_nested_expressions(self): def f( nested: list[Annotated[set[int], "set of ints", 4j]], From 8266a5e358818e686b789c166093a3677bf1259e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 13:41:27 -0700 Subject: [PATCH 06/12] fixes --- Lib/annotationlib.py | 5 +++++ Lib/ast.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 310c6a1801c970..18894d5985d101 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -235,6 +235,10 @@ def __eq__(self, other): and self.__forward_is_class__ == other.__forward_is_class__ and self.__cell__ == other.__cell__ and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) ) def __hash__(self): @@ -245,6 +249,7 @@ def __hash__(self): self.__forward_is_class__, self.__cell__, self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) def __or__(self, other): diff --git a/Lib/ast.py b/Lib/ast.py index 91624d2a7d6246..507fec5f2d3890 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -276,7 +276,6 @@ def iter_fields(node): Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` that is present on *node*. """ - print(node) for field in node._fields: try: yield field, getattr(node, field) From 1c629dc17366ea9dc243328520115ae1318407c4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 13:42:15 -0700 Subject: [PATCH 07/12] blurb --- .../next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst b/Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst new file mode 100644 index 00000000000000..d62b95775a67c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst @@ -0,0 +1,2 @@ +Fix incorrect handling of nested non-constant values in the FORWARDREF +format in :mod:`annotationlib`. From 3d7fec8d3efac475f988df409d2cf4c899d385f6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 14:07:22 -0700 Subject: [PATCH 08/12] Handle displays --- Lib/annotationlib.py | 24 ++++++++++++++++++++++++ Lib/test/test_annotationlib.py | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 18894d5985d101..330313fe285d56 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -307,6 +307,30 @@ def __convert_to_ast(self, other): return other.__ast_node__, other.__extra_names__ elif other is None or type(other) in (str, int, float, bool, complex): return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names else: name = self.__stringifier_dict__.create_unique_name() return ast.Name(id=name), {name: other} diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 2351880840575e..28ab5e653499ac 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -336,6 +336,31 @@ def f( }, ) + def test_displays(self): + # Simple case first + def f(x: a[[int, str], float]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[[int, str], float]"}) + + def g( + w: a[[int, str], float], + x: a[{int, str}, 3], + y: a[{int: str}, 4], + z: a[(int, str), 5], + ): + pass + anno = annotationlib.get_annotations(g, format=Format.STRING) + self.assertEqual( + anno, + { + "w": "a[[int, str], float]", + "x": "a[{int, str}, 3]", + "y": "a[{int: str}, 4]", + "z": "a[(int, str), 5]", + }, + ) + def test_nested_expressions(self): def f( nested: list[Annotated[set[int], "set of ints", 4j]], From ec4dd0e1fbc619a34f8ace9602c0bafab2eab9f1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 14:08:14 -0700 Subject: [PATCH 09/12] remove create_unions --- Lib/annotationlib.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 330313fe285d56..92f8b773576142 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -519,13 +519,12 @@ def unary_op(self): class _StringifierDict(dict): - def __init__(self, namespace, globals=None, owner=None, is_class=False, create_unions=False): + def __init__(self, namespace, globals=None, owner=None, is_class=False): super().__init__(namespace) self.namespace = namespace self.globals = globals self.owner = owner self.is_class = is_class - self.create_unions = False self.stringifiers = [] self.next_id = 1 @@ -641,7 +640,6 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): annotate.__globals__, owner, is_class, - create_unions=True ) if annotate.__closure__: freevars = annotate.__code__.co_freevars From 6e31c69e4f33fb3951cf16eb0000b28b1e2bb7a3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 16:48:31 -0700 Subject: [PATCH 10/12] prefer empty dict --- Lib/annotationlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 92f8b773576142..01c48de679cc2c 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -373,7 +373,7 @@ def __make_new(self, node, extra_names=None): self.__owner__, self.__forward_is_class__, stringifier_dict=self.__stringifier_dict__, - extra_names=new_extra_names, + extra_names=new_extra_names or None, ) self.__stringifier_dict__.stringifiers.append(stringifier) return stringifier From 525a4e499c68177b418f3dabe983dc91ab7e4028 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 23 Apr 2025 08:13:54 -0700 Subject: [PATCH 11/12] Do not use special names in STRING format --- Lib/annotationlib.py | 20 ++++++++++++++------ Lib/test/test_annotationlib.py | 11 +++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 01c48de679cc2c..f6318507ff50de 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -305,7 +305,13 @@ def __convert_to_ast(self, other): if isinstance(other.__ast_node__, str): return ast.Name(id=other.__ast_node__), other.__extra_names__ return other.__ast_node__, other.__extra_names__ - elif other is None or type(other) in (str, int, float, bool, complex): + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): return ast.Constant(value=other), None elif type(other) is dict: extra_names = {} @@ -519,7 +525,7 @@ def unary_op(self): class _StringifierDict(dict): - def __init__(self, namespace, globals=None, owner=None, is_class=False): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) self.namespace = namespace self.globals = globals @@ -527,6 +533,7 @@ def __init__(self, namespace, globals=None, owner=None, is_class=False): self.is_class = is_class self.stringifiers = [] self.next_id = 1 + self.format = format def __missing__(self, key): fwdref = _Stringifier( @@ -587,7 +594,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. - globals = _StringifierDict({}) + globals = _StringifierDict({}, format=format) if annotate.__closure__: freevars = annotate.__code__.co_freevars new_closure = [] @@ -637,9 +644,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class = isinstance(owner, type) globals = _StringifierDict( namespace, - annotate.__globals__, - owner, - is_class, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, ) if annotate.__closure__: freevars = annotate.__code__.co_freevars diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 28ab5e653499ac..1c95ae2ef3d7e2 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -406,6 +406,17 @@ def f(fstring_format: f"{a:02d}"): with self.assertRaisesRegex(TypeError, format_msg): annotationlib.get_annotations(f, format=Format.STRING) + def test_shenanigans(self): + # In cases like this we can't reconstruct the source; test that we do something + # halfway reasonable. + def f(x: x | (1).__class__, y: (1).__class__): + pass + + self.assertEqual( + annotationlib.get_annotations(f, format=Format.STRING), + {"x": "x | ", "y": ""}, + ) + class TestForwardRefClass(unittest.TestCase): def test_special_attrs(self): From 22a9691884f9db71bbac7741ce449a0111d793ae Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 4 May 2025 07:36:48 -0700 Subject: [PATCH 12/12] fix bad merge --- Lib/test/test_annotationlib.py | 106 --------------------------------- 1 file changed, 106 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index ec63d5dbc05f6d..d9000b6392277e 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -413,112 +413,6 @@ def f(x: x | (1).__class__, y: (1).__class__): ) -class TestForwardRefClass(unittest.TestCase): - def test_special_attrs(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = ForwardRef("set[Any]") - self.assertFalse(hasattr(fr, "__name__")) - self.assertFalse(hasattr(fr, "__qualname__")) - self.assertEqual(fr.__module__, "annotationlib") - # Forward refs are currently unpicklable once they contain a code object. - fr.__forward_code__ # fill the cache - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError): - pickle.dumps(fr, proto) - - def test_evaluate_with_type_params(self): - class Gen[T]: - alias = int - - with self.assertRaises(NameError): - ForwardRef("T").evaluate() - with self.assertRaises(NameError): - ForwardRef("T").evaluate(type_params=()) - with self.assertRaises(NameError): - ForwardRef("T").evaluate(owner=int) - - (T,) = Gen.__type_params__ - self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) - self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) - - with self.assertRaises(NameError): - ForwardRef("alias").evaluate(type_params=Gen.__type_params__) - self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) - # If you pass custom locals, we don't look at the owner's locals - with self.assertRaises(NameError): - ForwardRef("alias").evaluate(owner=Gen, locals={}) - # But if the name exists in the locals, it works - self.assertIs( - ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str - ) - - def test_fwdref_with_module(self): - self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) - self.assertIs( - ForwardRef("Counter", module="collections").evaluate(), collections.Counter - ) - self.assertEqual( - ForwardRef("Counter[int]", module="collections").evaluate(), - collections.Counter[int], - ) - - with self.assertRaises(NameError): - # If globals are passed explicitly, we don't look at the module dict - ForwardRef("Format", module="annotationlib").evaluate(globals={}) - - def test_fwdref_to_builtin(self): - self.assertIs(ForwardRef("int").evaluate(), int) - self.assertIs(ForwardRef("int", module="collections").evaluate(), int) - self.assertIs(ForwardRef("int", owner=str).evaluate(), int) - - # builtins are still searched with explicit globals - self.assertIs(ForwardRef("int").evaluate(globals={}), int) - - # explicit values in globals have precedence - obj = object() - self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) - - def test_fwdref_value_is_not_cached(self): - fr = ForwardRef("hello") - with self.assertRaises(NameError): - fr.evaluate() - self.assertIs(fr.evaluate(globals={"hello": str}), str) - with self.assertRaises(NameError): - fr.evaluate() - - def test_fwdref_with_owner(self): - self.assertEqual( - ForwardRef("Counter[int]", owner=collections).evaluate(), - collections.Counter[int], - ) - - def test_name_lookup_without_eval(self): - # test the codepath where we look up simple names directly in the - # namespaces without going through eval() - self.assertIs(ForwardRef("int").evaluate(), int) - self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) - self.assertIs( - ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), - float, - ) - self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) - with support.swap_attr(builtins, "int", dict): - self.assertIs(ForwardRef("int").evaluate(), dict) - - with self.assertRaises(NameError): - ForwardRef("doesntexist").evaluate() - - def test_fwdref_invalid_syntax(self): - fr = ForwardRef("if") - with self.assertRaises(SyntaxError): - fr.evaluate() - fr = ForwardRef("1+") - with self.assertRaises(SyntaxError): - fr.evaluate() - - class TestGetAnnotations(unittest.TestCase): def test_builtin_type(self): self.assertEqual(get_annotations(int), {})