gh-130907: Treat all module-level annotations as conditional (#131550)

Author: JelleZijlstra

Include/internal/pycore_compile.h

@@ -133,6 +133,8 @@ int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type
 void _PyCompile_ExitScope(struct _PyCompiler *c);
 Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o);
 _PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c);
+int _PyCompile_StartAnnotationSetup(struct _PyCompiler *c);
+int _PyCompile_EndAnnotationSetup(struct _PyCompiler *c);
 int _PyCompile_FutureFeatures(struct _PyCompiler *c);
 void _PyCompile_DeferredAnnotations(
     struct _PyCompiler *c, PyObject **deferred_annotations,

Include/internal/pycore_instruction_sequence.h

@@ -45,6 +45,9 @@ typedef struct instruction_sequence {
 
     /* PyList of instruction sequences of nested functions */
     PyObject *s_nested;
+
+    /* Code for creating annotations, spliced into the main sequence later */
+    struct instruction_sequence *s_annotations_code;
 } _PyInstructionSequence;
 
 typedef struct {
@@ -66,6 +69,8 @@ _PyJumpTargetLabel _PyInstructionSequence_NewLabel(_PyInstructionSequence *seq);
 int _PyInstructionSequence_ApplyLabelMap(_PyInstructionSequence *seq);
 int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int pos,
                                              int opcode, int oparg, _Py_SourceLocation loc);
+int _PyInstructionSequence_SetAnnotationsCode(_PyInstructionSequence *seq,
+                                              _PyInstructionSequence *annotations);
 int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested);
 void PyInstructionSequence_Fini(_PyInstructionSequence *seq);
 

Include/internal/pycore_opcode_metadata.h

@@ -26,6 +26,7 @@ def test_if_expression(self):
         false_lbl = self.Label()
         expected = [
             ('RESUME', 0, 0),
+            ('ANNOTATIONS_PLACEHOLDER', None),
             ('LOAD_CONST', 0, 1),
             ('TO_BOOL', 0, 1),
             ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1),
@@ -45,6 +46,7 @@ def test_for_loop(self):
         false_lbl = self.Label()
         expected = [
             ('RESUME', 0, 0),
+            ('ANNOTATIONS_PLACEHOLDER', None),
             ('LOAD_NAME', 0, 1),
             ('GET_ITER', None, 1),
             loop_lbl := self.Label(),
@@ -73,6 +75,7 @@ def f(x):
         expected = [
             # Function definition
             ('RESUME', 0),
+            ('ANNOTATIONS_PLACEHOLDER', None),
             ('LOAD_CONST', 0),
             ('MAKE_FUNCTION', None),
             ('STORE_NAME', 0),
@@ -106,6 +109,7 @@ def g():
         expected = [
             # Function definition
             ('RESUME', 0),
+            ('ANNOTATIONS_PLACEHOLDER', None),
             ('LOAD_CONST', 0),
             ('MAKE_FUNCTION', None),
             ('STORE_NAME', 0),

Include/opcode_ids.h

@@ -381,24 +381,36 @@ def wrap_func_w_kwargs():
 # leading newline is for a reason (tests lineno)
 
 dis_annot_stmt_str = """\
-  0           RESUME                   0
+  --           MAKE_CELL                0 (__conditional_annotations__)
 
-  2           LOAD_SMALL_INT           1
-              STORE_NAME               0 (x)
+   0           RESUME                   0
 
-  4           LOAD_SMALL_INT           1
-              LOAD_NAME                1 (lst)
-              LOAD_NAME                2 (fun)
-              PUSH_NULL
-              LOAD_SMALL_INT           0
-              CALL                     1
-              STORE_SUBSCR
+   2           LOAD_CONST               1 (<code object __annotate__ at 0x..., file "<dis>", line 2>)
+               MAKE_FUNCTION
+               STORE_NAME               4 (__annotate__)
+               BUILD_SET                0
+               STORE_NAME               0 (__conditional_annotations__)
+               LOAD_SMALL_INT           1
+               STORE_NAME               1 (x)
+               LOAD_NAME                0 (__conditional_annotations__)
+               LOAD_SMALL_INT           0
+               SET_ADD                  1
+               POP_TOP
 
-  2           LOAD_CONST               1 (<code object __annotate__ at 0x..., file "<dis>", line 2>)
-              MAKE_FUNCTION
-              STORE_NAME               3 (__annotate__)
-              LOAD_CONST               2 (None)
-              RETURN_VALUE
+   3           LOAD_NAME                0 (__conditional_annotations__)
+               LOAD_SMALL_INT           1
+               SET_ADD                  1
+               POP_TOP
+
+   4           LOAD_SMALL_INT           1
+               LOAD_NAME                2 (lst)
+               LOAD_NAME                3 (fun)
+               PUSH_NULL
+               LOAD_SMALL_INT           0
+               CALL                     1
+               STORE_SUBSCR
+               LOAD_CONST               2 (None)
+               RETURN_VALUE
 """
 
 fn_with_annotate_str = """
@@ -995,7 +1007,8 @@ def test_boundaries(self):
     def test_widths(self):
         long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT',
                             'LOAD_FAST_BORROW_LOAD_FAST_BORROW',
-                            'INSTRUMENTED_CALL_FUNCTION_EX'])
+                            'INSTRUMENTED_CALL_FUNCTION_EX',
+                            'ANNOTATIONS_PLACEHOLDER'])
         for op, opname in enumerate(dis.opname):
             if opname in long_opcodes or opname.startswith("INSTRUMENTED"):
                 continue

Lib/_opcode_metadata.py

@@ -383,9 +383,10 @@ def test_var_annot_simple_exec(self):
         gns = {}; lns = {}
         exec("'docstring'\n"
              "x: int = 5\n", gns, lns)
+        self.assertNotIn('__annotate__', gns)
+
+        gns.update(lns)  # __annotate__ looks at globals
         self.assertEqual(lns["__annotate__"](annotationlib.Format.VALUE), {'x': int})
-        with self.assertRaises(KeyError):
-            gns['__annotate__']
 
     def test_var_annot_rhs(self):
         ns = {}

Lib/test/test_compiler_codegen.py

@@ -3,7 +3,7 @@
 import textwrap
 import types
 import unittest
-from test.support import run_code, check_syntax_error, cpython_only
+from test.support import run_code, check_syntax_error, import_helper, cpython_only
 from test.test_inspect import inspect_stringized_annotations
 
 
@@ -151,6 +151,14 @@ class D(metaclass=C):
             del D.__annotations__
         self.assertEqual(D.__annotations__, {})
 
+    def test_partially_executed_module(self):
+        partialexe = import_helper.import_fresh_module("test.typinganndata.partialexecution")
+        self.assertEqual(
+            partialexe.a.__annotations__,
+            {"v1": int, "v2": int},
+        )
+        self.assertEqual(partialexe.b.annos, {"v1": int})
+
     @cpython_only
     def test_no_cell(self):
         # gh-130924: Test that uses of annotations in local scopes do not

Lib/test/test_dis.py

@@ -0,0 +1 @@
+from . import a

Lib/test/test_grammar.py

@@ -0,0 +1,5 @@
+v1: int
+
+from . import b
+
+v2: int

Lib/test/test_type_annotations.py

@@ -0,0 +1,3 @@
+from . import a
+
+annos = a.__annotations__

Lib/test/typinganndata/partialexecution/__init__.py

@@ -2679,6 +2679,7 @@ TESTSUBDIRS=	idlelib/idle_test \
 		test/translationdata/getopt \
 		test/translationdata/optparse \
 		test/typinganndata \
+		test/typinganndata/partialexecution \
 		test/wheeldata \
 		test/xmltestdata \
 		test/xmltestdata/c14n-20 \

Lib/test/typinganndata/partialexecution/a.py

@@ -0,0 +1,3 @@
+If the ``__annotations__`` of a module object are accessed while the
+module is executing, return the annotations that have been defined so far,
+without caching them.

Lib/test/typinganndata/partialexecution/b.py

@@ -1246,6 +1246,25 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored))
 
     PyObject *annotations;
     if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) == 0) {
+        PyObject *spec;
+        if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__spec__), &spec) < 0) {
+            Py_DECREF(dict);
+            return NULL;
+        }
+        bool is_initializing = false;
+        if (spec != NULL) {
+            int rc = _PyModuleSpec_IsInitializing(spec);
+            if (rc < 0) {
+                Py_DECREF(spec);
+                Py_DECREF(dict);
+                return NULL;
+            }
+            Py_DECREF(spec);
+            if (rc) {
+                is_initializing = true;
+            }
+        }
+
         PyObject *annotate;
         int annotate_result = PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate);
         if (annotate_result < 0) {
@@ -1273,7 +1292,8 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored))
             annotations = PyDict_New();
         }
         Py_XDECREF(annotate);
-        if (annotations) {
+        // Do not cache annotations if the module is still initializing
+        if (annotations && !is_initializing) {
             int result = PyDict_SetItem(
                     dict, &_Py_ID(__annotations__), annotations);
             if (result) {

Makefile.pre.in

@@ -2026,6 +2026,10 @@ dummy_func(
             }
         }
 
+        pseudo(ANNOTATIONS_PLACEHOLDER, (--)) = {
+            NOP,
+        };
+
         inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) {
             PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict);
             PyObject *update_o = PyStackRef_AsPyObjectBorrow(update);