pythongh-117174: Add a new route in linecache to fetch interactive source code (python#117500)

Author: pablogsal

Lib/_pyrepl/console.py

@@ -24,6 +24,7 @@
 from abc import ABC, abstractmethod
 import ast
 import code
+import linecache
 from dataclasses import dataclass, field
 import os.path
 import sys
@@ -205,6 +206,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
             item = wrapper([stmt])
             try:
                 code = self.compile.compiler(item, filename, the_symbol)
+                linecache._register_code(code, source, filename)
             except SyntaxError as e:
                 if e.args[0] == "'await' outside function":
                     python = os.path.basename(sys.executable)

Lib/_pyrepl/simple_interact.py

@@ -26,7 +26,6 @@
 from __future__ import annotations
 
 import _sitebuiltins
-import linecache
 import functools
 import os
 import sys
@@ -148,7 +147,6 @@ def maybe_run_command(statement: str) -> bool:
                 continue
 
             input_name = f"<python-input-{input_n}>"
-            linecache._register_code(input_name, statement, "<stdin>")  # type: ignore[attr-defined]
             more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single")  # type: ignore[call-arg]
             assert not more
             input_n += 1

Lib/inspect.py

@@ -969,6 +969,8 @@ def findsource(object):
     module = getmodule(object, file)
     if module:
         lines = linecache.getlines(file, module.__dict__)
+        if not lines and file.startswith('<') and hasattr(object, "__code__"):
+            lines = linecache._getlines_from_code(object.__code__)
     else:
         lines = linecache.getlines(file)
     if not lines:

Lib/linecache.py

@@ -11,6 +11,7 @@
 # The cache. Maps filenames to either a thunk which will provide source code,
 # or a tuple (size, mtime, lines, fullname) once loaded.
 cache = {}
+_interactive_cache = {}
 
 
 def clearcache():
@@ -44,6 +45,22 @@ def getlines(filename, module_globals=None):
         return []
 
 
+def _getline_from_code(filename, lineno):
+    lines = _getlines_from_code(filename)
+    if 1 <= lineno <= len(lines):
+        return lines[lineno - 1]
+    return ''
+
+
+def _getlines_from_code(code):
+    code_id = id(code)
+    if code_id in _interactive_cache:
+        entry = _interactive_cache[code_id]
+        if len(entry) != 1:
+            return _interactive_cache[code_id][2]
+    return []
+
+
 def checkcache(filename=None):
     """Discard cache entries that are out of date.
     (This is not checked upon each call!)"""
@@ -88,9 +105,13 @@ def updatecache(filename, module_globals=None):
     # These imports are not at top level because linecache is in the critical
     # path of the interpreter startup and importing os and sys take a lot of time
     # and slows down the startup sequence.
-    import os
-    import sys
-    import tokenize
+    try:
+        import os
+        import sys
+        import tokenize
+    except ImportError:
+        # These import can fail if the interpreter is shutting down
+        return []
 
     if filename in cache:
         if len(cache[filename]) != 1:
@@ -196,8 +217,14 @@ def get_lines(name=name, *args, **kwargs):
 
 
 def _register_code(code, string, name):
-    cache[code] = (
-            len(string),
-            None,
-            [line + '\n' for line in string.splitlines()],
-            name)
+    entry = (len(string),
+             None,
+             [line + '\n' for line in string.splitlines()],
+             name)
+    stack = [code]
+    while stack:
+        code = stack.pop()
+        for const in code.co_consts:
+            if isinstance(const, type(code)):
+                stack.append(const)
+        _interactive_cache[id(code)] = entry

Lib/test/test_capi/test_exceptions.py

@@ -85,9 +85,7 @@ def foo():
         warnings = proc.err.splitlines()
         self.assertEqual(warnings, [
             b'<string>:6: RuntimeWarning: Testing PyErr_WarnEx',
-            b'  foo()  # line 6',
             b'<string>:9: RuntimeWarning: Testing PyErr_WarnEx',
-            b'  foo()  # line 9',
         ])
 
     def test_warn_during_finalization(self):

Lib/test/test_gdb/gdb_sample.py

@@ -1,4 +1,5 @@
 # Sample script for use by test_gdb
+from _typing import _idfunc
 
 def foo(a, b, c):
     bar(a=a, b=b, c=c)
@@ -7,6 +8,6 @@ def bar(a, b, c):
     baz(a, b, c)
 
 def baz(*args):
-    id(42)
+    _idfunc(42)
 
 foo(1, 2, 3)

Lib/test/test_gdb/test_backtrace.py

@@ -20,14 +20,14 @@ def test_bt(self):
         self.assertMultilineMatches(bt,
                                     r'''^.*
 Traceback \(most recent call first\):
-  <built-in method id of module object .*>
-  File ".*gdb_sample.py", line 10, in baz
-    id\(42\)
-  File ".*gdb_sample.py", line 7, in bar
+  <built-in method _idfunc of module object .*>
+  File ".*gdb_sample.py", line 11, in baz
+    _idfunc\(42\)
+  File ".*gdb_sample.py", line 8, in bar
     baz\(a, b, c\)
-  File ".*gdb_sample.py", line 4, in foo
+  File ".*gdb_sample.py", line 5, in foo
     bar\(a=a, b=b, c=c\)
-  File ".*gdb_sample.py", line 12, in <module>
+  File ".*gdb_sample.py", line 13, in <module>
     foo\(1, 2, 3\)
 ''')
 
@@ -39,11 +39,11 @@ def test_bt_full(self):
                                   cmds_after_breakpoint=['py-bt-full'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
-#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 8, in bar \(a=1, b=2, c=3\)
     baz\(a, b, c\)
-#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 5, in foo \(a=1, b=2, c=3\)
     bar\(a=a, b=b, c=c\)
-#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 12, in <module> \(\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 13, in <module> \(\)
     foo\(1, 2, 3\)
 ''')
 
@@ -55,6 +55,7 @@ def test_threads(self):
         'Verify that "py-bt" indicates threads that are waiting for the GIL'
         cmd = '''
 from threading import Thread
+from _typing import _idfunc
 
 class TestThread(Thread):
     # These threads would run forever, but we'll interrupt things with the
@@ -70,7 +71,7 @@ def run(self):
    t[i].start()
 
 # Trigger a breakpoint on the main thread
-id(42)
+_idfunc(42)
 
 '''
         # Verify with "py-bt":
@@ -90,8 +91,8 @@ def run(self):
     # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround
     def test_gc(self):
         'Verify that "py-bt" indicates if a thread is garbage-collecting'
-        cmd = ('from gc import collect\n'
-               'id(42)\n'
+        cmd = ('from gc import collect; from _typing import _idfunc\n'
+               '_idfunc(42)\n'
                'def foo():\n'
                '    collect()\n'
                'def bar():\n'
@@ -113,11 +114,12 @@ def test_gc(self):
                      "Python was compiled with optimizations")
     def test_wrapper_call(self):
         cmd = textwrap.dedent('''
+            from typing import _idfunc
             class MyList(list):
                 def __init__(self):
                     super(*[]).__init__()   # wrapper_call()
 
-            id("first break point")
+            _idfunc("first break point")
             l = MyList()
         ''')
         cmds_after_breakpoint = ['break wrapper_call', 'continue']

Lib/test/test_gdb/test_misc.py

@@ -35,40 +35,42 @@ def test_basic_command(self):
         bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-list'])
 
-        self.assertListing('   5    \n'
-                           '   6    def bar(a, b, c):\n'
-                           '   7        baz(a, b, c)\n'
-                           '   8    \n'
-                           '   9    def baz(*args):\n'
-                           ' >10        id(42)\n'
-                           '  11    \n'
-                           '  12    foo(1, 2, 3)\n',
+        self.assertListing('   6    \n'
+                           '   7    def bar(a, b, c):\n'
+                           '   8        baz(a, b, c)\n'
+                           '   9    \n'
+                           '  10    def baz(*args):\n'
+                           ' >11        _idfunc(42)\n'
+                           '  12    \n'
+                           '  13    foo(1, 2, 3)\n',
                            bt)
 
     def test_one_abs_arg(self):
         'Verify the "py-list" command with one absolute argument'
         bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-list 9'])
 
-        self.assertListing('   9    def baz(*args):\n'
-                           ' >10        id(42)\n'
-                           '  11    \n'
-                           '  12    foo(1, 2, 3)\n',
+        self.assertListing('  10    def baz(*args):\n'
+                           ' >11        _idfunc(42)\n'
+                           '  12    \n'
+                           '  13    foo(1, 2, 3)\n',
                            bt)
 
     def test_two_abs_args(self):
         'Verify the "py-list" command with two absolute arguments'
         bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
-                                  cmds_after_breakpoint=['py-list 1,3'])
+                                  cmds_after_breakpoint=['py-list 1,4'])
 
         self.assertListing('   1    # Sample script for use by test_gdb\n'
-                           '   2    \n'
-                           '   3    def foo(a, b, c):\n',
+                           '   2    from _typing import _idfunc\n'
+                           '   3    \n'
+                           '   4    def foo(a, b, c):\n',
                            bt)
 
 SAMPLE_WITH_C_CALL = """
 
 from _testcapi import pyobject_vectorcall
+from _typing import _idfunc
 
 def foo(a, b, c):
     bar(a, b, c)
@@ -77,7 +79,7 @@ def bar(a, b, c):
     pyobject_vectorcall(baz, (a, b, c), None)
 
 def baz(*args):
-    id(42)
+    _idfunc(42)
 
 foo(1, 2, 3)
 
@@ -94,7 +96,7 @@ def test_pyup_command(self):
                                   cmds_after_breakpoint=['py-up', 'py-up'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
-#[0-9]+ Frame 0x-?[0-9a-f]+, for file <string>, line 12, in baz \(args=\(1, 2, 3\)\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file <string>, line 13, in baz \(args=\(1, 2, 3\)\)
 #[0-9]+ <built-in method pyobject_vectorcall of module object at remote 0x[0-9a-f]+>
 $''')
 
@@ -123,9 +125,9 @@ def test_up_then_down(self):
                                   cmds_after_breakpoint=['py-up', 'py-up', 'py-down'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
-#[0-9]+ Frame 0x-?[0-9a-f]+, for file <string>, line 12, in baz \(args=\(1, 2, 3\)\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file <string>, line 13, in baz \(args=\(1, 2, 3\)\)
 #[0-9]+ <built-in method pyobject_vectorcall of module object at remote 0x[0-9a-f]+>
-#[0-9]+ Frame 0x-?[0-9a-f]+, for file <string>, line 12, in baz \(args=\(1, 2, 3\)\)
+#[0-9]+ Frame 0x-?[0-9a-f]+, for file <string>, line 13, in baz \(args=\(1, 2, 3\)\)
 $''')
 
 class PyPrintTests(DebuggerTests):