Merge branch 'main' into pythongh-132042-precalc-mro-dict

Author: sergey-miryanov

.github/CODEOWNERS

@@ -167,6 +167,9 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 **/*imap*                     @python/email-team
 **/*poplib*                   @python/email-team
 
+# Exclude .mailmap from being owned by @python/email-team
+/.mailmap
+
 # Garbage collector
 /Modules/gcmodule.c           @pablogsal
 /Doc/library/gc.rst           @pablogsal

.mailmap

@@ -1,3 +1,4 @@
 # This file sets the canonical name for contributors to the repository.
 # Documentation: https://git-scm.com/docs/gitmailmap
+Willow Chargin <[email protected]>
 Amethyst Reese <[email protected]> <[email protected]>

Doc/library/shutil.rst

@@ -473,7 +473,7 @@ Directory and files operations
    This is also applied when *cmd* is a path that contains a directory
    component::
 
-      >> shutil.which("C:\\Python33\\python")
+      >>> shutil.which("C:\\Python33\\python")
       'C:\\Python33\\python.EXE'
 
    .. versionadded:: 3.3

Doc/library/socket.rst

@@ -498,6 +498,9 @@ Constants
    .. versionchanged:: 3.11
       NetBSD support was added.
 
+   .. versionchanged:: next
+      Restored missing ``CAN_RAW_ERR_FILTER`` on Linux.
+
 .. data:: CAN_BCM
           CAN_BCM_*
 

Doc/whatsnew/3.14.rst

@@ -479,6 +479,12 @@ Other language changes
   :func:`textwrap.dedent`.
   (Contributed by Jon Crall and Steven Sun in :gh:`103998`.)
 
+* Improve error message when an object supporting the synchronous (resp.
+  asynchronous) context manager protocol is entered using :keyword:`async
+  with` (resp. :keyword:`with`) instead of :keyword:`with` (resp.
+  :keyword:`async with`).
+  (Contributed by Bénédikt Tran in :gh:`128398`.)
+
 
 .. _whatsnew314-pep765:
 
@@ -819,6 +825,12 @@ json
   See the :ref:`JSON command-line interface <json-commandline>` documentation.
   (Contributed by Trey Hunner in :gh:`122873`.)
 
+* By default, the output of the :ref:`JSON command-line interface <json-commandline>`
+  is highlighted in color. This can be controlled via the
+  :envvar:`PYTHON_COLORS` environment variable as well as the canonical
+  |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. See also
+  :ref:`using-on-controlling-color`.
+  (Contributed by Tomas Roun in :gh:`131952`.)
 
 linecache
 ---------

Include/internal/pycore_ceval.h

@@ -279,6 +279,7 @@ PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[];
 typedef struct _special_method {
     PyObject *name;
     const char *error;
+    const char *error_suggestion;  // improved optional suggestion
 } _Py_SpecialMethod;
 
 PyAPI_DATA(const _Py_SpecialMethod) _Py_SpecialMethods[];
@@ -309,6 +310,16 @@ PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFra
 PyAPI_FUNC(int)
 _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args);
 
+/*
+ * Indicate whether a special method of given 'oparg' can use the (improved)
+ * alternative error message instead. Only methods loaded by LOAD_SPECIAL
+ * support alternative error messages.
+ *
+ * Symbol is exported for the JIT (see discussion on GH-132218).
+ */
+PyAPI_FUNC(int)
+_PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg);
+
 /* Bits that can be set in PyThreadState.eval_breaker */
 #define _PY_GIL_DROP_REQUEST_BIT (1U << 0)
 #define _PY_SIGNALS_PENDING_BIT (1U << 1)

Include/internal/pycore_interp_structs.h

@@ -754,6 +754,12 @@ struct _is {
      * and should be placed at the beginning. */
     struct _ceval_state ceval;
 
+   /* This structure is carefully allocated so that it's correctly aligned
+     * to avoid undefined behaviors during LOAD and STORE. The '_malloced'
+     * field stores the allocated pointer address that will later be freed.
+     */
+    void *_malloced;
+
     PyInterpreterState *next;
 
     int64_t id;

Lib/json/tool.py

@@ -5,7 +5,40 @@
 """
 import argparse
 import json
+import re
 import sys
+from _colorize import ANSIColors, can_colorize
+
+
+# The string we are colorizing is valid JSON,
+# so we can use a looser but simpler regex to match
+# the various parts, most notably strings and numbers,
+# where the regex given by the spec is much more complex.
+_color_pattern = re.compile(r'''
+    (?P<key>"(\\.|[^"\\])*")(?=:)           |
+    (?P<string>"(\\.|[^"\\])*")             |
+    (?P<boolean>true|false)                 |
+    (?P<null>null)
+''', re.VERBOSE)
+
+
+_colors = {
+    'key': ANSIColors.INTENSE_BLUE,
+    'string': ANSIColors.BOLD_GREEN,
+    'boolean': ANSIColors.BOLD_CYAN,
+    'null': ANSIColors.BOLD_CYAN,
+}
+
+
+def _replace_match_callback(match):
+    for key, color in _colors.items():
+        if m := match.group(key):
+            return f"{color}{m}{ANSIColors.RESET}"
+    return match.group()
+
+
+def _colorize_json(json_str):
+    return re.sub(_color_pattern, _replace_match_callback, json_str)
 
 
 def main():
@@ -68,7 +101,11 @@ def main():
             outfile = open(options.outfile, 'w', encoding='utf-8')
         with outfile:
             for obj in objs:
-                json.dump(obj, outfile, **dump_args)
+                if can_colorize(file=outfile):
+                    json_str = json.dumps(obj, **dump_args)
+                    outfile.write(_colorize_json(json_str))
+                else:
+                    json.dump(obj, outfile, **dump_args)
                 outfile.write('\n')
     except ValueError as e:
         raise SystemExit(e)

Lib/test/test_json/test_tool.py

@@ -6,7 +6,7 @@
 import subprocess
 
 from test import support
-from test.support import os_helper
+from test.support import force_not_colorized, os_helper
 from test.support.script_helper import assert_python_ok
 
 
@@ -87,6 +87,7 @@ class TestMain(unittest.TestCase):
     }
     """)
 
+    @force_not_colorized
     def test_stdin_stdout(self):
         args = sys.executable, '-m', self.module
         process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True)
@@ -102,7 +103,8 @@ def _create_infile(self, data=None):
 
     def test_infile_stdout(self):
         infile = self._create_infile()
-        rc, out, err = assert_python_ok('-m', self.module, infile)
+        rc, out, err = assert_python_ok('-m', self.module, infile,
+                                        PYTHON_COLORS='0')
         self.assertEqual(rc, 0)
         self.assertEqual(out.splitlines(), self.expect.encode().splitlines())
         self.assertEqual(err, b'')
@@ -116,7 +118,8 @@ def test_non_ascii_infile(self):
         ''').encode()
 
         infile = self._create_infile(data)
-        rc, out, err = assert_python_ok('-m', self.module, infile)
+        rc, out, err = assert_python_ok('-m', self.module, infile,
+                                        PYTHON_COLORS='0')
 
         self.assertEqual(rc, 0)
         self.assertEqual(out.splitlines(), expect.splitlines())
@@ -125,7 +128,8 @@ def test_non_ascii_infile(self):
     def test_infile_outfile(self):
         infile = self._create_infile()
         outfile = os_helper.TESTFN + '.out'
-        rc, out, err = assert_python_ok('-m', self.module, infile, outfile)
+        rc, out, err = assert_python_ok('-m', self.module, infile, outfile,
+                                        PYTHON_COLORS='0')
         self.addCleanup(os.remove, outfile)
         with open(outfile, "r", encoding="utf-8") as fp:
             self.assertEqual(fp.read(), self.expect)
@@ -135,33 +139,38 @@ def test_infile_outfile(self):
 
     def test_writing_in_place(self):
         infile = self._create_infile()
-        rc, out, err = assert_python_ok('-m', self.module, infile, infile)
+        rc, out, err = assert_python_ok('-m', self.module, infile, infile,
+                                        PYTHON_COLORS='0')
         with open(infile, "r", encoding="utf-8") as fp:
             self.assertEqual(fp.read(), self.expect)
         self.assertEqual(rc, 0)
         self.assertEqual(out, b'')
         self.assertEqual(err, b'')
 
+    @force_not_colorized
     def test_jsonlines(self):
         args = sys.executable, '-m', self.module, '--json-lines'
         process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True)
         self.assertEqual(process.stdout, self.jsonlines_expect)
         self.assertEqual(process.stderr, '')
 
     def test_help_flag(self):
-        rc, out, err = assert_python_ok('-m', self.module, '-h')
+        rc, out, err = assert_python_ok('-m', self.module, '-h',
+                                        PYTHON_COLORS='0')
         self.assertEqual(rc, 0)
         self.assertTrue(out.startswith(b'usage: '))
         self.assertEqual(err, b'')
 
     def test_sort_keys_flag(self):
         infile = self._create_infile()
-        rc, out, err = assert_python_ok('-m', self.module, '--sort-keys', infile)
+        rc, out, err = assert_python_ok('-m', self.module, '--sort-keys', infile,
+                                        PYTHON_COLORS='0')
         self.assertEqual(rc, 0)
         self.assertEqual(out.splitlines(),
                          self.expect_without_sort_keys.encode().splitlines())
         self.assertEqual(err, b'')
 
+    @force_not_colorized
     def test_indent(self):
         input_ = '[1, 2]'
         expect = textwrap.dedent('''\
@@ -175,6 +184,7 @@ def test_indent(self):
         self.assertEqual(process.stdout, expect)
         self.assertEqual(process.stderr, '')
 
+    @force_not_colorized
     def test_no_indent(self):
         input_ = '[1,\n2]'
         expect = '[1, 2]\n'
@@ -183,6 +193,7 @@ def test_no_indent(self):
         self.assertEqual(process.stdout, expect)
         self.assertEqual(process.stderr, '')
 
+    @force_not_colorized
     def test_tab(self):
         input_ = '[1, 2]'
         expect = '[\n\t1,\n\t2\n]\n'
@@ -191,6 +202,7 @@ def test_tab(self):
         self.assertEqual(process.stdout, expect)
         self.assertEqual(process.stderr, '')
 
+    @force_not_colorized
     def test_compact(self):
         input_ = '[ 1 ,\n 2]'
         expect = '[1,2]\n'
@@ -203,7 +215,8 @@ def test_no_ensure_ascii_flag(self):
         infile = self._create_infile('{"key":"💩"}')
         outfile = os_helper.TESTFN + '.out'
         self.addCleanup(os.remove, outfile)
-        assert_python_ok('-m', self.module, '--no-ensure-ascii', infile, outfile)
+        assert_python_ok('-m', self.module, '--no-ensure-ascii', infile,
+                         outfile, PYTHON_COLORS='0')
         with open(outfile, "rb") as f:
             lines = f.read().splitlines()
         # asserting utf-8 encoded output file
@@ -214,13 +227,14 @@ def test_ensure_ascii_default(self):
         infile = self._create_infile('{"key":"💩"}')
         outfile = os_helper.TESTFN + '.out'
         self.addCleanup(os.remove, outfile)
-        assert_python_ok('-m', self.module, infile, outfile)
+        assert_python_ok('-m', self.module, infile, outfile, PYTHON_COLORS='0')
         with open(outfile, "rb") as f:
             lines = f.read().splitlines()
         # asserting an ascii encoded output file
         expected = [b'{', rb'    "key": "\ud83d\udca9"', b"}"]
         self.assertEqual(lines, expected)
 
+    @force_not_colorized
     @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows")
     def test_broken_pipe_error(self):
         cmd = [sys.executable, '-m', self.module]
@@ -232,7 +246,73 @@ def test_broken_pipe_error(self):
         proc.communicate(b'"{}"')
         self.assertEqual(proc.returncode, errno.EPIPE)
 
+    def test_colors(self):
+        infile = os_helper.TESTFN
+        self.addCleanup(os.remove, infile)
+
+        cases = (
+            ('{}', b'{}'),
+            ('[]', b'[]'),
+            ('null', b'\x1b[1;36mnull\x1b[0m'),
+            ('true', b'\x1b[1;36mtrue\x1b[0m'),
+            ('false', b'\x1b[1;36mfalse\x1b[0m'),
+            ('NaN', b'NaN'),
+            ('Infinity', b'Infinity'),
+            ('-Infinity', b'-Infinity'),
+            ('"foo"', b'\x1b[1;32m"foo"\x1b[0m'),
+            (r'" \"foo\" "', b'\x1b[1;32m" \\"foo\\" "\x1b[0m'),
+            ('"α"', b'\x1b[1;32m"\\u03b1"\x1b[0m'),
+            ('123', b'123'),
+            ('-1.2345e+23', b'-1.2345e+23'),
+            (r'{"\\": ""}',
+             b'''\
+{
+    \x1b[94m"\\\\"\x1b[0m: \x1b[1;32m""\x1b[0m
+}'''),
+            (r'{"\\\\": ""}',
+             b'''\
+{
+    \x1b[94m"\\\\\\\\"\x1b[0m: \x1b[1;32m""\x1b[0m
+}'''),
+            ('''\
+{
+    "foo": "bar",
+    "baz": 1234,
+    "qux": [true, false, null],
+    "xyz": [NaN, -Infinity, Infinity]
+}''',
+             b'''\
+{
+    \x1b[94m"foo"\x1b[0m: \x1b[1;32m"bar"\x1b[0m,
+    \x1b[94m"baz"\x1b[0m: 1234,
+    \x1b[94m"qux"\x1b[0m: [
+        \x1b[1;36mtrue\x1b[0m,
+        \x1b[1;36mfalse\x1b[0m,
+        \x1b[1;36mnull\x1b[0m
+    ],
+    \x1b[94m"xyz"\x1b[0m: [
+        NaN,
+        -Infinity,
+        Infinity
+    ]
+}'''),
+        )
+
+        for input_, expected in cases:
+            with self.subTest(input=input_):
+                with open(infile, "w", encoding="utf-8") as fp:
+                    fp.write(input_)
+                _, stdout, _ = assert_python_ok('-m', self.module, infile,
+                                                PYTHON_COLORS='1')
+                stdout = stdout.replace(b'\r\n', b'\n')  # normalize line endings
+                stdout = stdout.strip()
+                self.assertEqual(stdout, expected)
+
 
 @support.requires_subprocess()
 class TestTool(TestMain):
     module = 'json.tool'
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_pyrepl/test_unix_console.py

@@ -23,6 +23,7 @@ def unix_console(events, **kwargs):
     height = kwargs.get("height", 25)
     width = kwargs.get("width", 80)
     console.getheightwidth = MagicMock(side_effect=lambda: (height, width))
+    console.wait = MagicMock()
 
     console.prepare()
     for key, val in kwargs.items():