From 783fb15bf4ca5e7e56632994ab1993fe91fb7a9e Mon Sep 17 00:00:00 2001 From: donBarbos Date: Sun, 4 May 2025 14:04:37 +0400 Subject: [PATCH 01/12] gh-133367: Add missing options to `ast` CLI --- Doc/library/ast.rst | 21 ++++++++++++--- Doc/whatsnew/3.14.rst | 4 +++ Lib/ast.py | 26 +++++++++++++++++-- ...-05-04-13-40-05.gh-issue-133367.E5nl2u.rst | 2 ++ 4 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 776c63d1f0fda0..39211fada9b284 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -29,7 +29,7 @@ compiled into a Python code object using the built-in :func:`compile` function. .. _abstract-grammar: -Abstract Grammar +Abstract grammar ---------------- The abstract grammar is currently defined as follows: @@ -2159,7 +2159,7 @@ Async and await occurrences of the same value (e.g. :class:`ast.Add`). -:mod:`ast` Helpers +:mod:`ast` helpers ------------------ Apart from the node classes, the :mod:`ast` module defines these utility functions @@ -2484,7 +2484,7 @@ and classes for traversing abstract syntax trees: .. _ast-compiler-flags: -Compiler Flags +Compiler flags -------------- The following flags may be passed to :func:`compile` in order to change @@ -2533,7 +2533,7 @@ effects on the compilation of a program: .. _ast-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 3.9 @@ -2572,6 +2572,19 @@ The following options are accepted: Indentation of nodes in AST (number of spaces). +.. option:: --feature-version + + Minor version (int) or 3.x tuple (e.g., 3.10). + +.. option:: -o + --optimize + + Optimization level for parser. + +.. option:: --show-empty + + Show empty lists and fields in dump output. + If :file:`infile` is specified its contents are parsed to AST and dumped to stdout. Otherwise, the content is read from stdin. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 81581b30d2194b..c34520502e4829 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1991,6 +1991,10 @@ ast Use :attr:`!ast.Constant.value` instead. (Contributed by Alex Waygood in :gh:`119562`.) +* Added new ``--feature-version``, ``--optimize``, ``--show-empty`` options to + command-line interface. + (Contributed by Semyon Moroz in :gh:`133367`.) + asyncio ------- diff --git a/Lib/ast.py b/Lib/ast.py index aa788e6eb62c4f..6e638d568706b4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -643,6 +643,14 @@ def main(): 'column offsets') parser.add_argument('-i', '--indent', type=int, default=3, help='indentation of nodes (number of spaces)') + parser.add_argument('--feature-version', + type=str, default=None, metavar='VERSION', + help='minor version (int) or 3.x tuple (e.g., 3.10)') + parser.add_argument('-o', '--optimize', + type=int, default=-1, metavar='LEVEL', + help='optimization level for parser (default -1)') + parser.add_argument('--show-empty', default=False, action='store_true', + help='show empty lists and fields in dump output') args = parser.parse_args() if args.infile == '-': @@ -652,8 +660,22 @@ def main(): name = args.infile with open(args.infile, 'rb') as infile: source = infile.read() - tree = parse(source, name, args.mode, type_comments=args.no_type_comments) - print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) + + # Process feature_version + feature_version = None + if args.feature_version: + if '.' in args.feature_version: + major_minor = tuple(map(int, args.feature_version.split('.', 1))) + if len(major_minor) != 2 or major_minor[0] != 3: + parser.error("--feature-version must be 3.x tuple (e.g., 3.10)") + feature_version = major_minor + else: + feature_version = int(args.feature_version) + + tree = parse(source, name, args.mode, type_comments=args.no_type_comments, + feature_version=feature_version, optimize=args.optimize) + print(dump(tree, include_attributes=args.include_attributes, + indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': main() diff --git a/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst b/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst new file mode 100644 index 00000000000000..8b03bfc299fc16 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst @@ -0,0 +1,2 @@ +Add ``--feature-version``, ``--optimize``, ``--show-empty`` flags for +:mod:`ast` command-line interface. Patch by Semyon Moroz. From 3138d2e3ec2f68bf6dff913f6ec4cc16c71a3fb4 Mon Sep 17 00:00:00 2001 From: donBarbos Date: Sun, 4 May 2025 14:21:45 +0400 Subject: [PATCH 02/12] doc fix --- Doc/whatsnew/3.14.rst | 8 ++++---- .../2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index c34520502e4829..866b18788ccee5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -870,6 +870,10 @@ ast that the root node type is appropriate. (Contributed by Irit Katriel in :gh:`130139`.) +* Add new ``--feature-version``, ``--optimize``, ``--show-empty`` options to + command-line interface. + (Contributed by Semyon Moroz in :gh:`133367`.) + bdb --- @@ -1991,10 +1995,6 @@ ast Use :attr:`!ast.Constant.value` instead. (Contributed by Alex Waygood in :gh:`119562`.) -* Added new ``--feature-version``, ``--optimize``, ``--show-empty`` options to - command-line interface. - (Contributed by Semyon Moroz in :gh:`133367`.) - asyncio ------- diff --git a/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst b/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst index 8b03bfc299fc16..9205b584c6ce48 100644 --- a/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst +++ b/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst @@ -1,2 +1,2 @@ -Add ``--feature-version``, ``--optimize``, ``--show-empty`` flags for +Add ``--feature-version``, ``--optimize``, ``--show-empty`` options for :mod:`ast` command-line interface. Patch by Semyon Moroz. From 5a2433fa5a8d8d5df4462a421c869ec3ae0151ce Mon Sep 17 00:00:00 2001 From: donBarbos Date: Sun, 4 May 2025 15:20:27 +0400 Subject: [PATCH 03/12] Add `.. versionadded::` directive --- Doc/library/ast.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 39211fada9b284..a6a701d9daa75a 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2576,15 +2576,21 @@ The following options are accepted: Minor version (int) or 3.x tuple (e.g., 3.10). + .. versionadded:: next + .. option:: -o --optimize Optimization level for parser. + .. versionadded:: next + .. option:: --show-empty Show empty lists and fields in dump output. + .. versionadded:: next + If :file:`infile` is specified its contents are parsed to AST and dumped to stdout. Otherwise, the content is read from stdin. From 8fe372a2b7b0d1460f6a1e329b894a99328dc685 Mon Sep 17 00:00:00 2001 From: donBarbos Date: Sun, 4 May 2025 16:04:38 +0400 Subject: [PATCH 04/12] Update `--feature-version` option --- Doc/library/ast.rst | 4 ++-- Lib/ast.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index a6a701d9daa75a..4c9dcfff557334 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2574,7 +2574,7 @@ The following options are accepted: .. option:: --feature-version - Minor version (int) or 3.x tuple (e.g., 3.10). + Python version in the format 3.x (e.g., 3.10). .. versionadded:: next @@ -2587,7 +2587,7 @@ The following options are accepted: .. option:: --show-empty - Show empty lists and fields in dump output. + Show empty lists and fields that are ``None``. .. versionadded:: next diff --git a/Lib/ast.py b/Lib/ast.py index 6e638d568706b4..db4f3a8fd1c977 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -645,7 +645,7 @@ def main(): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='minor version (int) or 3.x tuple (e.g., 3.10)') + help='python version in the format 3.x (e.g., 3.10)') parser.add_argument('-o', '--optimize', type=int, default=-1, metavar='LEVEL', help='optimization level for parser (default -1)') @@ -664,13 +664,13 @@ def main(): # Process feature_version feature_version = None if args.feature_version: - if '.' in args.feature_version: - major_minor = tuple(map(int, args.feature_version.split('.', 1))) - if len(major_minor) != 2 or major_minor[0] != 3: - parser.error("--feature-version must be 3.x tuple (e.g., 3.10)") - feature_version = major_minor - else: - feature_version = int(args.feature_version) + try: + major, minor = map(int, args.feature_version.split('.', 1)) + except ValueError: + parser.error('Invalid format for --feature-version; ' + 'expected format 3.x (e.g., 3.10)') + + feature_version = (major, minor) tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) From c29610744f71a23684c701613acaa016b1b559eb Mon Sep 17 00:00:00 2001 From: Semyon Moroz Date: Sun, 4 May 2025 16:46:24 +0000 Subject: [PATCH 05/12] Apply suggestions from code review Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/ast.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 4c9dcfff557334..2af80be68841c1 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2574,10 +2574,11 @@ The following options are accepted: .. option:: --feature-version - Python version in the format 3.x (e.g., 3.10). + Python version in the format 3.x (for example, 3.10). .. versionadded:: next + .. option:: -o --optimize @@ -2585,12 +2586,14 @@ The following options are accepted: .. versionadded:: next + .. option:: --show-empty Show empty lists and fields that are ``None``. .. versionadded:: next + If :file:`infile` is specified its contents are parsed to AST and dumped to stdout. Otherwise, the content is read from stdin. From af4d1c3ac9e4803572549d1878f24c0f0457acb8 Mon Sep 17 00:00:00 2001 From: donBarbos Date: Sun, 4 May 2025 20:52:35 +0400 Subject: [PATCH 06/12] Revert "add gaps" --- Doc/library/ast.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 2af80be68841c1..35ffec866e899d 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2578,7 +2578,6 @@ The following options are accepted: .. versionadded:: next - .. option:: -o --optimize @@ -2586,7 +2585,6 @@ The following options are accepted: .. versionadded:: next - .. option:: --show-empty Show empty lists and fields that are ``None``. From de65026c242118337ca7504d04549056900ffadb Mon Sep 17 00:00:00 2001 From: donBarbos Date: Sun, 4 May 2025 22:10:22 +0400 Subject: [PATCH 07/12] Change flag alias --- Doc/library/ast.rst | 2 +- Doc/whatsnew/3.14.rst | 1 + Lib/ast.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 35ffec866e899d..a7b01d5b669186 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2578,7 +2578,7 @@ The following options are accepted: .. versionadded:: next -.. option:: -o +.. option:: -O --optimize Optimization level for parser. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 866b18788ccee5..ffbe9d48343ed6 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -874,6 +874,7 @@ ast command-line interface. (Contributed by Semyon Moroz in :gh:`133367`.) + bdb --- diff --git a/Lib/ast.py b/Lib/ast.py index db4f3a8fd1c977..c0ed8795637bbf 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -646,7 +646,7 @@ def main(): parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', help='python version in the format 3.x (e.g., 3.10)') - parser.add_argument('-o', '--optimize', + parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', help='optimization level for parser (default -1)') parser.add_argument('--show-empty', default=False, action='store_true', From 6aac4bf308402f99d8da68fe3f43d4e5fa3dea2a Mon Sep 17 00:00:00 2001 From: Semyon Moroz Date: Sun, 4 May 2025 22:27:25 +0000 Subject: [PATCH 08/12] Update Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- .../Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst b/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst index 9205b584c6ce48..1c1b0352180e70 100644 --- a/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst +++ b/Misc/NEWS.d/next/Library/2025-05-04-13-40-05.gh-issue-133367.E5nl2u.rst @@ -1,2 +1,2 @@ -Add ``--feature-version``, ``--optimize``, ``--show-empty`` options for -:mod:`ast` command-line interface. Patch by Semyon Moroz. +Add the ``--feature-version``, ``--optimize``, and ``--show-empty`` options +to the :mod:`ast` command-line interface. Patch by Semyon Moroz. From 948b453960425745bd03b47ad3787be8c6f90c39 Mon Sep 17 00:00:00 2001 From: donBarbos Date: Mon, 5 May 2025 09:56:13 +0400 Subject: [PATCH 09/12] Add tests --- Lib/test/test_ast/test_ast.py | 95 ++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 6a9b7812ef6185..ae82395e9a005a 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3272,6 +3272,9 @@ def test_invocation(self): ('--no-type-comments', '--no-type-comments'), ('-a', '--include-attributes'), ('-i=4', '--indent=4'), + ('--feature-version=3.13', '--feature-version=3.13'), + ('-O=-1', '--optimize=-1'), + ('--show-empty', '--show-empty'), ) self.set_source(''' print(1, 2, 3) @@ -3389,7 +3392,7 @@ def test_include_attributes_flag(self): self.check_output(source, expect, flag) def test_indent_flag(self): - # test 'python -m ast -i/--indent' + # test 'python -m ast -i/--indent 0' source = 'pass' expect = ''' Module( @@ -3400,6 +3403,96 @@ def test_indent_flag(self): with self.subTest(flag=flag): self.check_output(source, expect, flag) + def test_feature_version_flag(self): + # test 'python -m ast --feature-version 3.9/3.10' + source = ''' + match x: + case 1: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=1)), + body=[ + Pass()])])]) + ''' + self.check_output(source, expect, '--feature-version=3.10') + with self.assertRaises(SyntaxError): + self.invoke_ast('--feature-version=3.9') + + def test_no_optimize_flag(self): + # test 'python -m ast -O/--optimize -1/0' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=BinOp( + left=Constant(value=1), + op=Add(), + right=Constant(value=2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=-1', '--optimize=-1', '-O=0', '--optimize=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_optimize_flag(self): + # test 'python -m ast -O/--optimize 1/2' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=(1+2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=1', '--optimize=1', '-O=2', '--optimize=2'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_show_empty_flag(self): + # test 'python -m ast --show-empty' + source = 'print(1, 2, 3)' + expect = ''' + Module( + body=[ + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + keywords=[]))], + type_ignores=[]) + ''' + self.check_output(source, expect, '--show-empty') + class ASTOptimiziationTests(unittest.TestCase): def wrap_expr(self, expr): From f2d0e391aa996c36c3147bdabc7c6e1a6942850d Mon Sep 17 00:00:00 2001 From: donBarbos Date: Mon, 5 May 2025 10:27:00 +0400 Subject: [PATCH 10/12] Accept suggestions --- Doc/library/ast.rst | 4 ++-- Lib/ast.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index a7b01d5b669186..3ef931a28e9187 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1,4 +1,4 @@ -:mod:`!ast` --- Abstract Syntax Trees +:mod:`!ast` --- Abstract syntax trees ===================================== .. module:: ast @@ -2156,7 +2156,7 @@ Async and await of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree will be singletons. Changes to one will be reflected in all other - occurrences of the same value (e.g. :class:`ast.Add`). + occurrences of the same value (for example, :class:`ast.Add`). :mod:`ast` helpers diff --git a/Lib/ast.py b/Lib/ast.py index 7b1a7f4be81850..be6ed0805d63dd 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -645,7 +645,8 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='python version in the format 3.x (e.g., 3.10)') + help='Python version in the format 3.x ' + '(for example, 3.10)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', help='optimization level for parser (default -1)') @@ -668,7 +669,7 @@ def main(args=None): major, minor = map(int, args.feature_version.split('.', 1)) except ValueError: parser.error('Invalid format for --feature-version; ' - 'expected format 3.x (e.g., 3.10)') + 'expected format 3.x (for example, 3.10)') feature_version = (major, minor) From d5921da596b32737e7266ab2d3be00ecdaf5c044 Mon Sep 17 00:00:00 2001 From: donBarbos Date: Mon, 5 May 2025 20:48:48 +0400 Subject: [PATCH 11/12] Add default to options --- Doc/library/ast.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 3ef931a28e9187..eff64c5a2cf163 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2574,20 +2574,22 @@ The following options are accepted: .. option:: --feature-version - Python version in the format 3.x (for example, 3.10). + Python version in the format 3.x (for example, 3.10). Defaults to the + current version of the interpreter. .. versionadded:: next .. option:: -O --optimize - Optimization level for parser. + Optimization level for parser. Defaults to no optimization. .. versionadded:: next .. option:: --show-empty - Show empty lists and fields that are ``None``. + Show empty lists and fields that are ``None``. Defaults to showing empty + objects. .. versionadded:: next From d6d9c22a92218dc0224a499419c64eeaf70ce9c5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 5 May 2025 19:51:31 +0300 Subject: [PATCH 12/12] Update Doc/library/ast.rst --- Doc/library/ast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index eff64c5a2cf163..c9ae0abdd663c7 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2588,7 +2588,7 @@ The following options are accepted: .. option:: --show-empty - Show empty lists and fields that are ``None``. Defaults to showing empty + Show empty lists and fields that are ``None``. Defaults to not showing empty objects. .. versionadded:: next