Skip to content

Commit 75a562d

Browse files
ntBrentBre
authored
[syntax-errors] Parenthesized context managers before Python 3.9 (#16523)
Summary -- I thought this was very complicated based on the comment here: #16106 (comment) and on some of the discussion in the CPython issue here: python/cpython#56991. However, after a little bit of experimentation, I think it boils down to this example: ```python with (x as y): ... ``` The issue is parentheses around a `with` item with an `optional_var`, as we (and [Python](https://docs.python.org/3/library/ast.html#ast.withitem)) call the trailing variable name (`y` in this case). It's not actually about line breaks after all, except that line breaks are allowed in parenthesized expressions, which explains the validity of cases like ```pycon >>> with ( ... x, ... y ... ) as foo: ... pass ... ``` even on Python 3.8. I followed [pyright]'s example again here on the diagnostic range (just the opening paren) and the wording of the error. Test Plan -- Inline tests [pyright]: https://pyright-play.net/?pythonVersion=3.7&strict=true&code=FAdwlgLgFgBAFAewA4FMB2cBEAzBCB0EAHhJgJQwCGAzjLgmQFwz6tA
1 parent 8d3643f commit 75a562d

10 files changed

+800
-0
lines changed

‎crates/ruff_python_parser/resources/inline/err/parenthesized_context_manager_py38.py

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# parse_options: {"target-version": "3.8"}
2+
with (foo as x, bar as y): ...
3+
with (foo, bar as y): ...
4+
with (foo as x, bar): ...

‎crates/ruff_python_parser/resources/inline/err/tuple_context_manager_py38.py

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# parse_options: {"target-version": "3.8"}
2+
# these cases are _syntactically_ valid before Python 3.9 because the `with` item
3+
# is parsed as a tuple, but this will always cause a runtime error, so we flag it
4+
# anyway
5+
with (foo, bar): ...
6+
with (
7+
open('foo.txt')) as foo: ...
8+
with (
9+
foo,
10+
bar,
11+
baz,
12+
): ...
13+
with (foo,): ...

‎crates/ruff_python_parser/resources/inline/ok/parenthesized_context_manager_py39.py

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# parse_options: {"target-version": "3.9"}
2+
with (foo as x, bar as y): ...
3+
with (foo, bar as y): ...
4+
with (foo as x, bar): ...

‎crates/ruff_python_parser/resources/inline/ok/tuple_context_manager_py38.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# parse_options: {"target-version": "3.8"}
2+
with (
3+
foo,
4+
bar,
5+
baz,
6+
) as tup: ...

‎crates/ruff_python_parser/src/error.rs

+46
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,46 @@ pub enum UnsupportedSyntaxErrorKind {
661661
TypeAliasStatement,
662662
TypeParamDefault,
663663

664+
/// Represents the use of a parenthesized `with` item before Python 3.9.
665+
///
666+
/// ## Examples
667+
///
668+
/// As described in [BPO 12782], `with` uses like this were not allowed on Python 3.8:
669+
///
670+
/// ```python
671+
/// with (open("a_really_long_foo") as foo,
672+
/// open("a_really_long_bar") as bar):
673+
/// pass
674+
/// ```
675+
///
676+
/// because parentheses were not allowed within the `with` statement itself (see [this comment]
677+
/// in particular). However, parenthesized expressions were still allowed, including the cases
678+
/// below, so the issue can be pretty subtle and relates specifically to parenthesized items
679+
/// with `as` bindings.
680+
///
681+
/// ```python
682+
/// with (foo, bar): ... # okay
683+
/// with (
684+
/// open('foo.txt')) as foo: ... # also okay
685+
/// with (
686+
/// foo,
687+
/// bar,
688+
/// baz,
689+
/// ): ... # also okay, just a tuple
690+
/// with (
691+
/// foo,
692+
/// bar,
693+
/// baz,
694+
/// ) as tup: ... # also okay, binding the tuple
695+
/// ```
696+
///
697+
/// This restriction was lifted in 3.9 but formally included in the [release notes] for 3.10.
698+
///
699+
/// [BPO 12782]: https://github.com/python/cpython/issues/56991
700+
/// [this comment]: https://github.com/python/cpython/issues/56991#issuecomment-1093555141
701+
/// [release notes]: https://docs.python.org/3/whatsnew/3.10.html#summary-release-highlights
702+
ParenthesizedContextManager,
703+
664704
/// Represents the use of a [PEP 646] star expression in an index.
665705
///
666706
/// ## Examples
@@ -798,6 +838,9 @@ impl Display for UnsupportedSyntaxError {
798838
UnsupportedSyntaxErrorKind::TypeParamDefault => {
799839
"Cannot set default type for a type parameter"
800840
}
841+
UnsupportedSyntaxErrorKind::ParenthesizedContextManager => {
842+
"Cannot use parentheses within a `with` statement"
843+
}
801844
UnsupportedSyntaxErrorKind::StarExpressionInIndex => {
802845
"Cannot use star expression in index"
803846
}
@@ -861,6 +904,9 @@ impl UnsupportedSyntaxErrorKind {
861904
UnsupportedSyntaxErrorKind::TypeParameterList => Change::Added(PythonVersion::PY312),
862905
UnsupportedSyntaxErrorKind::TypeAliasStatement => Change::Added(PythonVersion::PY312),
863906
UnsupportedSyntaxErrorKind::TypeParamDefault => Change::Added(PythonVersion::PY313),
907+
UnsupportedSyntaxErrorKind::ParenthesizedContextManager => {
908+
Change::Added(PythonVersion::PY39)
909+
}
864910
UnsupportedSyntaxErrorKind::StarExpressionInIndex => {
865911
Change::Added(PythonVersion::PY311)
866912
}

‎crates/ruff_python_parser/src/parser/statement.rs

+41
Original file line numberDiff line numberDiff line change
@@ -2066,8 +2066,49 @@ impl<'src> Parser<'src> {
20662066
return vec![];
20672067
}
20682068

2069+
let open_paren_range = self.current_token_range();
2070+
20692071
if self.at(TokenKind::Lpar) {
20702072
if let Some(items) = self.try_parse_parenthesized_with_items() {
2073+
// test_ok tuple_context_manager_py38
2074+
// # parse_options: {"target-version": "3.8"}
2075+
// with (
2076+
// foo,
2077+
// bar,
2078+
// baz,
2079+
// ) as tup: ...
2080+
2081+
// test_err tuple_context_manager_py38
2082+
// # parse_options: {"target-version": "3.8"}
2083+
// # these cases are _syntactically_ valid before Python 3.9 because the `with` item
2084+
// # is parsed as a tuple, but this will always cause a runtime error, so we flag it
2085+
// # anyway
2086+
// with (foo, bar): ...
2087+
// with (
2088+
// open('foo.txt')) as foo: ...
2089+
// with (
2090+
// foo,
2091+
// bar,
2092+
// baz,
2093+
// ): ...
2094+
// with (foo,): ...
2095+
2096+
// test_ok parenthesized_context_manager_py39
2097+
// # parse_options: {"target-version": "3.9"}
2098+
// with (foo as x, bar as y): ...
2099+
// with (foo, bar as y): ...
2100+
// with (foo as x, bar): ...
2101+
2102+
// test_err parenthesized_context_manager_py38
2103+
// # parse_options: {"target-version": "3.8"}
2104+
// with (foo as x, bar as y): ...
2105+
// with (foo, bar as y): ...
2106+
// with (foo as x, bar): ...
2107+
self.add_unsupported_syntax_error(
2108+
UnsupportedSyntaxErrorKind::ParenthesizedContextManager,
2109+
open_paren_range,
2110+
);
2111+
20712112
self.expect(TokenKind::Rpar);
20722113
items
20732114
} else {

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@parenthesized_context_manager_py38.py.snap

+199
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
---
2+
source: crates/ruff_python_parser/tests/fixtures.rs
3+
input_file: crates/ruff_python_parser/resources/inline/err/parenthesized_context_manager_py38.py
4+
---
5+
## AST
6+
7+
```
8+
Module(
9+
ModModule {
10+
range: 0..126,
11+
body: [
12+
With(
13+
StmtWith {
14+
range: 43..73,
15+
is_async: false,
16+
items: [
17+
WithItem {
18+
range: 49..57,
19+
context_expr: Name(
20+
ExprName {
21+
range: 49..52,
22+
id: Name("foo"),
23+
ctx: Load,
24+
},
25+
),
26+
optional_vars: Some(
27+
Name(
28+
ExprName {
29+
range: 56..57,
30+
id: Name("x"),
31+
ctx: Store,
32+
},
33+
),
34+
),
35+
},
36+
WithItem {
37+
range: 59..67,
38+
context_expr: Name(
39+
ExprName {
40+
range: 59..62,
41+
id: Name("bar"),
42+
ctx: Load,
43+
},
44+
),
45+
optional_vars: Some(
46+
Name(
47+
ExprName {
48+
range: 66..67,
49+
id: Name("y"),
50+
ctx: Store,
51+
},
52+
),
53+
),
54+
},
55+
],
56+
body: [
57+
Expr(
58+
StmtExpr {
59+
range: 70..73,
60+
value: EllipsisLiteral(
61+
ExprEllipsisLiteral {
62+
range: 70..73,
63+
},
64+
),
65+
},
66+
),
67+
],
68+
},
69+
),
70+
With(
71+
StmtWith {
72+
range: 74..99,
73+
is_async: false,
74+
items: [
75+
WithItem {
76+
range: 80..83,
77+
context_expr: Name(
78+
ExprName {
79+
range: 80..83,
80+
id: Name("foo"),
81+
ctx: Load,
82+
},
83+
),
84+
optional_vars: None,
85+
},
86+
WithItem {
87+
range: 85..93,
88+
context_expr: Name(
89+
ExprName {
90+
range: 85..88,
91+
id: Name("bar"),
92+
ctx: Load,
93+
},
94+
),
95+
optional_vars: Some(
96+
Name(
97+
ExprName {
98+
range: 92..93,
99+
id: Name("y"),
100+
ctx: Store,
101+
},
102+
),
103+
),
104+
},
105+
],
106+
body: [
107+
Expr(
108+
StmtExpr {
109+
range: 96..99,
110+
value: EllipsisLiteral(
111+
ExprEllipsisLiteral {
112+
range: 96..99,
113+
},
114+
),
115+
},
116+
),
117+
],
118+
},
119+
),
120+
With(
121+
StmtWith {
122+
range: 100..125,
123+
is_async: false,
124+
items: [
125+
WithItem {
126+
range: 106..114,
127+
context_expr: Name(
128+
ExprName {
129+
range: 106..109,
130+
id: Name("foo"),
131+
ctx: Load,
132+
},
133+
),
134+
optional_vars: Some(
135+
Name(
136+
ExprName {
137+
range: 113..114,
138+
id: Name("x"),
139+
ctx: Store,
140+
},
141+
),
142+
),
143+
},
144+
WithItem {
145+
range: 116..119,
146+
context_expr: Name(
147+
ExprName {
148+
range: 116..119,
149+
id: Name("bar"),
150+
ctx: Load,
151+
},
152+
),
153+
optional_vars: None,
154+
},
155+
],
156+
body: [
157+
Expr(
158+
StmtExpr {
159+
range: 122..125,
160+
value: EllipsisLiteral(
161+
ExprEllipsisLiteral {
162+
range: 122..125,
163+
},
164+
),
165+
},
166+
),
167+
],
168+
},
169+
),
170+
],
171+
},
172+
)
173+
```
174+
## Unsupported Syntax Errors
175+
176+
|
177+
1 | # parse_options: {"target-version": "3.8"}
178+
2 | with (foo as x, bar as y): ...
179+
| ^ Syntax Error: Cannot use parentheses within a `with` statement on Python 3.8 (syntax was added in Python 3.9)
180+
3 | with (foo, bar as y): ...
181+
4 | with (foo as x, bar): ...
182+
|
183+
184+
185+
|
186+
1 | # parse_options: {"target-version": "3.8"}
187+
2 | with (foo as x, bar as y): ...
188+
3 | with (foo, bar as y): ...
189+
| ^ Syntax Error: Cannot use parentheses within a `with` statement on Python 3.8 (syntax was added in Python 3.9)
190+
4 | with (foo as x, bar): ...
191+
|
192+
193+
194+
|
195+
2 | with (foo as x, bar as y): ...
196+
3 | with (foo, bar as y): ...
197+
4 | with (foo as x, bar): ...
198+
| ^ Syntax Error: Cannot use parentheses within a `with` statement on Python 3.8 (syntax was added in Python 3.9)
199+
|

0 commit comments

Comments
 (0)