diff --git a/crates/ruff_linter/resources/test/fixtures/syntax_errors/async_comprehension.ipynb b/crates/ruff_linter/resources/test/fixtures/syntax_errors/async_comprehension.ipynb new file mode 100644 index 0000000000000..8232efb4d2957 --- /dev/null +++ b/crates/ruff_linter/resources/test/fixtures/syntax_errors/async_comprehension.ipynb @@ -0,0 +1,57 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "6a70904e-dbfe-441c-99ec-12e6cf57f8ba", + "metadata": {}, + "outputs": [], + "source": [ + "async def elements(n): yield n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5412fc2f-76eb-42c0-8db1-b5af6fdc46aa", + "metadata": {}, + "outputs": [], + "source": [ + "[x async for x in elements(5)] # okay, async at top level" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc3c94a7-2e64-42de-9351-260b3f41c3fd", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "[[x async for x in elements(5)] for i in range(5)] # error on 3.10, okay after" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 15f0d4f08a42b..6941fe447d5c8 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -27,7 +27,8 @@ use std::path::Path; use itertools::Itertools; use log::debug; use ruff_python_parser::semantic_errors::{ - SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError, SemanticSyntaxErrorKind, + Checkpoint, SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError, + SemanticSyntaxErrorKind, }; use rustc_hash::{FxHashMap, FxHashSet}; @@ -282,7 +283,7 @@ impl<'a> Checker<'a> { last_stmt_end: TextSize::default(), docstring_state: DocstringState::default(), target_version, - semantic_checker: SemanticSyntaxChecker::new(), + semantic_checker: SemanticSyntaxChecker::new(source_type), semantic_errors: RefCell::default(), } } @@ -525,10 +526,14 @@ impl<'a> Checker<'a> { self.target_version } - fn with_semantic_checker(&mut self, f: impl FnOnce(&mut SemanticSyntaxChecker, &Checker)) { + fn with_semantic_checker( + &mut self, + f: impl FnOnce(&mut SemanticSyntaxChecker, &Checker) -> Checkpoint, + ) -> Checkpoint { let mut checker = std::mem::take(&mut self.semantic_checker); - f(&mut checker, self); + let checkpoint = f(&mut checker, self); self.semantic_checker = checker; + checkpoint } } @@ -576,7 +581,8 @@ impl SemanticSyntaxContext for Checker<'_> { | SemanticSyntaxErrorKind::InvalidExpression(..) | SemanticSyntaxErrorKind::DuplicateMatchKey(_) | SemanticSyntaxErrorKind::DuplicateMatchClassAttribute(_) - | SemanticSyntaxErrorKind::InvalidStarExpression => { + | SemanticSyntaxErrorKind::InvalidStarExpression + | SemanticSyntaxErrorKind::AsyncComprehensionOutsideAsyncFunction(_) => { if self.settings.preview.is_enabled() { self.semantic_errors.borrow_mut().push(error); } @@ -591,7 +597,13 @@ impl SemanticSyntaxContext for Checker<'_> { impl<'a> Visitor<'a> for Checker<'a> { fn visit_stmt(&mut self, stmt: &'a Stmt) { - self.with_semantic_checker(|semantic, context| semantic.visit_stmt(stmt, context)); + // For functions, defer semantic syntax error checks until the body of the function is + // visited + let checkpoint = if stmt.is_function_def_stmt() { + None + } else { + Some(self.with_semantic_checker(|semantic, context| semantic.enter_stmt(stmt, context))) + }; // Step 0: Pre-processing self.semantic.push_node(stmt); @@ -1194,6 +1206,10 @@ impl<'a> Visitor<'a> for Checker<'a> { self.semantic.flags = flags_snapshot; self.semantic.pop_node(); self.last_stmt_end = stmt.end(); + + if let Some(checkpoint) = checkpoint { + self.semantic_checker.exit_stmt(checkpoint); + } } fn visit_annotation(&mut self, expr: &'a Expr) { @@ -1204,7 +1220,8 @@ impl<'a> Visitor<'a> for Checker<'a> { } fn visit_expr(&mut self, expr: &'a Expr) { - self.with_semantic_checker(|semantic, context| semantic.visit_expr(expr, context)); + let checkpoint = + self.with_semantic_checker(|semantic, context| semantic.enter_expr(expr, context)); // Step 0: Pre-processing if self.source_type.is_stub() @@ -1743,6 +1760,8 @@ impl<'a> Visitor<'a> for Checker<'a> { self.semantic.flags = flags_snapshot; analyze::expression(expr, self); self.semantic.pop_node(); + + self.semantic_checker.exit_expr(checkpoint); } fn visit_except_handler(&mut self, except_handler: &'a ExceptHandler) { @@ -2578,17 +2597,24 @@ impl<'a> Checker<'a> { for snapshot in deferred_functions { self.semantic.restore(snapshot); + let stmt = self.semantic.current_statement(); + let Stmt::FunctionDef(ast::StmtFunctionDef { body, parameters, .. - }) = self.semantic.current_statement() + }) = stmt else { unreachable!("Expected Stmt::FunctionDef") }; + let checkpoint = self + .with_semantic_checker(|semantic, context| semantic.enter_stmt(stmt, context)); + self.visit_parameters(parameters); // Set the docstring state before visiting the function body. self.docstring_state = DocstringState::Expected(ExpectedDocstringKind::Function); self.visit_body(body); + + self.semantic_checker.exit_stmt(checkpoint); } } self.semantic.restore(snapshot); diff --git a/crates/ruff_linter/src/linter.rs b/crates/ruff_linter/src/linter.rs index 24add1e6d8768..0f2aa8eeefe3c 100644 --- a/crates/ruff_linter/src/linter.rs +++ b/crates/ruff_linter/src/linter.rs @@ -777,14 +777,22 @@ mod tests { use std::path::Path; use anyhow::Result; + use ruff_python_ast::{PySourceType, PythonVersion}; + use ruff_python_codegen::Stylist; + use ruff_python_index::Indexer; + use ruff_python_parser::ParseOptions; + use ruff_python_trivia::textwrap::dedent; + use ruff_text_size::Ranged; use test_case::test_case; use ruff_notebook::{Notebook, NotebookError}; + use crate::linter::check_path; + use crate::message::Message; use crate::registry::Rule; use crate::source_kind::SourceKind; use crate::test::{assert_notebook_path, test_contents, TestedNotebook}; - use crate::{assert_messages, settings}; + use crate::{assert_messages, directives, settings, Locator}; /// Construct a path to a Jupyter notebook in the `resources/test/fixtures/jupyter` directory. fn notebook_path(path: impl AsRef) -> std::path::PathBuf { @@ -934,4 +942,122 @@ mod tests { } Ok(()) } + + /// Wrapper around `test_contents_syntax_errors` for testing a snippet of code instead of a + /// file. + fn test_snippet_syntax_errors( + contents: &str, + settings: &settings::LinterSettings, + ) -> Vec { + let contents = dedent(contents); + test_contents_syntax_errors( + &SourceKind::Python(contents.to_string()), + Path::new(""), + settings, + ) + } + + /// A custom test runner that prints syntax errors in addition to other diagnostics. Adapted + /// from `flakes` in pyflakes/mod.rs. + fn test_contents_syntax_errors( + source_kind: &SourceKind, + path: &Path, + settings: &settings::LinterSettings, + ) -> Vec { + let source_type = PySourceType::from(path); + let options = + ParseOptions::from(source_type).with_target_version(settings.unresolved_target_version); + let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options) + .try_into_module() + .expect("PySourceType always parses into a module"); + let locator = Locator::new(source_kind.source_code()); + let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents()); + let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents()); + let directives = directives::extract_directives( + parsed.tokens(), + directives::Flags::from_settings(settings), + &locator, + &indexer, + ); + let mut messages = check_path( + path, + None, + &locator, + &stylist, + &indexer, + &directives, + settings, + settings::flags::Noqa::Enabled, + source_kind, + source_type, + &parsed, + settings.unresolved_target_version, + ); + messages.sort_by_key(Ranged::start); + messages + } + + #[test_case( + "error_on_310", + "async def f(): return [[x async for x in foo(n)] for n in range(3)]", + PythonVersion::PY310 + )] + #[test_case( + "okay_on_311", + "async def f(): return [[x async for x in foo(n)] for n in range(3)]", + PythonVersion::PY311 + )] + #[test_case( + "okay_on_310", + "async def test(): return [[x async for x in elements(n)] async for n in range(3)]", + PythonVersion::PY310 + )] + #[test_case( + "deferred_function_body", + " + async def f(): [x for x in foo()] and [x async for x in foo()] + async def f(): + def g(): ... + [x async for x in foo()] + ", + PythonVersion::PY310 + )] + fn test_async_comprehension_in_sync_comprehension( + name: &str, + contents: &str, + python_version: PythonVersion, + ) { + let snapshot = format!("async_comprehension_in_sync_comprehension_{name}_{python_version}"); + let messages = test_snippet_syntax_errors( + contents, + &settings::LinterSettings { + rules: settings::rule_table::RuleTable::empty(), + unresolved_target_version: python_version, + preview: settings::types::PreviewMode::Enabled, + ..Default::default() + }, + ); + assert_messages!(snapshot, messages); + } + + #[test_case(PythonVersion::PY310)] + #[test_case(PythonVersion::PY311)] + fn test_async_comprehension_notebook(python_version: PythonVersion) -> Result<()> { + let snapshot = + format!("async_comprehension_in_sync_comprehension_notebook_{python_version}"); + let path = Path::new("resources/test/fixtures/syntax_errors/async_comprehension.ipynb"); + let messages = test_contents_syntax_errors( + &SourceKind::IpyNotebook(Notebook::from_path(path)?), + path, + &settings::LinterSettings { + unresolved_target_version: python_version, + rules: settings::rule_table::RuleTable::empty(), + preview: settings::types::PreviewMode::Enabled, + ..Default::default() + }, + ); + assert_messages!(snapshot, messages); + + Ok(()) + } } diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_deferred_function_body_3.10.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_deferred_function_body_3.10.snap new file mode 100644 index 0000000000000..4ba33c756c494 --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_deferred_function_body_3.10.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- + diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_error_on_310_3.10.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_error_on_310_3.10.snap new file mode 100644 index 0000000000000..86380f014a57f --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_error_on_310_3.10.snap @@ -0,0 +1,8 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- +:1:27: SyntaxError: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) + | +1 | async def f(): return [[x async for x in foo(n)] for n in range(3)] + | ^^^^^^^^^^^^^^^^^^^^^ + | diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_notebook_3.10.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_notebook_3.10.snap new file mode 100644 index 0000000000000..d55d10cfc6594 --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_notebook_3.10.snap @@ -0,0 +1,10 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- +resources/test/fixtures/syntax_errors/async_comprehension.ipynb:3:5: SyntaxError: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) + | +1 | async def elements(n): yield n +2 | [x async for x in elements(5)] # okay, async at top level +3 | [[x async for x in elements(5)] for i in range(5)] # error on 3.10, okay after + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + | diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_notebook_3.11.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_notebook_3.11.snap new file mode 100644 index 0000000000000..4ba33c756c494 --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_notebook_3.11.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- + diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_okay_on_310_3.10.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_okay_on_310_3.10.snap new file mode 100644 index 0000000000000..4ba33c756c494 --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_okay_on_310_3.10.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- + diff --git a/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_okay_on_311_3.11.snap b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_okay_on_311_3.11.snap new file mode 100644 index 0000000000000..4ba33c756c494 --- /dev/null +++ b/crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__async_comprehension_in_sync_comprehension_okay_on_311_3.11.snap @@ -0,0 +1,4 @@ +--- +source: crates/ruff_linter/src/linter.rs +--- + diff --git a/crates/ruff_python_parser/resources/inline/err/nested_async_comprehension_py310.py b/crates/ruff_python_parser/resources/inline/err/nested_async_comprehension_py310.py new file mode 100644 index 0000000000000..132dfa3c7d4f8 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/nested_async_comprehension_py310.py @@ -0,0 +1,6 @@ +# parse_options: {"target-version": "3.10"} +async def f(): return [[x async for x in foo(n)] for n in range(3)] # list +async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict +async def h(): return [{x async for x in foo(n)} for n in range(3)] # set +async def i(): return [([y async for y in range(1)], [z for z in range(2)]) for x in range(5)] +async def j(): return [([y for y in range(1)], [z async for z in range(2)]) for x in range(5)] diff --git a/crates/ruff_python_parser/resources/inline/ok/all_async_comprehension_py310.py b/crates/ruff_python_parser/resources/inline/ok/all_async_comprehension_py310.py new file mode 100644 index 0000000000000..36b12b7562857 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/ok/all_async_comprehension_py310.py @@ -0,0 +1,2 @@ +# parse_options: {"target-version": "3.10"} +async def test(): return [[x async for x in elements(n)] async for n in range(3)] diff --git a/crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py310.py b/crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py310.py new file mode 100644 index 0000000000000..3df095c1b4f3c --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py310.py @@ -0,0 +1,9 @@ +# parse_options: {"target-version": "3.10"} +# this case fails if exit_expr doesn't run +async def f(): + [_ for n in range(3)] + [_ async for n in range(3)] +# and this fails without exit_stmt +async def f(): + def g(): ... + [_ async for n in range(3)] diff --git a/crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py311.py b/crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py311.py new file mode 100644 index 0000000000000..b591eda48aa34 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py311.py @@ -0,0 +1,4 @@ +# parse_options: {"target-version": "3.11"} +async def f(): return [[x async for x in foo(n)] for n in range(3)] # list +async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict +async def h(): return [{x async for x in foo(n)} for n in range(3)] # set diff --git a/crates/ruff_python_parser/src/semantic_errors.rs b/crates/ruff_python_parser/src/semantic_errors.rs index 1ed9458280bc6..8059664b68b64 100644 --- a/crates/ruff_python_parser/src/semantic_errors.rs +++ b/crates/ruff_python_parser/src/semantic_errors.rs @@ -1,8 +1,10 @@ //! [`SemanticSyntaxChecker`] for AST-based syntax errors. //! //! This checker is not responsible for traversing the AST itself. Instead, its -//! [`SemanticSyntaxChecker::visit_stmt`] and [`SemanticSyntaxChecker::visit_expr`] methods should -//! be called in a parent `Visitor`'s `visit_stmt` and `visit_expr` methods, respectively. +//! [`SemanticSyntaxChecker::enter_stmt`] and [`SemanticSyntaxChecker::enter_expr`] methods should +//! be called in a parent `Visitor`'s `visit_stmt` and `visit_expr` methods, respectively, and +//! followed by matching calls to [`SemanticSyntaxChecker::exit_stmt`] and +//! [`SemanticSyntaxChecker::exit_expr`]. use std::fmt::Display; @@ -10,13 +12,18 @@ use ruff_python_ast::{ self as ast, comparable::ComparableExpr, visitor::{walk_expr, Visitor}, - Expr, ExprContext, IrrefutablePatternKind, Pattern, PythonVersion, Stmt, StmtExpr, - StmtImportFrom, + Expr, ExprContext, IrrefutablePatternKind, Pattern, PySourceType, PythonVersion, Stmt, + StmtExpr, StmtImportFrom, }; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustc_hash::FxHashSet; #[derive(Debug)] +pub struct Checkpoint { + in_async_context: bool, +} + +#[derive(Debug, Default)] pub struct SemanticSyntaxChecker { /// The checker has traversed past the `__future__` import boundary. /// @@ -33,12 +40,20 @@ pub struct SemanticSyntaxChecker { /// Python considers it a syntax error to import from `__future__` after any other /// non-`__future__`-importing statements. seen_futures_boundary: bool, + + /// The checker is currently in an `async` context: either the body of an `async` function or an + /// `async` comprehension. + /// + /// Note that this should be updated *after* checking the current statement or expression + /// because the parent context is what matters. + in_async_context: bool, } impl SemanticSyntaxChecker { - pub fn new() -> Self { + pub fn new(source_type: PySourceType) -> Self { Self { seen_futures_boundary: false, + in_async_context: source_type.is_ipynb(), } } } @@ -424,7 +439,27 @@ impl SemanticSyntaxChecker { } } - pub fn visit_stmt(&mut self, stmt: &ast::Stmt, ctx: &Ctx) { + /// Check `stmt` for semantic syntax errors and update the checker's internal state. + /// + /// This should be followed by a call to [`SemanticSyntaxChecker::exit_stmt`] to reset any state + /// specific to scopes introduced by `stmt`, such as whether the body of a function is async. + /// + /// Note that this method should only be called when traversing `stmt` *and* its children. For + /// example, if traversal of function bodies needs to be deferred, avoid calling `enter_stmt` on + /// the function itself until the deferred body is visited too. Failing to defer `enter_stmt` in + /// this case will break any internal state that depends on function scopes, such as `async` + /// context detection. + #[must_use] + pub fn enter_stmt( + &mut self, + stmt: &ast::Stmt, + ctx: &Ctx, + ) -> Checkpoint { + // check for errors + self.check_stmt(stmt, ctx); + + let checkpoint = self.checkpoint(); + // update internal state match stmt { Stmt::Expr(StmtExpr { value, .. }) @@ -435,26 +470,63 @@ impl SemanticSyntaxChecker { self.seen_futures_boundary = true; } } + Stmt::FunctionDef(ast::StmtFunctionDef { is_async, .. }) => { + self.in_async_context = *is_async; + self.seen_futures_boundary = true; + } _ => { self.seen_futures_boundary = true; } } - // check for errors - self.check_stmt(stmt, ctx); + checkpoint } - pub fn visit_expr(&mut self, expr: &Expr, ctx: &Ctx) { + pub fn exit_stmt(&mut self, checkpoint: Checkpoint) { + self.restore_checkpoint(checkpoint); + } + + /// Check `expr` for semantic syntax errors and update the checker's internal state. + /// + /// This should be followed by a call to [`SemanticSyntaxChecker::exit_expr`] to reset any state + /// specific to scopes introduced by `expr`, such as whether the body of a comprehension is + /// async. + #[must_use] + pub fn enter_expr(&mut self, expr: &Expr, ctx: &Ctx) -> Checkpoint { + self.check_expr(expr, ctx); + let checkpoint = self.checkpoint(); + match expr { + Expr::ListComp(ast::ExprListComp { generators, .. }) + | Expr::SetComp(ast::ExprSetComp { generators, .. }) + | Expr::DictComp(ast::ExprDictComp { generators, .. }) => { + self.in_async_context = generators.iter().any(|g| g.is_async); + } + _ => {} + } + + checkpoint + } + + pub fn exit_expr(&mut self, checkpoint: Checkpoint) { + self.restore_checkpoint(checkpoint); + } + + fn check_expr(&mut self, expr: &Expr, ctx: &Ctx) { match expr { Expr::ListComp(ast::ExprListComp { elt, generators, .. }) | Expr::SetComp(ast::ExprSetComp { elt, generators, .. - }) - | Expr::Generator(ast::ExprGenerator { + }) => { + Self::check_generator_expr(elt, generators, ctx); + self.async_comprehension_outside_async_function(ctx, generators); + } + Expr::Generator(ast::ExprGenerator { elt, generators, .. - }) => Self::check_generator_expr(elt, generators, ctx), + }) => { + Self::check_generator_expr(elt, generators, ctx); + } Expr::DictComp(ast::ExprDictComp { key, value, @@ -463,6 +535,7 @@ impl SemanticSyntaxChecker { }) => { Self::check_generator_expr(key, generators, ctx); Self::check_generator_expr(value, generators, ctx); + self.async_comprehension_outside_async_function(ctx, generators); } Expr::Name(ast::ExprName { range, @@ -574,11 +647,64 @@ impl SemanticSyntaxChecker { ); } } -} -impl Default for SemanticSyntaxChecker { - fn default() -> Self { - Self::new() + fn async_comprehension_outside_async_function( + &self, + ctx: &Ctx, + generators: &[ast::Comprehension], + ) { + let python_version = ctx.python_version(); + if python_version >= PythonVersion::PY311 { + return; + } + for generator in generators { + if generator.is_async && !self.in_async_context { + // test_ok nested_async_comprehension_py311 + // # parse_options: {"target-version": "3.11"} + // async def f(): return [[x async for x in foo(n)] for n in range(3)] # list + // async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict + // async def h(): return [{x async for x in foo(n)} for n in range(3)] # set + + // test_ok nested_async_comprehension_py310 + // # parse_options: {"target-version": "3.10"} + // # this case fails if exit_expr doesn't run + // async def f(): + // [_ for n in range(3)] + // [_ async for n in range(3)] + // # and this fails without exit_stmt + // async def f(): + // def g(): ... + // [_ async for n in range(3)] + + // test_ok all_async_comprehension_py310 + // # parse_options: {"target-version": "3.10"} + // async def test(): return [[x async for x in elements(n)] async for n in range(3)] + + // test_err nested_async_comprehension_py310 + // # parse_options: {"target-version": "3.10"} + // async def f(): return [[x async for x in foo(n)] for n in range(3)] # list + // async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict + // async def h(): return [{x async for x in foo(n)} for n in range(3)] # set + // async def i(): return [([y async for y in range(1)], [z for z in range(2)]) for x in range(5)] + // async def j(): return [([y for y in range(1)], [z async for z in range(2)]) for x in range(5)] + Self::add_error( + ctx, + SemanticSyntaxErrorKind::AsyncComprehensionOutsideAsyncFunction(python_version), + generator.range, + ); + } + } + } + + fn checkpoint(&self) -> Checkpoint { + Checkpoint { + in_async_context: self.in_async_context, + } + } + + #[allow(clippy::needless_pass_by_value)] + fn restore_checkpoint(&mut self, checkpoint: Checkpoint) { + self.in_async_context = checkpoint.in_async_context; } } @@ -650,6 +776,13 @@ impl Display for SemanticSyntaxError { SemanticSyntaxErrorKind::InvalidStarExpression => { f.write_str("can't use starred expression here") } + SemanticSyntaxErrorKind::AsyncComprehensionOutsideAsyncFunction(python_version) => { + write!( + f, + "cannot use an asynchronous comprehension outside of an asynchronous \ + function on Python {python_version} (syntax was added in 3.11)", + ) + } } } } @@ -849,6 +982,25 @@ pub enum SemanticSyntaxErrorKind { /// for *x in xs: ... /// ``` InvalidStarExpression, + + /// Represents the use of an asynchronous comprehension inside of a synchronous comprehension + /// before Python 3.11. + /// + /// ## Examples + /// + /// Before Python 3.11, code like this produces a syntax error because of the implicit function + /// scope introduced by the outer comprehension: + /// + /// ```python + /// async def elements(n): yield n + /// + /// async def test(): return { n: [x async for x in elements(n)] for n in range(3)} + /// ``` + /// + /// This was discussed in [BPO 33346] and fixed in Python 3.11. + /// + /// [BPO 33346]: https://github.com/python/cpython/issues/77527 + AsyncComprehensionOutsideAsyncFunction(PythonVersion), } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -1187,7 +1339,7 @@ pub struct SemanticSyntaxCheckerVisitor { impl SemanticSyntaxCheckerVisitor { pub fn new(context: Ctx) -> Self { Self { - checker: SemanticSyntaxChecker::new(), + checker: SemanticSyntaxChecker::new(PySourceType::Python), context, } } @@ -1202,13 +1354,15 @@ where Ctx: SemanticSyntaxContext, { fn visit_stmt(&mut self, stmt: &'_ Stmt) { - self.checker.visit_stmt(stmt, &self.context); + let checkpoint = self.checker.enter_stmt(stmt, &self.context); ruff_python_ast::visitor::walk_stmt(self, stmt); + self.checker.exit_stmt(checkpoint); } fn visit_expr(&mut self, expr: &'_ Expr) { - self.checker.visit_expr(expr, &self.context); + let checkpoint = self.checker.enter_expr(expr, &self.context); ruff_python_ast::visitor::walk_expr(self, expr); + self.checker.exit_expr(checkpoint); } } diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@nested_async_comprehension_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@nested_async_comprehension_py310.py.snap new file mode 100644 index 0000000000000..465ef924e78d6 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@nested_async_comprehension_py310.py.snap @@ -0,0 +1,823 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/nested_async_comprehension_py310.py +--- +## AST + +``` +Module( + ModModule { + range: 0..467, + body: [ + FunctionDef( + StmtFunctionDef { + range: 44..111, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("f"), + range: 54..55, + }, + type_params: None, + parameters: Parameters { + range: 55..57, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 59..111, + value: Some( + ListComp( + ExprListComp { + range: 66..111, + elt: ListComp( + ExprListComp { + range: 67..92, + elt: Name( + ExprName { + range: 68..69, + id: Name("x"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 70..91, + target: Name( + ExprName { + range: 80..81, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 85..91, + func: Name( + ExprName { + range: 85..88, + id: Name("foo"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 88..91, + args: [ + Name( + ExprName { + range: 89..90, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 93..110, + target: Name( + ExprName { + range: 97..98, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 102..110, + func: Name( + ExprName { + range: 102..107, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 107..110, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 108..109, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 122..192, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("g"), + range: 132..133, + }, + type_params: None, + parameters: Parameters { + range: 133..135, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 137..192, + value: Some( + ListComp( + ExprListComp { + range: 144..192, + elt: DictComp( + ExprDictComp { + range: 145..173, + key: Name( + ExprName { + range: 146..147, + id: Name("x"), + ctx: Load, + }, + ), + value: NumberLiteral( + ExprNumberLiteral { + range: 149..150, + value: Int( + 1, + ), + }, + ), + generators: [ + Comprehension { + range: 151..172, + target: Name( + ExprName { + range: 161..162, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 166..172, + func: Name( + ExprName { + range: 166..169, + id: Name("foo"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 169..172, + args: [ + Name( + ExprName { + range: 170..171, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 174..191, + target: Name( + ExprName { + range: 178..179, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 183..191, + func: Name( + ExprName { + range: 183..188, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 188..191, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 189..190, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 200..267, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("h"), + range: 210..211, + }, + type_params: None, + parameters: Parameters { + range: 211..213, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 215..267, + value: Some( + ListComp( + ExprListComp { + range: 222..267, + elt: SetComp( + ExprSetComp { + range: 223..248, + elt: Name( + ExprName { + range: 224..225, + id: Name("x"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 226..247, + target: Name( + ExprName { + range: 236..237, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 241..247, + func: Name( + ExprName { + range: 241..244, + id: Name("foo"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 244..247, + args: [ + Name( + ExprName { + range: 245..246, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 249..266, + target: Name( + ExprName { + range: 253..254, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 258..266, + func: Name( + ExprName { + range: 258..263, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 263..266, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 264..265, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 277..371, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("i"), + range: 287..288, + }, + type_params: None, + parameters: Parameters { + range: 288..290, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 292..371, + value: Some( + ListComp( + ExprListComp { + range: 299..371, + elt: Tuple( + ExprTuple { + range: 300..352, + elts: [ + ListComp( + ExprListComp { + range: 301..328, + elt: Name( + ExprName { + range: 302..303, + id: Name("y"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 304..327, + target: Name( + ExprName { + range: 314..315, + id: Name("y"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 319..327, + func: Name( + ExprName { + range: 319..324, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 324..327, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 325..326, + value: Int( + 1, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + ListComp( + ExprListComp { + range: 330..351, + elt: Name( + ExprName { + range: 331..332, + id: Name("z"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 333..350, + target: Name( + ExprName { + range: 337..338, + id: Name("z"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 342..350, + func: Name( + ExprName { + range: 342..347, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 347..350, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 348..349, + value: Int( + 2, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ], + ctx: Load, + parenthesized: true, + }, + ), + generators: [ + Comprehension { + range: 353..370, + target: Name( + ExprName { + range: 357..358, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 362..370, + func: Name( + ExprName { + range: 362..367, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 367..370, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 368..369, + value: Int( + 5, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 372..466, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("j"), + range: 382..383, + }, + type_params: None, + parameters: Parameters { + range: 383..385, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 387..466, + value: Some( + ListComp( + ExprListComp { + range: 394..466, + elt: Tuple( + ExprTuple { + range: 395..447, + elts: [ + ListComp( + ExprListComp { + range: 396..417, + elt: Name( + ExprName { + range: 397..398, + id: Name("y"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 399..416, + target: Name( + ExprName { + range: 403..404, + id: Name("y"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 408..416, + func: Name( + ExprName { + range: 408..413, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 413..416, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 414..415, + value: Int( + 1, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ListComp( + ExprListComp { + range: 419..446, + elt: Name( + ExprName { + range: 420..421, + id: Name("z"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 422..445, + target: Name( + ExprName { + range: 432..433, + id: Name("z"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 437..445, + func: Name( + ExprName { + range: 437..442, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 442..445, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 443..444, + value: Int( + 2, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + ], + ctx: Load, + parenthesized: true, + }, + ), + generators: [ + Comprehension { + range: 448..465, + target: Name( + ExprName { + range: 452..453, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 457..465, + func: Name( + ExprName { + range: 457..462, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 462..465, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 463..464, + value: Int( + 5, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + ], + }, +) +``` +## Semantic Syntax Errors + + | +1 | # parse_options: {"target-version": "3.10"} +2 | async def f(): return [[x async for x in foo(n)] for n in range(3)] # list + | ^^^^^^^^^^^^^^^^^^^^^ Syntax Error: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) +3 | async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict +4 | async def h(): return [{x async for x in foo(n)} for n in range(3)] # set + | + + + | +1 | # parse_options: {"target-version": "3.10"} +2 | async def f(): return [[x async for x in foo(n)] for n in range(3)] # list +3 | async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict + | ^^^^^^^^^^^^^^^^^^^^^ Syntax Error: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) +4 | async def h(): return [{x async for x in foo(n)} for n in range(3)] # set +5 | async def i(): return [([y async for y in range(1)], [z for z in range(2)]) for x in range(5)] + | + + + | +2 | async def f(): return [[x async for x in foo(n)] for n in range(3)] # list +3 | async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict +4 | async def h(): return [{x async for x in foo(n)} for n in range(3)] # set + | ^^^^^^^^^^^^^^^^^^^^^ Syntax Error: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) +5 | async def i(): return [([y async for y in range(1)], [z for z in range(2)]) for x in range(5)] +6 | async def j(): return [([y for y in range(1)], [z async for z in range(2)]) for x in range(5)] + | + + + | +3 | async def g(): return [{x: 1 async for x in foo(n)} for n in range(3)] # dict +4 | async def h(): return [{x async for x in foo(n)} for n in range(3)] # set +5 | async def i(): return [([y async for y in range(1)], [z for z in range(2)]) for x in range(5)] + | ^^^^^^^^^^^^^^^^^^^^^^^ Syntax Error: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) +6 | async def j(): return [([y for y in range(1)], [z async for z in range(2)]) for x in range(5)] + | + + + | +4 | async def h(): return [{x async for x in foo(n)} for n in range(3)] # set +5 | async def i(): return [([y async for y in range(1)], [z for z in range(2)]) for x in range(5)] +6 | async def j(): return [([y for y in range(1)], [z async for z in range(2)]) for x in range(5)] + | ^^^^^^^^^^^^^^^^^^^^^^^ Syntax Error: cannot use an asynchronous comprehension outside of an asynchronous function on Python 3.10 (syntax was added in 3.11) + | diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@all_async_comprehension_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@all_async_comprehension_py310.py.snap new file mode 100644 index 0000000000000..85cb132698689 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@all_async_comprehension_py310.py.snap @@ -0,0 +1,141 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/ok/all_async_comprehension_py310.py +--- +## AST + +``` +Module( + ModModule { + range: 0..126, + body: [ + FunctionDef( + StmtFunctionDef { + range: 44..125, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("test"), + range: 54..58, + }, + type_params: None, + parameters: Parameters { + range: 58..60, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 62..125, + value: Some( + ListComp( + ExprListComp { + range: 69..125, + elt: ListComp( + ExprListComp { + range: 70..100, + elt: Name( + ExprName { + range: 71..72, + id: Name("x"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 73..99, + target: Name( + ExprName { + range: 83..84, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 88..99, + func: Name( + ExprName { + range: 88..96, + id: Name("elements"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 96..99, + args: [ + Name( + ExprName { + range: 97..98, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 101..124, + target: Name( + ExprName { + range: 111..112, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 116..124, + func: Name( + ExprName { + range: 116..121, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 121..124, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 122..123, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + ], + }, +) +``` diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@nested_async_comprehension_py310.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@nested_async_comprehension_py310.py.snap new file mode 100644 index 0000000000000..e143cc966bc07 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@nested_async_comprehension_py310.py.snap @@ -0,0 +1,265 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py310.py +--- +## AST + +``` +Module( + ModModule { + range: 0..259, + body: [ + FunctionDef( + StmtFunctionDef { + range: 87..159, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("f"), + range: 97..98, + }, + type_params: None, + parameters: Parameters { + range: 98..100, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Expr( + StmtExpr { + range: 106..127, + value: ListComp( + ExprListComp { + range: 106..127, + elt: Name( + ExprName { + range: 107..108, + id: Name("_"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 109..126, + target: Name( + ExprName { + range: 113..114, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 118..126, + func: Name( + ExprName { + range: 118..123, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 123..126, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 124..125, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + }, + ), + Expr( + StmtExpr { + range: 132..159, + value: ListComp( + ExprListComp { + range: 132..159, + elt: Name( + ExprName { + range: 133..134, + id: Name("_"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 135..158, + target: Name( + ExprName { + range: 145..146, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 150..158, + func: Name( + ExprName { + range: 150..155, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 155..158, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 156..157, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 195..258, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("f"), + range: 205..206, + }, + type_params: None, + parameters: Parameters { + range: 206..208, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + FunctionDef( + StmtFunctionDef { + range: 214..226, + is_async: false, + decorator_list: [], + name: Identifier { + id: Name("g"), + range: 218..219, + }, + type_params: None, + parameters: Parameters { + range: 219..221, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Expr( + StmtExpr { + range: 223..226, + value: EllipsisLiteral( + ExprEllipsisLiteral { + range: 223..226, + }, + ), + }, + ), + ], + }, + ), + Expr( + StmtExpr { + range: 231..258, + value: ListComp( + ExprListComp { + range: 231..258, + elt: Name( + ExprName { + range: 232..233, + id: Name("_"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 234..257, + target: Name( + ExprName { + range: 244..245, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 249..257, + func: Name( + ExprName { + range: 249..254, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 254..257, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 255..256, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + }, + ), + ], + }, + ), + ], + }, +) +``` diff --git a/crates/ruff_python_parser/tests/snapshots/valid_syntax@nested_async_comprehension_py311.py.snap b/crates/ruff_python_parser/tests/snapshots/valid_syntax@nested_async_comprehension_py311.py.snap new file mode 100644 index 0000000000000..b0fc9d849b97b --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/valid_syntax@nested_async_comprehension_py311.py.snap @@ -0,0 +1,401 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/ok/nested_async_comprehension_py311.py +--- +## AST + +``` +Module( + ModModule { + range: 0..277, + body: [ + FunctionDef( + StmtFunctionDef { + range: 44..111, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("f"), + range: 54..55, + }, + type_params: None, + parameters: Parameters { + range: 55..57, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 59..111, + value: Some( + ListComp( + ExprListComp { + range: 66..111, + elt: ListComp( + ExprListComp { + range: 67..92, + elt: Name( + ExprName { + range: 68..69, + id: Name("x"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 70..91, + target: Name( + ExprName { + range: 80..81, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 85..91, + func: Name( + ExprName { + range: 85..88, + id: Name("foo"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 88..91, + args: [ + Name( + ExprName { + range: 89..90, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 93..110, + target: Name( + ExprName { + range: 97..98, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 102..110, + func: Name( + ExprName { + range: 102..107, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 107..110, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 108..109, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 122..192, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("g"), + range: 132..133, + }, + type_params: None, + parameters: Parameters { + range: 133..135, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 137..192, + value: Some( + ListComp( + ExprListComp { + range: 144..192, + elt: DictComp( + ExprDictComp { + range: 145..173, + key: Name( + ExprName { + range: 146..147, + id: Name("x"), + ctx: Load, + }, + ), + value: NumberLiteral( + ExprNumberLiteral { + range: 149..150, + value: Int( + 1, + ), + }, + ), + generators: [ + Comprehension { + range: 151..172, + target: Name( + ExprName { + range: 161..162, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 166..172, + func: Name( + ExprName { + range: 166..169, + id: Name("foo"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 169..172, + args: [ + Name( + ExprName { + range: 170..171, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 174..191, + target: Name( + ExprName { + range: 178..179, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 183..191, + func: Name( + ExprName { + range: 183..188, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 188..191, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 189..190, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + FunctionDef( + StmtFunctionDef { + range: 200..267, + is_async: true, + decorator_list: [], + name: Identifier { + id: Name("h"), + range: 210..211, + }, + type_params: None, + parameters: Parameters { + range: 211..213, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kwarg: None, + }, + returns: None, + body: [ + Return( + StmtReturn { + range: 215..267, + value: Some( + ListComp( + ExprListComp { + range: 222..267, + elt: SetComp( + ExprSetComp { + range: 223..248, + elt: Name( + ExprName { + range: 224..225, + id: Name("x"), + ctx: Load, + }, + ), + generators: [ + Comprehension { + range: 226..247, + target: Name( + ExprName { + range: 236..237, + id: Name("x"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 241..247, + func: Name( + ExprName { + range: 241..244, + id: Name("foo"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 244..247, + args: [ + Name( + ExprName { + range: 245..246, + id: Name("n"), + ctx: Load, + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: true, + }, + ], + }, + ), + generators: [ + Comprehension { + range: 249..266, + target: Name( + ExprName { + range: 253..254, + id: Name("n"), + ctx: Store, + }, + ), + iter: Call( + ExprCall { + range: 258..266, + func: Name( + ExprName { + range: 258..263, + id: Name("range"), + ctx: Load, + }, + ), + arguments: Arguments { + range: 263..266, + args: [ + NumberLiteral( + ExprNumberLiteral { + range: 264..265, + value: Int( + 3, + ), + }, + ), + ], + keywords: [], + }, + }, + ), + ifs: [], + is_async: false, + }, + ], + }, + ), + ), + }, + ), + ], + }, + ), + ], + }, +) +```