[syntax-errors] Start detecting compile-time syntax errors

Cargo.toml

@@ -30,6 +30,7 @@ ruff_python_index = { path = "crates/ruff_python_index" }
 ruff_python_literal = { path = "crates/ruff_python_literal" }
 ruff_python_parser = { path = "crates/ruff_python_parser" }
 ruff_python_semantic = { path = "crates/ruff_python_semantic" }
+ruff_python_syntax_errors = { path = "crates/ruff_python_syntax_errors" }
 ruff_python_stdlib = { path = "crates/ruff_python_stdlib" }
 ruff_python_trivia = { path = "crates/ruff_python_trivia" }
 ruff_server = { path = "crates/ruff_server" }

crates/ruff/tests/lint.rs

@@ -5491,3 +5491,47 @@ fn cookiecutter_globbing_no_project_root() -> Result<()> {
 
     Ok(())
 }
+
+/// Test that semantic syntax errors (1) are emitted, (2) are not cached, and (3) don't affect the
+/// reporting of normal diagnostics.
+#[test]
+fn semantic_syntax_errors() -> Result<()> {
+    let tempdir = TempDir::new()?;
+    fs::write(tempdir.path().join("main.py"), "[(x := 1) for x in foo]")?;
+
+    let mut cmd = Command::new(get_cargo_bin(BIN_NAME));
+    // inline STDIN_BASE_OPTIONS to remove --no-cache
+    cmd.args(["check", "--output-format", "concise"])
+        .arg("--preview")
+        .arg("--quiet") // suppress `debug build without --no-cache` warnings
+        .current_dir(&tempdir);
+
+    assert_cmd_snapshot!(
+        cmd,
+        @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    main.py:1:3: SyntaxError: assignment expression cannot rebind comprehension variable
+    main.py:1:20: F821 Undefined name `foo`
+
+    ----- stderr -----
+    "
+    );
+
+    // this should *not* be cached, like normal parse errors
+    assert_cmd_snapshot!(
+        cmd,
+        @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    main.py:1:3: SyntaxError: assignment expression cannot rebind comprehension variable
+    main.py:1:20: F821 Undefined name `foo`
+
+    ----- stderr -----
+    "
+    );
+
+    Ok(())
+}

crates/ruff_linter/Cargo.toml

@@ -25,6 +25,7 @@ ruff_python_index = { workspace = true }
 ruff_python_literal = { workspace = true }
 ruff_python_semantic = { workspace = true }
 ruff_python_stdlib = { workspace = true }
+ruff_python_syntax_errors = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_source_file = { workspace = true, features = ["serde"] }

crates/ruff_linter/src/checkers/ast/analyze/statement.rs

@@ -853,14 +853,6 @@ pub(crate) fn statement(stmt: &Stmt, checker: &mut Checker) {
                     if checker.enabled(Rule::FutureFeatureNotDefined) {
                         pyflakes::rules::future_feature_not_defined(checker, alias);
                     }
-                    if checker.enabled(Rule::LateFutureImport) {
-                        if checker.semantic.seen_futures_boundary() {
-                            checker.report_diagnostic(Diagnostic::new(
-                                pyflakes::rules::LateFutureImport,
-                                stmt.range(),
-                            ));
-                        }
-                    }
                 } else if &alias.name == "*" {
                     if checker.enabled(Rule::UndefinedLocalWithNestedImportStarUsage) {
                         if !matches!(checker.semantic.current_scope().kind, ScopeKind::Module) {

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -26,6 +26,7 @@ use std::path::Path;
 
 use itertools::Itertools;
 use log::debug;
+use ruff_python_syntax_errors::{SyntaxChecker, SyntaxError, SyntaxErrorKind};
 use rustc_hash::{FxHashMap, FxHashSet};
 
 use ruff_diagnostics::{Diagnostic, IsolationLevel};
@@ -63,6 +64,7 @@ use crate::importer::Importer;
 use crate::noqa::NoqaMapping;
 use crate::package::PackageRoot;
 use crate::registry::Rule;
+use crate::rules::pyflakes::rules::LateFutureImport;
 use crate::rules::{flake8_pyi, flake8_type_checking, pyflakes, pyupgrade};
 use crate::settings::{flags, LinterSettings};
 use crate::{docstrings, noqa, Locator};
@@ -225,6 +227,9 @@ pub(crate) struct Checker<'a> {
     docstring_state: DocstringState,
     /// The target [`PythonVersion`] for version-dependent checks
     target_version: PythonVersion,
+
+    #[allow(clippy::struct_field_names)]
+    syntax_checker: SyntaxChecker,
 }
 
 impl<'a> Checker<'a> {
@@ -272,6 +277,7 @@ impl<'a> Checker<'a> {
             last_stmt_end: TextSize::default(),
             docstring_state: DocstringState::default(),
             target_version,
+            syntax_checker: SyntaxChecker::new(target_version),
         }
     }
 }
@@ -516,6 +522,8 @@ impl<'a> Checker<'a> {
 
 impl<'a> Visitor<'a> for Checker<'a> {
     fn visit_stmt(&mut self, stmt: &'a Stmt) {
+        self.syntax_checker.enter_stmt(stmt);
+
         // Step 0: Pre-processing
         self.semantic.push_node(stmt);
 
@@ -550,17 +558,13 @@ impl<'a> Visitor<'a> for Checker<'a> {
                     {
                         self.semantic.flags |= SemanticModelFlags::FUTURE_ANNOTATIONS;
                     }
-                } else {
-                    self.semantic.flags |= SemanticModelFlags::FUTURES_BOUNDARY;
                 }
             }
             Stmt::Import(_) => {
                 self.semantic.flags |= SemanticModelFlags::MODULE_DOCSTRING_BOUNDARY;
-                self.semantic.flags |= SemanticModelFlags::FUTURES_BOUNDARY;
             }
             _ => {
                 self.semantic.flags |= SemanticModelFlags::MODULE_DOCSTRING_BOUNDARY;
-                self.semantic.flags |= SemanticModelFlags::FUTURES_BOUNDARY;
                 if !(self.semantic.seen_import_boundary()
                     || stmt.is_ipy_escape_command_stmt()
                     || helpers::is_assignment_to_a_dunder(stmt)
@@ -1131,6 +1135,8 @@ impl<'a> Visitor<'a> for Checker<'a> {
     }
 
     fn visit_expr(&mut self, expr: &'a Expr) {
+        self.syntax_checker.enter_expr(expr);
+
         // Step 0: Pre-processing
         if self.source_type.is_stub()
             && self.semantic.in_class_base()
@@ -2674,7 +2680,7 @@ pub(crate) fn check_ast(
     cell_offsets: Option<&CellOffsets>,
     notebook_index: Option<&NotebookIndex>,
     target_version: PythonVersion,
-) -> Vec<Diagnostic> {
+) -> (Vec<Diagnostic>, Vec<SyntaxError>) {
     let module_path = package
         .map(PackageRoot::path)
         .and_then(|package| to_module_path(package, path));
@@ -2739,5 +2745,31 @@ pub(crate) fn check_ast(
     checker.analyze.scopes.push(ScopeId::global());
     analyze::deferred_scopes(&checker);
 
-    checker.diagnostics.take()
+    let Checker {
+        diagnostics,
+        syntax_checker,
+        settings,
+        ..
+    } = checker;
+
+    // partition the semantic syntax errors into Diagnostics and regular errors
+    let mut diagnostics = diagnostics.take();
+    let mut semantic_syntax_errors = Vec::new();
+    for semantic_syntax_error in syntax_checker.finish() {
+        match semantic_syntax_error.kind {
+            SyntaxErrorKind::LateFutureImport => {
+                if settings.rules.enabled(Rule::LateFutureImport) {
+                    diagnostics.push(Diagnostic::new(
+                        LateFutureImport,
+                        semantic_syntax_error.range,
+                    ));
+                }
+            }
+            SyntaxErrorKind::ReboundComprehensionVariable => {
+                semantic_syntax_errors.push(semantic_syntax_error);
+            }
+        }
+    }
+
+    (diagnostics, semantic_syntax_errors)
 }

crates/ruff_linter/src/linter.rs

@@ -6,6 +6,7 @@ use std::path::Path;
 use anyhow::{anyhow, Result};
 use colored::Colorize;
 use itertools::Itertools;
+use ruff_python_syntax_errors::SyntaxError;
 use rustc_hash::FxHashMap;
 
 use ruff_diagnostics::Diagnostic;
@@ -77,6 +78,9 @@ pub fn check_path(
     // Aggregate all diagnostics.
     let mut diagnostics = vec![];
 
+    // Aggregate all semantic syntax errors.
+    let mut semantic_syntax_errors = vec![];
+
     let tokens = parsed.tokens();
     let comment_ranges = indexer.comment_ranges();
 
@@ -148,7 +152,7 @@ pub fn check_path(
             let cell_offsets = source_kind.as_ipy_notebook().map(Notebook::cell_offsets);
             let notebook_index = source_kind.as_ipy_notebook().map(Notebook::index);
             if use_ast {
-                diagnostics.extend(check_ast(
+                let (new_diagnostics, new_semantic_syntax_errors) = check_ast(
                     parsed,
                     locator,
                     stylist,
@@ -162,7 +166,9 @@ pub fn check_path(
                     cell_offsets,
                     notebook_index,
                     target_version,
-                ));
+                );
+                diagnostics.extend(new_diagnostics);
+                semantic_syntax_errors.extend(new_semantic_syntax_errors);
             }
             if use_imports {
                 let import_diagnostics = check_imports(
@@ -327,10 +333,17 @@ pub fn check_path(
         &[]
     };
 
+    let semantic_syntax_errors = if settings.preview.is_enabled() {
+        semantic_syntax_errors.as_slice()
+    } else {
+        &[]
+    };
+
     diagnostics_to_messages(
         diagnostics,
         parsed.errors(),
         syntax_errors,
+        semantic_syntax_errors,
         path,
         locator,
         directives,
@@ -445,8 +458,8 @@ pub fn lint_only(
     );
 
     LinterResult {
+        has_syntax_error: messages.iter().any(Message::is_syntax_error),
         messages,
-        has_syntax_error: parsed.has_syntax_errors(),
     }
 }
 
@@ -455,6 +468,7 @@ fn diagnostics_to_messages(
     diagnostics: Vec<Diagnostic>,
     parse_errors: &[ParseError],
     unsupported_syntax_errors: &[UnsupportedSyntaxError],
+    semantic_syntax_errors: &[SyntaxError],
     path: &Path,
     locator: &Locator,
     directives: &Directives,
@@ -476,6 +490,11 @@ fn diagnostics_to_messages(
         .chain(unsupported_syntax_errors.iter().map(|syntax_error| {
             Message::from_unsupported_syntax_error(syntax_error, file.deref().clone())
         }))
+        .chain(
+            semantic_syntax_errors
+                .iter()
+                .map(|error| Message::from_semantic_syntax_error(error, file.deref().clone())),
+        )
         .chain(diagnostics.into_iter().map(|diagnostic| {
             let noqa_offset = directives.noqa_line_for.resolve(diagnostic.start());
             Message::from_diagnostic(diagnostic, file.deref().clone(), noqa_offset)
@@ -547,7 +566,7 @@ pub fn lint_fix<'a>(
         );
 
         if iterations == 0 {
-            is_valid_syntax = parsed.has_no_syntax_errors();
+            is_valid_syntax = !messages.iter().any(Message::is_syntax_error);
         } else {
             // If the source code was parseable on the first pass, but is no
             // longer parseable on a subsequent pass, then we've introduced a

crates/ruff_linter/src/message/mod.rs

@@ -3,6 +3,7 @@ use std::collections::BTreeMap;
 use std::io::Write;
 use std::ops::Deref;
 
+use ruff_python_syntax_errors::SyntaxError;
 use rustc_hash::FxHashMap;
 
 pub use azure::AzureEmitter;
@@ -135,6 +136,18 @@ impl Message {
         })
     }
 
+    /// Create a [`Message`] from the given [`SyntaxError`].
+    pub fn from_semantic_syntax_error(
+        semantic_syntax_error: &SyntaxError,
+        file: SourceFile,
+    ) -> Message {
+        Message::SyntaxError(SyntaxErrorMessage {
+            message: format!("SyntaxError: {semantic_syntax_error}"),
+            range: semantic_syntax_error.range,
+            file,
+        })
+    }
+
     pub const fn as_diagnostic_message(&self) -> Option<&DiagnosticMessage> {
         match self {
             Message::Diagnostic(m) => Some(m),